From fb9d30224b1d885fac17708144417b092fa6b51f Mon Sep 17 00:00:00 2001 From: milan Date: Fri, 24 Jul 2026 19:04:22 +0000 Subject: [PATCH 001/124] fix(responses): forward proxy client headers to the provider Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/responses/main.py | 7 +++ litellm/responses/utils.py | 15 +++++++ .../test_responses_api_request_body.py | 45 +++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 944cf58df1c..aa5da40a02a 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -927,6 +927,13 @@ def responses( _is_async = kwargs.pop("aresponses", False) is True use_chat_completions_api = _pop_use_chat_completions_api_kw(kwargs) + client_headers = kwargs.get("headers") + extra_headers = ResponsesAPIRequestUtils.merge_client_forwarded_headers( + extra_headers=extra_headers, + client_headers=client_headers if isinstance(client_headers, dict) else None, + ) + local_vars["extra_headers"] = extra_headers + # Convert text_format to text parameter if provided text = ResponsesAPIRequestUtils.convert_text_format_to_text_param(text_format=text_format, text=text) if text is not None: diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 12c890ec91d..096103b3fae 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -89,6 +89,21 @@ class ResponsesAPIRequestUtils: ) return [*merged_input] + @staticmethod + def merge_client_forwarded_headers( + extra_headers: dict[str, Any] | None, + client_headers: dict[str, str] | None, + ) -> dict[str, Any] | None: + """ + Merge headers forwarded by the proxy (`headers` kwarg, set when + `forward_client_headers_to_llm_api` is enabled) into `extra_headers`. + + `extra_headers` wins on conflicts, since it is set explicitly by the caller. + """ + if not client_headers: + return extra_headers + return {**client_headers, **(extra_headers or {})} + @staticmethod def _check_valid_arg( supported_params: Optional[List[str]], diff --git a/tests/test_litellm/responses/test_responses_api_request_body.py b/tests/test_litellm/responses/test_responses_api_request_body.py index 44dfa240d42..0d067a99b97 100644 --- a/tests/test_litellm/responses/test_responses_api_request_body.py +++ b/tests/test_litellm/responses/test_responses_api_request_body.py @@ -259,3 +259,48 @@ async def test_aresponses_bedrock_mantle_service_tier_raises_without_drop_params mock_post.assert_not_called() assert "drop_params" in str(excinfo.value) assert "priority" in str(excinfo.value) + + +async def _aresponses_and_get_request_headers(**request_kwargs) -> dict: + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_headers_test", "gpt-4o"), 200) + + await litellm.aresponses( + model="openai/gpt-4o", + api_key="fake-api-key", + input="hi", + **request_kwargs, + ) + + mock_post.assert_called_once() + return dict(mock_post.call_args.kwargs["headers"]) + + +@pytest.mark.asyncio +async def test_aresponses_forwards_client_headers_kwarg_to_provider(): + """ + The proxy passes client headers it forwards (`forward_client_headers_to_llm_api`) + as a `headers` kwarg; those must reach the provider request. + """ + request_headers = await _aresponses_and_get_request_headers(headers={"x-my-new-header": "hello-from-client"}) + + assert request_headers["x-my-new-header"] == "hello-from-client" + + +@pytest.mark.asyncio +async def test_aresponses_merges_client_headers_with_extra_headers(): + """ + A `headers` kwarg and an explicit `extra_headers` are merged, with + `extra_headers` winning on conflicts. + """ + request_headers = await _aresponses_and_get_request_headers( + headers={"x-my-new-header": "hello-from-client", "x-shared": "from-client"}, + extra_headers={"x-explicit": "from-caller", "x-shared": "from-caller"}, + ) + + assert request_headers["x-my-new-header"] == "hello-from-client" + assert request_headers["x-explicit"] == "from-caller" + assert request_headers["x-shared"] == "from-caller" From 7716e47519426bda2ba55791e9518a0a5459a209 Mon Sep 17 00:00:00 2001 From: milan Date: Fri, 24 Jul 2026 20:16:25 +0000 Subject: [PATCH 002/124] fix(responses): compare forwarded header names case-insensitively Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/responses/utils.py | 9 ++++++++- .../responses/test_responses_api_request_body.py | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 096103b3fae..25a355edbc1 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -99,10 +99,17 @@ class ResponsesAPIRequestUtils: `forward_client_headers_to_llm_api` is enabled) into `extra_headers`. `extra_headers` wins on conflicts, since it is set explicitly by the caller. + Header names are compared case-insensitively, as HTTP defines them. """ if not client_headers: return extra_headers - return {**client_headers, **(extra_headers or {})} + if not extra_headers: + return dict(client_headers) + explicit_names = frozenset(name.lower() for name in extra_headers) + return { + **{name: value for name, value in client_headers.items() if name.lower() not in explicit_names}, + **extra_headers, + } @staticmethod def _check_valid_arg( diff --git a/tests/test_litellm/responses/test_responses_api_request_body.py b/tests/test_litellm/responses/test_responses_api_request_body.py index 0d067a99b97..1c217d1a67f 100644 --- a/tests/test_litellm/responses/test_responses_api_request_body.py +++ b/tests/test_litellm/responses/test_responses_api_request_body.py @@ -304,3 +304,18 @@ async def test_aresponses_merges_client_headers_with_extra_headers(): assert request_headers["x-my-new-header"] == "hello-from-client" assert request_headers["x-explicit"] == "from-caller" assert request_headers["x-shared"] == "from-caller" + + +@pytest.mark.asyncio +async def test_aresponses_client_header_conflict_is_case_insensitive(): + """ + HTTP header names are case-insensitive, so a differently cased client header + must not survive alongside the explicit `extra_headers` value. + """ + request_headers = await _aresponses_and_get_request_headers( + headers={"X-Shared": "from-client"}, + extra_headers={"x-shared": "from-caller"}, + ) + + assert [name for name in request_headers if name.lower() == "x-shared"] == ["x-shared"] + assert request_headers["x-shared"] == "from-caller" From e1afe2e29cee700710faa85063a9a0f7927104f6 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 28 Jul 2026 16:42:15 -0700 Subject: [PATCH 003/124] test(e2e): bound the post-/model/new servable wait at 40s _await_model_servable used poll_timeout (120s), the spend/log read-back budget. A stuck model reload therefore stalled every suite that creates a deployment for two minutes before failing Give create_model a fixed harness middle ground: model_servable_timeout=40s, polled every 2s, with each /v1/models call capped at 5s and clamped to the remaining deadline so one slow GET cannot overrun the wait. Happy path still returns on the first listing. Not derived from proxy general_settings or env Transport.get accepts an optional per-call timeout for that clamp. Unit tests cover the deadline arithmetic and clamp without a live proxy (cherry picked from commit c082a0e6488f50978bf5255f6b5298ba7e8fd8da) --- tests/e2e/proxy_client.py | 134 ++++++++++-- tests/e2e/test_proxy_client_model_servable.py | 190 ++++++++++++++++++ tests/e2e/transport.py | 13 +- 3 files changed, 313 insertions(+), 24 deletions(-) create mode 100644 tests/e2e/test_proxy_client_model_servable.py diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 6c6b948e29c..87693175e62 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -75,12 +75,95 @@ from transport import HttpTransport, SplitTransport, Transport RowsPredicate = Callable[[list[SpendLogRow]], bool] +# After /model/new, poll the data plane until the model is listed (or fail). +# Shorter than poll_timeout (spend/log read-backs ~120s); longer than a single +# request. 40s is the harness middle ground: happy path returns on the first +# poll, a stuck reload fails in under a minute instead of two. +MODEL_SERVABLE_TIMEOUT = 40.0 +MODEL_SERVABLE_INTERVAL = 2.0 +# Cap each /v1/models poll so one slow request cannot outlast the budget. +# Clamped further to remaining deadline inside await_servable. +MODEL_SERVABLE_REQUEST_TIMEOUT = 5.0 + + +@dataclass(frozen=True, slots=True) +class Servable: + """The data plane listed the model within the deadline.""" + + +@dataclass(frozen=True, slots=True) +class NotServable: + """The deadline passed without the data plane listing the model. + + `last_result` is the final /v1/models read, so the caller can tell "the proxy + answered but omitted the model" (propagation) from "the read itself failed" + (network/auth) when reporting.""" + + last_result: Result[ModelsListResponse] | None + + +ServableOutcome = Servable | NotServable + + +def await_servable( + list_models: Callable[[float], Result[ModelsListResponse]], + *, + model_name: str, + timeout: float, + interval: float, + request_timeout: float, + now: Callable[[], float], + sleep: Callable[[float], None], +) -> ServableOutcome: + """Poll `list_models` until the data plane lists `model_name` or `timeout` elapses. + + `list_models` receives the per-poll request timeout, clamped to the remaining + deadline so a slow final poll cannot overrun the overall budget. Clock and sleep + are injected so this is exercised without wall-clock waits. Always polls at least + once when the loop starts with a positive budget.""" + deadline = now() + timeout + last_result: Result[ModelsListResponse] | None = None + while True: + remaining = deadline - now() + if remaining <= 0 and last_result is not None: + return NotServable(last_result=last_result) + poll_timeout = min(request_timeout, remaining) if remaining > 0 else request_timeout + last_result = list_models(poll_timeout) + if isinstance(last_result, Success) and any( + entry.id == model_name for entry in last_result.data.data + ): + return Servable() + if now() + interval >= deadline: + return NotServable(last_result=last_result) + sleep(interval) + + +def servable_timeout_message( + *, + model_name: str, + timeout: float, + last_result: Result[ModelsListResponse] | None, +) -> str: + last_error = ( + f"; last /v1/models poll did not succeed: {last_result}" + if last_result is not None and not isinstance(last_result, Success) + else "" + ) + return ( + f"model {model_name!r} was created but never became servable on the data " + f"plane within {timeout}s of /model/new (control/data-plane propagation or " + f"STORE_MODEL_IN_DB reload issue){last_error}" + ) + @dataclass(frozen=True, slots=True) class ProxyClient: transport: Transport poll_timeout: float = 120.0 poll_interval: float = 5.0 + model_servable_timeout: float = MODEL_SERVABLE_TIMEOUT + model_servable_interval: float = MODEL_SERVABLE_INTERVAL + model_servable_request_timeout: float = MODEL_SERVABLE_REQUEST_TIMEOUT # ---- keys / customers (satisfies lifecycle.ResourceClient) ---------- @@ -167,7 +250,12 @@ class ProxyClient: this returns can race the reload and 400 with "Invalid model name passed". We therefore poll the data-plane /v1/models until the model appears before handing back, so callers can invoke it immediately. In the monolithic case - it is already present on the first poll, so this adds one request.""" + it is already present on the first poll, so this adds one request. + + The wait is bounded by `model_servable_timeout` rather than the much longer + `poll_timeout` used for batched read-backs, so a stuck reload fails in under + a minute instead of two. Happy path still returns as soon as /v1/models lists + the model (usually the first poll).""" model_id = unwrap( self.transport.post( "/model/new", @@ -185,32 +273,34 @@ class ProxyClient: def _await_model_servable(self, model_name: str) -> None: """Block until the data plane lists `model_name`, or fail loudly if it does - not within poll_timeout (a real propagation/config problem, surfaced here - instead of as a downstream "Invalid model name passed").""" - deadline = time.monotonic() + self.poll_timeout - last_result: Result[ModelsListResponse] | None = None - while time.monotonic() < deadline: - last_result = self.transport.get( + not within model_servable_timeout (a real propagation/config problem, + surfaced here instead of as a downstream "Invalid model name passed").""" + outcome = await_servable( + lambda poll_timeout: self.transport.get( "/v1/models", headers=self.transport.master, params=NoBody(), response_type=ModelsListResponse, - ) - if isinstance(last_result, Success) and any( - entry.id == model_name for entry in last_result.data.data - ): + timeout=poll_timeout, + ), + model_name=model_name, + timeout=self.model_servable_timeout, + interval=self.model_servable_interval, + request_timeout=self.model_servable_request_timeout, + now=time.monotonic, + sleep=time.sleep, + ) + match outcome: + case Servable(): return - time.sleep(self.poll_interval) - last_error = ( - f"; last /v1/models poll did not succeed: {last_result}" - if last_result is not None and not isinstance(last_result, Success) - else "" - ) - raise AssertionError( - f"model {model_name!r} was created but never became servable on the data " - f"plane within {self.poll_timeout}s of /model/new (control/data-plane " - f"propagation or STORE_MODEL_IN_DB reload issue){last_error}" - ) + case NotServable(last_result=last_result): + raise AssertionError( + servable_timeout_message( + model_name=model_name, + timeout=self.model_servable_timeout, + last_result=last_result, + ) + ) def update_model(self, model_id: str, litellm_params: LiteLLMParamsBody) -> None: """Merge `litellm_params` over the deployment `model_id`'s stored params via diff --git a/tests/e2e/test_proxy_client_model_servable.py b/tests/e2e/test_proxy_client_model_servable.py new file mode 100644 index 00000000000..0cc63f882e2 --- /dev/null +++ b/tests/e2e/test_proxy_client_model_servable.py @@ -0,0 +1,190 @@ +"""Harness coverage for the bounded wait after /model/new (no live proxy). + +Model propagation is polled to a deadline so a stuck control/data-plane reload fails +fast instead of stalling every test that creates a model. The clock and sleep are +injected, so these assert the deadline arithmetic without waiting. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from e2e_http import NetworkError, Result, Success +from models import ModelListEntry, ModelsListResponse +from proxy_client import ( + MODEL_SERVABLE_REQUEST_TIMEOUT, + MODEL_SERVABLE_TIMEOUT, + NotServable, + Servable, + await_servable, + servable_timeout_message, +) + + +def _listing(*model_names: str) -> Result[ModelsListResponse]: + return Success( + status_code=200, + data=ModelsListResponse(data=tuple(ModelListEntry(id=name) for name in model_names)), + ) + + +@dataclass(slots=True) +class FakeClock: + """A clock that only advances when the code under test sleeps or a slow poll runs.""" + + seconds: float = 0.0 + slept: list[float] = field(default_factory=list) # mutable-ok: records calls for assertions + + def now(self) -> float: + return self.seconds + + def sleep(self, duration: float) -> None: + self.slept.append(duration) + self.seconds += duration + + +@dataclass(slots=True) +class FakeModelList: + """Returns each queued /v1/models read in turn, repeating the last forever.""" + + responses: tuple[Result[ModelsListResponse], ...] + calls: int = 0 + timeouts: list[float] = field(default_factory=list) # mutable-ok: records call timeouts + + def __call__(self, request_timeout: float) -> Result[ModelsListResponse]: + self.timeouts.append(request_timeout) + response = self.responses[min(self.calls, len(self.responses) - 1)] + self.calls += 1 + return response + + +def test_returns_servable_on_first_listing_without_sleeping() -> None: + clock = FakeClock() + list_models = FakeModelList(responses=(_listing("my-model"),)) + + outcome = await_servable( + list_models, + model_name="my-model", + timeout=40.0, + interval=2.0, + request_timeout=5.0, + now=clock.now, + sleep=clock.sleep, + ) + + assert outcome == Servable() + assert list_models.calls == 1 + assert list_models.timeouts == [5.0] + assert clock.slept == [] + + +def test_polls_until_the_model_appears() -> None: + clock = FakeClock() + list_models = FakeModelList(responses=(_listing("other"), _listing("other"), _listing("other", "my-model"))) + + outcome = await_servable( + list_models, + model_name="my-model", + timeout=40.0, + interval=2.0, + request_timeout=5.0, + now=clock.now, + sleep=clock.sleep, + ) + + assert outcome == Servable() + assert list_models.calls == 3 + assert clock.seconds == 4.0 + + +def test_gives_up_at_the_deadline_rather_than_polling_forever() -> None: + clock = FakeClock() + list_models = FakeModelList(responses=(_listing("other"),)) + + outcome = await_servable( + list_models, + model_name="my-model", + timeout=10.0, + interval=2.0, + request_timeout=5.0, + now=clock.now, + sleep=clock.sleep, + ) + + assert isinstance(outcome, NotServable) + assert clock.seconds == 8.0 + assert list_models.calls == 5 + assert list_models.timeouts == [5.0, 5.0, 5.0, 4.0, 2.0] + + +def test_does_not_wait_past_the_overall_budget() -> None: + clock = FakeClock() + + outcome = await_servable( + FakeModelList(responses=(_listing("other"),)), + model_name="my-model", + timeout=MODEL_SERVABLE_TIMEOUT, + interval=2.0, + request_timeout=MODEL_SERVABLE_REQUEST_TIMEOUT, + now=clock.now, + sleep=clock.sleep, + ) + + assert isinstance(outcome, NotServable) + assert clock.seconds <= MODEL_SERVABLE_TIMEOUT + + +def test_clamps_request_timeout_to_remaining_deadline() -> None: + """A slow final poll must not receive the full request cap when less budget remains. + + Without the clamp, remaining=3 and cap=5 lets the transport block for 5s and the + overall wait overruns model_servable_timeout by up to ~cap seconds. + """ + clock = FakeClock() + timeouts: list[float] = [] + + def list_models(request_timeout: float) -> Result[ModelsListResponse]: + timeouts.append(request_timeout) + clock.seconds += request_timeout + return _listing("other") + + outcome = await_servable( + list_models, + model_name="my-model", + timeout=10.0, + interval=2.0, + request_timeout=5.0, + now=clock.now, + sleep=clock.sleep, + ) + + assert isinstance(outcome, NotServable) + assert timeouts[0] == 5.0 + assert any(timeout < 5.0 for timeout in timeouts) + assert timeouts[-1] == 3.0 + assert clock.seconds <= 10.0 + + +def test_reports_a_failed_read_distinctly_from_a_missing_model() -> None: + clock = FakeClock() + unreachable: Result[ModelsListResponse] = NetworkError(message="connection refused") + + outcome = await_servable( + FakeModelList(responses=(unreachable,)), + model_name="my-model", + timeout=1.0, + interval=0.5, + request_timeout=5.0, + now=clock.now, + sleep=clock.sleep, + ) + + assert outcome == NotServable(last_result=unreachable) + message = servable_timeout_message(model_name="my-model", timeout=1.0, last_result=unreachable) + assert "connection refused" in message + + listed_without_model = _listing("other") + propagation_message = servable_timeout_message( + model_name="my-model", timeout=1.0, last_result=listed_without_model + ) + assert "did not succeed" not in propagation_message diff --git a/tests/e2e/transport.py b/tests/e2e/transport.py index a6adf83ed1f..27b11befc8e 100644 --- a/tests/e2e/transport.py +++ b/tests/e2e/transport.py @@ -58,6 +58,7 @@ class Transport(Protocol): headers: BaseModel, params: BaseModel, response_type: type[R], + timeout: float | None = None, ) -> Result[R]: ... def delete[R: BaseModel]( @@ -136,13 +137,16 @@ class HttpTransport: headers: BaseModel, params: BaseModel, response_type: type[R], + timeout: float | None = None, ) -> Result[R]: + """`timeout` overrides the transport-wide request_timeout for this call, for + pollers whose own deadline is shorter than it.""" return e2e_http.get( self._url(path), headers=headers, params=params, response_type=response_type, - timeout=self.request_timeout, + timeout=self.request_timeout if timeout is None else timeout, ) def delete[R: BaseModel]( @@ -336,9 +340,14 @@ class SplitTransport: headers: BaseModel, params: BaseModel, response_type: type[R], + timeout: float | None = None, ) -> Result[R]: return self._route(path).get( - path, headers=headers, params=params, response_type=response_type + path, + headers=headers, + params=params, + response_type=response_type, + timeout=timeout, ) def delete[R: BaseModel]( From 5aa66ea33e6c194f66d360e22292f22eddac38e7 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 28 Jul 2026 17:09:20 -0700 Subject: [PATCH 004/124] fix(e2e): wait one default DB reload interval of continuous listing create_model returned after the first /v1/models hit that listed the model, so chat could still land on a cold gateway worker (numWorkers>1 / peer pod) and 400 Invalid model name. Require continuous listing for the product default add_deployment interval (30s) after first sight so every worker has synced from the DB; first listing still bounded at 40s (cherry picked from commit 7d1ee2ff861b970f6de3f6759ff015947af9d2a1) --- tests/e2e/proxy_client.py | 85 ++++++++---- tests/e2e/test_proxy_client_model_servable.py | 125 ++++++++++++------ 2 files changed, 148 insertions(+), 62 deletions(-) diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 87693175e62..dfbb90ac08e 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -75,14 +75,17 @@ from transport import HttpTransport, SplitTransport, Transport RowsPredicate = Callable[[list[SpendLogRow]], bool] -# After /model/new, poll the data plane until the model is listed (or fail). -# Shorter than poll_timeout (spend/log read-backs ~120s); longer than a single -# request. 40s is the harness middle ground: happy path returns on the first -# poll, a stuck reload fails in under a minute instead of two. +# After /model/new, the control-plane writer reloads itself immediately, but every +# other gateway worker (and peer pod) only picks the model up on its add_deployment +# job. That job runs every proxy_config_reload_interval_seconds (product default 30). +# A single /v1/models hit can land on a hot worker while the next /chat hits a cold +# one ("Invalid model name"). Wait for first listing within MODEL_SERVABLE_TIMEOUT, +# then require continuous listing for MODEL_SERVABLE_DB_SYNC_SECONDS (the default +# reload interval) so every worker has had a chance to sync from the DB. MODEL_SERVABLE_TIMEOUT = 40.0 +MODEL_SERVABLE_DB_SYNC_SECONDS = 30.0 MODEL_SERVABLE_INTERVAL = 2.0 -# Cap each /v1/models poll so one slow request cannot outlast the budget. -# Clamped further to remaining deadline inside await_servable. +# Cap each /v1/models poll so one slow request cannot outlast the remaining budget. MODEL_SERVABLE_REQUEST_TIMEOUT = 5.0 @@ -112,29 +115,55 @@ def await_servable( timeout: float, interval: float, request_timeout: float, + db_sync_seconds: float, now: Callable[[], float], sleep: Callable[[float], None], ) -> ServableOutcome: - """Poll `list_models` until the data plane lists `model_name` or `timeout` elapses. + """Poll until `model_name` is listed long enough for every worker to DB-sync. - `list_models` receives the per-poll request timeout, clamped to the remaining - deadline so a slow final poll cannot overrun the overall budget. Clock and sleep - are injected so this is exercised without wall-clock waits. Always polls at least - once when the loop starts with a positive budget.""" - deadline = now() + timeout + First listing must happen within `timeout`. After that, the model must stay + listed continuously for `db_sync_seconds` (any miss resets the continuous + window). `db_sync_seconds=0` returns on the first listing. Each poll's request + timeout is clamped to the remaining budget. Clock and sleep are injected.""" + started = now() + first_seen_at: float | None = None last_result: Result[ModelsListResponse] | None = None while True: - remaining = deadline - now() + t = now() + if first_seen_at is None: + deadline = started + timeout + else: + deadline = first_seen_at + db_sync_seconds + remaining = deadline - t if remaining <= 0 and last_result is not None: + if first_seen_at is not None and db_sync_seconds <= 0: + return Servable() + if first_seen_at is not None and t - first_seen_at >= db_sync_seconds: + return Servable() return NotServable(last_result=last_result) poll_timeout = min(request_timeout, remaining) if remaining > 0 else request_timeout last_result = list_models(poll_timeout) - if isinstance(last_result, Success) and any( + listed = isinstance(last_result, Success) and any( entry.id == model_name for entry in last_result.data.data - ): + ) + t = now() + if not listed: + first_seen_at = None + elif first_seen_at is None: + first_seen_at = t + if db_sync_seconds <= 0: + return Servable() + elif t - first_seen_at >= db_sync_seconds: return Servable() - if now() + interval >= deadline: - return NotServable(last_result=last_result) + if first_seen_at is None: + if now() + interval >= started + timeout: + return NotServable(last_result=last_result) + elif now() + interval >= first_seen_at + db_sync_seconds: + # Final stretch: sleep only the remainder of the continuous window. + remainder = first_seen_at + db_sync_seconds - now() + if remainder > 0: + sleep(remainder) + continue sleep(interval) @@ -142,6 +171,7 @@ def servable_timeout_message( *, model_name: str, timeout: float, + db_sync_seconds: float, last_result: Result[ModelsListResponse] | None, ) -> str: last_error = ( @@ -151,7 +181,8 @@ def servable_timeout_message( ) return ( f"model {model_name!r} was created but never became servable on the data " - f"plane within {timeout}s of /model/new (control/data-plane propagation or " + f"plane within {timeout}s of first listing (plus {db_sync_seconds}s continuous " + f"DB sync) after /model/new (control/data-plane propagation or " f"STORE_MODEL_IN_DB reload issue){last_error}" ) @@ -162,6 +193,7 @@ class ProxyClient: poll_timeout: float = 120.0 poll_interval: float = 5.0 model_servable_timeout: float = MODEL_SERVABLE_TIMEOUT + model_servable_db_sync_seconds: float = MODEL_SERVABLE_DB_SYNC_SECONDS model_servable_interval: float = MODEL_SERVABLE_INTERVAL model_servable_request_timeout: float = MODEL_SERVABLE_REQUEST_TIMEOUT @@ -252,10 +284,10 @@ class ProxyClient: handing back, so callers can invoke it immediately. In the monolithic case it is already present on the first poll, so this adds one request. - The wait is bounded by `model_servable_timeout` rather than the much longer - `poll_timeout` used for batched read-backs, so a stuck reload fails in under - a minute instead of two. Happy path still returns as soon as /v1/models lists - the model (usually the first poll).""" + First listing must arrive within `model_servable_timeout` (not the longer + spend `poll_timeout`). The model must then stay listed for + `model_servable_db_sync_seconds` (product default DB reload interval) so every + gateway worker has run add_deployment before callers use the model.""" model_id = unwrap( self.transport.post( "/model/new", @@ -272,9 +304,10 @@ class ProxyClient: return model_id def _await_model_servable(self, model_name: str) -> None: - """Block until the data plane lists `model_name`, or fail loudly if it does - not within model_servable_timeout (a real propagation/config problem, - surfaced here instead of as a downstream "Invalid model name passed").""" + """Block until the data plane lists `model_name` long enough for DB sync. + + Fails if first listing misses model_servable_timeout, or if continuous listing + for model_servable_db_sync_seconds never holds (multi-worker / peer reload).""" outcome = await_servable( lambda poll_timeout: self.transport.get( "/v1/models", @@ -287,6 +320,7 @@ class ProxyClient: timeout=self.model_servable_timeout, interval=self.model_servable_interval, request_timeout=self.model_servable_request_timeout, + db_sync_seconds=self.model_servable_db_sync_seconds, now=time.monotonic, sleep=time.sleep, ) @@ -298,6 +332,7 @@ class ProxyClient: servable_timeout_message( model_name=model_name, timeout=self.model_servable_timeout, + db_sync_seconds=self.model_servable_db_sync_seconds, last_result=last_result, ) ) diff --git a/tests/e2e/test_proxy_client_model_servable.py b/tests/e2e/test_proxy_client_model_servable.py index 0cc63f882e2..c9edd148c2b 100644 --- a/tests/e2e/test_proxy_client_model_servable.py +++ b/tests/e2e/test_proxy_client_model_servable.py @@ -1,8 +1,9 @@ """Harness coverage for the bounded wait after /model/new (no live proxy). -Model propagation is polled to a deadline so a stuck control/data-plane reload fails -fast instead of stalling every test that creates a model. The clock and sleep are -injected, so these assert the deadline arithmetic without waiting. +create_model must wait for the product default DB reload interval of continuous +listing so multi-worker gateways finish add_deployment before callers use the +model. Clock and sleep are injected so these assert the deadline arithmetic +without wall-clock waits. """ from __future__ import annotations @@ -12,6 +13,7 @@ from dataclasses import dataclass, field from e2e_http import NetworkError, Result, Success from models import ModelListEntry, ModelsListResponse from proxy_client import ( + MODEL_SERVABLE_DB_SYNC_SECONDS, MODEL_SERVABLE_REQUEST_TIMEOUT, MODEL_SERVABLE_TIMEOUT, NotServable, @@ -30,8 +32,6 @@ def _listing(*model_names: str) -> Result[ModelsListResponse]: @dataclass(slots=True) class FakeClock: - """A clock that only advances when the code under test sleeps or a slow poll runs.""" - seconds: float = 0.0 slept: list[float] = field(default_factory=list) # mutable-ok: records calls for assertions @@ -45,8 +45,6 @@ class FakeClock: @dataclass(slots=True) class FakeModelList: - """Returns each queued /v1/models read in turn, repeating the last forever.""" - responses: tuple[Result[ModelsListResponse], ...] calls: int = 0 timeouts: list[float] = field(default_factory=list) # mutable-ok: records call timeouts @@ -58,7 +56,7 @@ class FakeModelList: return response -def test_returns_servable_on_first_listing_without_sleeping() -> None: +def test_returns_on_first_listing_when_db_sync_is_zero() -> None: clock = FakeClock() list_models = FakeModelList(responses=(_listing("my-model"),)) @@ -68,17 +66,64 @@ def test_returns_servable_on_first_listing_without_sleeping() -> None: timeout=40.0, interval=2.0, request_timeout=5.0, + db_sync_seconds=0.0, now=clock.now, sleep=clock.sleep, ) assert outcome == Servable() assert list_models.calls == 1 - assert list_models.timeouts == [5.0] assert clock.slept == [] -def test_polls_until_the_model_appears() -> None: +def test_requires_continuous_listing_for_default_db_sync_interval() -> None: + clock = FakeClock() + list_models = FakeModelList(responses=(_listing("my-model"),)) + + outcome = await_servable( + list_models, + model_name="my-model", + timeout=40.0, + interval=2.0, + request_timeout=5.0, + db_sync_seconds=MODEL_SERVABLE_DB_SYNC_SECONDS, + now=clock.now, + sleep=clock.sleep, + ) + + assert outcome == Servable() + assert clock.seconds >= MODEL_SERVABLE_DB_SYNC_SECONDS + assert list_models.calls >= 2 + + +def test_resets_db_sync_window_when_a_poll_misses() -> None: + clock = FakeClock() + list_models = FakeModelList( + responses=( + _listing("my-model"), + _listing("my-model"), + _listing("other"), + _listing("my-model"), + ) + ) + + outcome = await_servable( + list_models, + model_name="my-model", + timeout=40.0, + interval=2.0, + request_timeout=5.0, + db_sync_seconds=6.0, + now=clock.now, + sleep=clock.sleep, + ) + + assert outcome == Servable() + assert list_models.calls >= 4 + assert clock.seconds >= 6.0 + + +def test_polls_until_the_model_first_appears() -> None: clock = FakeClock() list_models = FakeModelList(responses=(_listing("other"), _listing("other"), _listing("other", "my-model"))) @@ -88,6 +133,7 @@ def test_polls_until_the_model_appears() -> None: timeout=40.0, interval=2.0, request_timeout=5.0, + db_sync_seconds=0.0, now=clock.now, sleep=clock.sleep, ) @@ -97,7 +143,7 @@ def test_polls_until_the_model_appears() -> None: assert clock.seconds == 4.0 -def test_gives_up_at_the_deadline_rather_than_polling_forever() -> None: +def test_gives_up_if_first_listing_never_arrives() -> None: clock = FakeClock() list_models = FakeModelList(responses=(_listing("other"),)) @@ -107,6 +153,7 @@ def test_gives_up_at_the_deadline_rather_than_polling_forever() -> None: timeout=10.0, interval=2.0, request_timeout=5.0, + db_sync_seconds=30.0, now=clock.now, sleep=clock.sleep, ) @@ -114,32 +161,9 @@ def test_gives_up_at_the_deadline_rather_than_polling_forever() -> None: assert isinstance(outcome, NotServable) assert clock.seconds == 8.0 assert list_models.calls == 5 - assert list_models.timeouts == [5.0, 5.0, 5.0, 4.0, 2.0] - - -def test_does_not_wait_past_the_overall_budget() -> None: - clock = FakeClock() - - outcome = await_servable( - FakeModelList(responses=(_listing("other"),)), - model_name="my-model", - timeout=MODEL_SERVABLE_TIMEOUT, - interval=2.0, - request_timeout=MODEL_SERVABLE_REQUEST_TIMEOUT, - now=clock.now, - sleep=clock.sleep, - ) - - assert isinstance(outcome, NotServable) - assert clock.seconds <= MODEL_SERVABLE_TIMEOUT def test_clamps_request_timeout_to_remaining_deadline() -> None: - """A slow final poll must not receive the full request cap when less budget remains. - - Without the clamp, remaining=3 and cap=5 lets the transport block for 5s and the - overall wait overruns model_servable_timeout by up to ~cap seconds. - """ clock = FakeClock() timeouts: list[float] = [] @@ -154,6 +178,7 @@ def test_clamps_request_timeout_to_remaining_deadline() -> None: timeout=10.0, interval=2.0, request_timeout=5.0, + db_sync_seconds=30.0, now=clock.now, sleep=clock.sleep, ) @@ -161,10 +186,27 @@ def test_clamps_request_timeout_to_remaining_deadline() -> None: assert isinstance(outcome, NotServable) assert timeouts[0] == 5.0 assert any(timeout < 5.0 for timeout in timeouts) - assert timeouts[-1] == 3.0 assert clock.seconds <= 10.0 +def test_does_not_wait_past_first_listing_budget_when_missing() -> None: + clock = FakeClock() + + outcome = await_servable( + FakeModelList(responses=(_listing("other"),)), + model_name="my-model", + timeout=MODEL_SERVABLE_TIMEOUT, + interval=2.0, + request_timeout=MODEL_SERVABLE_REQUEST_TIMEOUT, + db_sync_seconds=MODEL_SERVABLE_DB_SYNC_SECONDS, + now=clock.now, + sleep=clock.sleep, + ) + + assert isinstance(outcome, NotServable) + assert clock.seconds <= MODEL_SERVABLE_TIMEOUT + + def test_reports_a_failed_read_distinctly_from_a_missing_model() -> None: clock = FakeClock() unreachable: Result[ModelsListResponse] = NetworkError(message="connection refused") @@ -175,16 +217,25 @@ def test_reports_a_failed_read_distinctly_from_a_missing_model() -> None: timeout=1.0, interval=0.5, request_timeout=5.0, + db_sync_seconds=0.0, now=clock.now, sleep=clock.sleep, ) assert outcome == NotServable(last_result=unreachable) - message = servable_timeout_message(model_name="my-model", timeout=1.0, last_result=unreachable) + message = servable_timeout_message( + model_name="my-model", + timeout=1.0, + db_sync_seconds=0.0, + last_result=unreachable, + ) assert "connection refused" in message listed_without_model = _listing("other") propagation_message = servable_timeout_message( - model_name="my-model", timeout=1.0, last_result=listed_without_model + model_name="my-model", + timeout=1.0, + db_sync_seconds=30.0, + last_result=listed_without_model, ) assert "did not succeed" not in propagation_message From 5953a66eab8212beac24d3265cd118b7f4a51176 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 28 Jul 2026 17:10:04 -0700 Subject: [PATCH 005/124] test(e2e): drop proxy_client model-servable unit tests Keep the create_model DB-sync wait in the harness; the pure-function unit file is not needed for this PR (cherry picked from commit 89204651d1a4537c6f21550c5ab85448ae0923f8) --- tests/e2e/test_proxy_client_model_servable.py | 241 ------------------ 1 file changed, 241 deletions(-) delete mode 100644 tests/e2e/test_proxy_client_model_servable.py diff --git a/tests/e2e/test_proxy_client_model_servable.py b/tests/e2e/test_proxy_client_model_servable.py deleted file mode 100644 index c9edd148c2b..00000000000 --- a/tests/e2e/test_proxy_client_model_servable.py +++ /dev/null @@ -1,241 +0,0 @@ -"""Harness coverage for the bounded wait after /model/new (no live proxy). - -create_model must wait for the product default DB reload interval of continuous -listing so multi-worker gateways finish add_deployment before callers use the -model. Clock and sleep are injected so these assert the deadline arithmetic -without wall-clock waits. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field - -from e2e_http import NetworkError, Result, Success -from models import ModelListEntry, ModelsListResponse -from proxy_client import ( - MODEL_SERVABLE_DB_SYNC_SECONDS, - MODEL_SERVABLE_REQUEST_TIMEOUT, - MODEL_SERVABLE_TIMEOUT, - NotServable, - Servable, - await_servable, - servable_timeout_message, -) - - -def _listing(*model_names: str) -> Result[ModelsListResponse]: - return Success( - status_code=200, - data=ModelsListResponse(data=tuple(ModelListEntry(id=name) for name in model_names)), - ) - - -@dataclass(slots=True) -class FakeClock: - seconds: float = 0.0 - slept: list[float] = field(default_factory=list) # mutable-ok: records calls for assertions - - def now(self) -> float: - return self.seconds - - def sleep(self, duration: float) -> None: - self.slept.append(duration) - self.seconds += duration - - -@dataclass(slots=True) -class FakeModelList: - responses: tuple[Result[ModelsListResponse], ...] - calls: int = 0 - timeouts: list[float] = field(default_factory=list) # mutable-ok: records call timeouts - - def __call__(self, request_timeout: float) -> Result[ModelsListResponse]: - self.timeouts.append(request_timeout) - response = self.responses[min(self.calls, len(self.responses) - 1)] - self.calls += 1 - return response - - -def test_returns_on_first_listing_when_db_sync_is_zero() -> None: - clock = FakeClock() - list_models = FakeModelList(responses=(_listing("my-model"),)) - - outcome = await_servable( - list_models, - model_name="my-model", - timeout=40.0, - interval=2.0, - request_timeout=5.0, - db_sync_seconds=0.0, - now=clock.now, - sleep=clock.sleep, - ) - - assert outcome == Servable() - assert list_models.calls == 1 - assert clock.slept == [] - - -def test_requires_continuous_listing_for_default_db_sync_interval() -> None: - clock = FakeClock() - list_models = FakeModelList(responses=(_listing("my-model"),)) - - outcome = await_servable( - list_models, - model_name="my-model", - timeout=40.0, - interval=2.0, - request_timeout=5.0, - db_sync_seconds=MODEL_SERVABLE_DB_SYNC_SECONDS, - now=clock.now, - sleep=clock.sleep, - ) - - assert outcome == Servable() - assert clock.seconds >= MODEL_SERVABLE_DB_SYNC_SECONDS - assert list_models.calls >= 2 - - -def test_resets_db_sync_window_when_a_poll_misses() -> None: - clock = FakeClock() - list_models = FakeModelList( - responses=( - _listing("my-model"), - _listing("my-model"), - _listing("other"), - _listing("my-model"), - ) - ) - - outcome = await_servable( - list_models, - model_name="my-model", - timeout=40.0, - interval=2.0, - request_timeout=5.0, - db_sync_seconds=6.0, - now=clock.now, - sleep=clock.sleep, - ) - - assert outcome == Servable() - assert list_models.calls >= 4 - assert clock.seconds >= 6.0 - - -def test_polls_until_the_model_first_appears() -> None: - clock = FakeClock() - list_models = FakeModelList(responses=(_listing("other"), _listing("other"), _listing("other", "my-model"))) - - outcome = await_servable( - list_models, - model_name="my-model", - timeout=40.0, - interval=2.0, - request_timeout=5.0, - db_sync_seconds=0.0, - now=clock.now, - sleep=clock.sleep, - ) - - assert outcome == Servable() - assert list_models.calls == 3 - assert clock.seconds == 4.0 - - -def test_gives_up_if_first_listing_never_arrives() -> None: - clock = FakeClock() - list_models = FakeModelList(responses=(_listing("other"),)) - - outcome = await_servable( - list_models, - model_name="my-model", - timeout=10.0, - interval=2.0, - request_timeout=5.0, - db_sync_seconds=30.0, - now=clock.now, - sleep=clock.sleep, - ) - - assert isinstance(outcome, NotServable) - assert clock.seconds == 8.0 - assert list_models.calls == 5 - - -def test_clamps_request_timeout_to_remaining_deadline() -> None: - clock = FakeClock() - timeouts: list[float] = [] - - def list_models(request_timeout: float) -> Result[ModelsListResponse]: - timeouts.append(request_timeout) - clock.seconds += request_timeout - return _listing("other") - - outcome = await_servable( - list_models, - model_name="my-model", - timeout=10.0, - interval=2.0, - request_timeout=5.0, - db_sync_seconds=30.0, - now=clock.now, - sleep=clock.sleep, - ) - - assert isinstance(outcome, NotServable) - assert timeouts[0] == 5.0 - assert any(timeout < 5.0 for timeout in timeouts) - assert clock.seconds <= 10.0 - - -def test_does_not_wait_past_first_listing_budget_when_missing() -> None: - clock = FakeClock() - - outcome = await_servable( - FakeModelList(responses=(_listing("other"),)), - model_name="my-model", - timeout=MODEL_SERVABLE_TIMEOUT, - interval=2.0, - request_timeout=MODEL_SERVABLE_REQUEST_TIMEOUT, - db_sync_seconds=MODEL_SERVABLE_DB_SYNC_SECONDS, - now=clock.now, - sleep=clock.sleep, - ) - - assert isinstance(outcome, NotServable) - assert clock.seconds <= MODEL_SERVABLE_TIMEOUT - - -def test_reports_a_failed_read_distinctly_from_a_missing_model() -> None: - clock = FakeClock() - unreachable: Result[ModelsListResponse] = NetworkError(message="connection refused") - - outcome = await_servable( - FakeModelList(responses=(unreachable,)), - model_name="my-model", - timeout=1.0, - interval=0.5, - request_timeout=5.0, - db_sync_seconds=0.0, - now=clock.now, - sleep=clock.sleep, - ) - - assert outcome == NotServable(last_result=unreachable) - message = servable_timeout_message( - model_name="my-model", - timeout=1.0, - db_sync_seconds=0.0, - last_result=unreachable, - ) - assert "connection refused" in message - - listed_without_model = _listing("other") - propagation_message = servable_timeout_message( - model_name="my-model", - timeout=1.0, - db_sync_seconds=30.0, - last_result=listed_without_model, - ) - assert "did not succeed" not in propagation_message From 38d03fd341bf5b7030b2ebbc45c2555a7f1ee1e6 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 28 Jul 2026 17:16:30 -0700 Subject: [PATCH 006/124] fix(e2e): never skip the final deadline-clamped model-servable poll When less than one full poll interval remained in the first-listing budget, the pre-sleep check returned NotServable without another /v1/models call. Sleep only min(interval, time left) so a model that becomes listable in the last seconds of the timeout still gets a clamped final poll (cherry picked from commit 8439195922c913d118cb146409c75cc081d23e6e) --- tests/e2e/proxy_client.py | 43 ++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index dfbb90ac08e..36dada5770a 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -124,24 +124,28 @@ def await_servable( First listing must happen within `timeout`. After that, the model must stay listed continuously for `db_sync_seconds` (any miss resets the continuous window). `db_sync_seconds=0` returns on the first listing. Each poll's request - timeout is clamped to the remaining budget. Clock and sleep are injected.""" + timeout is clamped to the remaining budget. Sleeps only min(interval, time left) + so a final deadline-clamped poll is never skipped just because a full interval + does not fit. Clock and sleep are injected.""" started = now() first_seen_at: float | None = None last_result: Result[ModelsListResponse] | None = None while True: t = now() - if first_seen_at is None: - deadline = started + timeout - else: - deadline = first_seen_at + db_sync_seconds - remaining = deadline - t - if remaining <= 0 and last_result is not None: - if first_seen_at is not None and db_sync_seconds <= 0: - return Servable() - if first_seen_at is not None and t - first_seen_at >= db_sync_seconds: + phase_deadline = ( + started + timeout if first_seen_at is None else first_seen_at + db_sync_seconds + ) + remaining = phase_deadline - t + if remaining <= 0: + if ( + last_result is not None + and first_seen_at is not None + and (db_sync_seconds <= 0 or t - first_seen_at >= db_sync_seconds) + ): return Servable() return NotServable(last_result=last_result) - poll_timeout = min(request_timeout, remaining) if remaining > 0 else request_timeout + + poll_timeout = min(request_timeout, remaining) last_result = list_models(poll_timeout) listed = isinstance(last_result, Success) and any( entry.id == model_name for entry in last_result.data.data @@ -155,16 +159,13 @@ def await_servable( return Servable() elif t - first_seen_at >= db_sync_seconds: return Servable() - if first_seen_at is None: - if now() + interval >= started + timeout: - return NotServable(last_result=last_result) - elif now() + interval >= first_seen_at + db_sync_seconds: - # Final stretch: sleep only the remainder of the continuous window. - remainder = first_seen_at + db_sync_seconds - now() - if remainder > 0: - sleep(remainder) - continue - sleep(interval) + + phase_deadline = ( + started + timeout if first_seen_at is None else first_seen_at + db_sync_seconds + ) + wait = min(interval, phase_deadline - now()) + if wait > 0: + sleep(wait) def servable_timeout_message( From 87be33f9354f399100c7ededdfba31b0e831d225 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 28 Jul 2026 17:35:13 -0700 Subject: [PATCH 007/124] fix(e2e): reject first listing that returns after the 40s deadline A poll may start with remaining budget and still return after started+timeout if the transport overruns its clamp. Recheck the first-listing deadline after the response so a late listing does not open the continuous DB-sync phase (cherry picked from commit 7ff2bcbf1498ee82f7dbe0b4330c1ab48927ed01) --- tests/e2e/proxy_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 36dada5770a..b3fc8538322 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -154,6 +154,8 @@ def await_servable( if not listed: first_seen_at = None elif first_seen_at is None: + if t > started + timeout: + return NotServable(last_result=last_result) first_seen_at = t if db_sync_seconds <= 0: return Servable() From 82fa66908bb72748efc574a8a1a8750155a85c14 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 28 Jul 2026 22:15:21 -0700 Subject: [PATCH 008/124] test(e2e): poll MCP tools across multi-worker lag (#35047) * fix(mcp): resolve call_tool by registry without requiring tool map Multi-worker reloads put MCP servers in the registry from the DB but do not re-run tools/list on every process. Gating call_tool on tool_name_to_mcp_server_name_mapping made cold workers 500 with Tool not found after another worker had already listed the tool. Treat a registry match on server id/name/alias as enough; upstream rejects unknown tools * test(e2e): poll MCP register, tools/list, and tools/call across multi-worker lag Stage multi-worker gateways only load MCP servers and tool maps on the process that handled the request. Poll until the server is listed, the tool appears on tools/list, and tools/call is not a cold-worker 500 so key-access and Datadog MCP e2e stop racing the LB * Revert "fix(mcp): resolve call_tool by registry without requiring tool map" This reverts commit 8b56e51e39b876d13d1112efa4130554ddf5f173. * test(e2e): tighten MCP multi-worker lag classifier Only retry tools/call on gateway shapes Tool not found and server_not_found, not any 500 that mentions tool/server not found, so upstream failures are not retried until the poll deadline * test(e2e): drop unit file for MCP lag classifier The live await_call_tool polls already cover multi-worker lag; a separate string-match unit module is not worth keeping (cherry picked from commit c274cf321c5c35c629220a89bb497d15b56f870f) --- tests/e2e/mcp/mcp_client.py | 91 +++++++++++++++++++++- tests/e2e/mcp/test_mcp_access_group_e2e.py | 1 + tests/e2e/mcp/test_mcp_datadog_e2e.py | 28 ++++--- tests/e2e/mcp/test_mcp_key_access_e2e.py | 15 ++-- 4 files changed, 111 insertions(+), 24 deletions(-) diff --git a/tests/e2e/mcp/mcp_client.py b/tests/e2e/mcp/mcp_client.py index 6d0f6ddc760..33ec557c339 100644 --- a/tests/e2e/mcp/mcp_client.py +++ b/tests/e2e/mcp/mcp_client.py @@ -11,13 +11,14 @@ request/response bodies are co-located here because only this suite speaks MCP. from __future__ import annotations +import re import time from collections.abc import Mapping from dataclasses import dataclass from pydantic import BaseModel, ConfigDict, Field, RootModel -from e2e_http import Headers, NoBody, Result, Success, unwrap +from e2e_http import Headers, NoBody, Result, Success, UnknownApiError, unwrap from models import KeyGenerateBody, ObjectPermission from proxy_client import ProxyClient @@ -270,6 +271,60 @@ class McpClient: ) time.sleep(self.proxy.poll_interval) + def await_call_tool( + self, + key: str, + *, + server_id: str, + name: str, + arguments: McpToolArguments, + ) -> McpCallToolResponse: + """Poll tools/call until the result is not a multi-worker registry miss. + + Retries only on the gateway's own cold-worker 500 shapes (Tool + not found / server_not_found). Upstream tool errors and other 500s fail + immediately so non-idempotent calls are not repeated. + """ + deadline = time.monotonic() + self.proxy.poll_timeout + last: Result[McpCallToolResponse] | None = None + while True: + last = self.call_tool(key, server_id=server_id, name=name, arguments=arguments) + if not _is_mcp_not_synced(last, tool_name=name): + return unwrap(last) + if time.monotonic() >= deadline: + raise AssertionError( + f"tools/call for {name!r} on server {server_id} still missing on the " + f"data plane after {self.proxy.poll_timeout}s (multi-worker registry lag); " + f"last result: {last}" + ) + time.sleep(self.proxy.poll_interval) + + def await_call_tool_denied( + self, + key: str, + *, + server_id: str, + name: str, + arguments: McpToolArguments, + ) -> UnknownApiError: + """Poll tools/call until a cold-worker miss clears and the call is 403 access_denied.""" + deadline = time.monotonic() + self.proxy.poll_timeout + last: Result[McpCallToolResponse] | None = None + while True: + last = self.call_tool(key, server_id=server_id, name=name, arguments=arguments) + if isinstance(last, UnknownApiError) and last.status_code == 403: + return last + if not _is_mcp_not_synced(last, tool_name=name): + raise AssertionError( + f"ungranted key's tools/call was not 403 access_denied: {last}" + ) + if time.monotonic() >= deadline: + raise AssertionError( + f"ungranted key never got 403 for {name!r} within {self.proxy.poll_timeout}s; " + f"last result: {last}" + ) + time.sleep(self.proxy.poll_interval) + def register_mcp_content_filter(self, *, name: str, blocked_keyword: str) -> str: """Register a default-on content-filter guardrail that runs on the MCP tool-call hook (pre_mcp_call) and blocks a single keyword. The keyword is @@ -317,5 +372,39 @@ class McpClient: ) +def _is_mcp_not_synced( + result: Result[McpCallToolResponse], + *, + tool_name: str | None = None, +) -> bool: + """True only for gateway multi-worker registry misses, not upstream errors. + + Matches the proxy's own shapes: + - ValueError ``Tool not found`` wrapped as HTTP 500 (cold tool map / + unresolved server on this process) + - REST ``server_not_found`` when this worker has not loaded the MCP server row + + Does not treat arbitrary 500 bodies that merely mention "tool" and "not found" + (e.g. upstream MCP payload text) as lag, so await_call_tool does not retry + real failures or non-idempotent calls. + """ + if not isinstance(result, UnknownApiError) or result.status_code != 500: + return False + body = result.body + body_l = body.lower() + + if "server_not_found" in body_l: + return True + if re.search(r"mcp server ['\"][^'\"]+['\"] was not found", body_l): + return True + + # Gateway: "Tool search_datadog_logs not found" (optionally inside a longer message) + if tool_name is not None: + return ( + re.search(rf"\btool\s+{re.escape(tool_name)}\s+not found\b", body_l) is not None + ) + return re.search(r"\btool\s+\S+\s+not found\b", body_l) is not None + + def build_client(proxy: ProxyClient) -> McpClient: return McpClient(proxy=proxy) diff --git a/tests/e2e/mcp/test_mcp_access_group_e2e.py b/tests/e2e/mcp/test_mcp_access_group_e2e.py index 1b53d1ca0b4..f72b75fd43d 100644 --- a/tests/e2e/mcp/test_mcp_access_group_e2e.py +++ b/tests/e2e/mcp/test_mcp_access_group_e2e.py @@ -29,6 +29,7 @@ class TestMcpAccessGroupToolSelection: ) -> None: group = f"e2e-mcp-grp-{unique_marker()}" server_id = register_datadog_mcp(client, resources, mcp_access_groups=[group]) + client.await_registered(server_id) granted = client.generate_key( user_id=f"e2e-mcp-ag-granted-{unique_marker()}", diff --git a/tests/e2e/mcp/test_mcp_datadog_e2e.py b/tests/e2e/mcp/test_mcp_datadog_e2e.py index 8a539b86bff..d093e307f99 100644 --- a/tests/e2e/mcp/test_mcp_datadog_e2e.py +++ b/tests/e2e/mcp/test_mcp_datadog_e2e.py @@ -60,6 +60,7 @@ class TestDatadogMcpRoundTrip: _assert_datadog_logger_active(client.proxy) server_id = register_datadog_mcp(client, resources) + client.await_registered(server_id) marker = f"{MARKER_PREFIX}{unique_marker()}" key = client.generate_key( @@ -78,22 +79,19 @@ class TestDatadogMcpRoundTrip: ) tool_name = client.await_tool(key, server_id, SEARCH_LOGS_TOOL) - - call = unwrap( - client.call_tool( - key, - server_id=server_id, - name=tool_name, - arguments={ - "query": marker, - "from": DD_SEARCH_FROM, - "to": "now", - "max_tokens": 5000, - "telemetry": { - "intent": "e2e assert seeded litellm completion log is searchable via MCP" - }, + call = client.await_call_tool( + key, + server_id=server_id, + name=tool_name, + arguments={ + "query": marker, + "from": DD_SEARCH_FROM, + "to": "now", + "max_tokens": 5000, + "telemetry": { + "intent": "e2e assert seeded litellm completion log is searchable via MCP" }, - ) + }, ) assert call.is_error is not True, f"search_datadog_logs errored: {call}" body = call.all_text diff --git a/tests/e2e/mcp/test_mcp_key_access_e2e.py b/tests/e2e/mcp/test_mcp_key_access_e2e.py index 35c864c07d8..678424e36d1 100644 --- a/tests/e2e/mcp/test_mcp_key_access_e2e.py +++ b/tests/e2e/mcp/test_mcp_key_access_e2e.py @@ -16,7 +16,7 @@ import pytest from datadog_mcp import SEARCH_LOGS_TOOL, register_datadog_mcp from e2e_config import DD_SEARCH_FROM, unique_marker -from e2e_http import UnknownApiError, unwrap +from e2e_http import unwrap from lifecycle import ResourceManager from mcp_client import McpClient @@ -72,13 +72,12 @@ class TestMcpKeyWithoutAccessIsDenied: "max_tokens": 1000, "telemetry": {"intent": "e2e control call proving granted key can invoke Datadog MCP"}, } - permitted_call = unwrap( - client.call_tool(permitted_key, server_id=server_id, name=tool_name, arguments=search_args) + permitted_call = client.await_call_tool( + permitted_key, server_id=server_id, name=tool_name, arguments=search_args ) assert permitted_call.is_error is not True, f"granted key's tool call errored: {permitted_call}" - match client.call_tool(denied_key, server_id=server_id, name=tool_name, arguments=search_args): - case UnknownApiError(status_code=403, body=body): - assert "access_denied" in body, f"403 was not an MCP access denial: {body}" - case other: - pytest.fail(f"ungranted key's tool call was not refused with 403 access_denied: {other}") + denied = client.await_call_tool_denied( + denied_key, server_id=server_id, name=tool_name, arguments=search_args + ) + assert "access_denied" in denied.body, f"403 was not an MCP access denial: {denied.body}" From ea472267881743014e42634ff85fb595a2d3fc3c Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 30 Jul 2026 16:06:33 -0700 Subject: [PATCH 009/124] ci: add fork GHCR publish workflow for Concourse releases --- .github/workflows/publish-ghcr.yml | 129 +++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 .github/workflows/publish-ghcr.yml diff --git a/.github/workflows/publish-ghcr.yml b/.github/workflows/publish-ghcr.yml new file mode 100644 index 00000000000..7530e85116a --- /dev/null +++ b/.github/workflows/publish-ghcr.yml @@ -0,0 +1,129 @@ +# Build and push LiteLLM images to THIS fork's GHCR. +name: Publish GHCR (fork) + +on: + workflow_dispatch: + inputs: + image_tag: + description: Primary image tag (e.g. dev, rc, short sha) + required: true + type: string + default: dev + git_ref: + description: Git ref to build. Empty uses the branch the workflow runs on. + required: false + type: string + default: "" + variants: + description: "Comma-separated: litellm,database,non_root" + required: false + type: string + default: litellm + dry_run: + description: Build only; skip push + required: false + type: boolean + default: false + +permissions: + contents: read + packages: write + +concurrency: + group: publish-ghcr-${{ github.event.inputs.image_tag }} + cancel-in-progress: false + +jobs: + publish: + name: Build and push ${{ matrix.name }} + runs-on: ubuntu-latest + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + include: + - name: litellm + dockerfile: Dockerfile + image_suffix: litellm + - name: database + dockerfile: docker/Dockerfile.database + image_suffix: litellm-database + - name: non_root + dockerfile: docker/Dockerfile.non_root + image_suffix: litellm-non_root + steps: + - name: Select variant + id: pick + shell: bash + run: | + set -euo pipefail + wanted="${{ github.event.inputs.variants }}" + name="${{ matrix.name }}" + if [[ ",${wanted}," == *",${name},"* ]] || [[ "${wanted}" == "${name}" ]]; then + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "run=false" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout + if: steps.pick.outputs.run == 'true' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.git_ref != '' && github.event.inputs.git_ref || github.ref }} + fetch-depth: 1 + + - name: Set up Docker Buildx + if: steps.pick.outputs.run == 'true' + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + if: steps.pick.outputs.run == 'true' && github.event.inputs.dry_run != 'true' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Image metadata + if: steps.pick.outputs.run == 'true' + id: meta + shell: bash + run: | + set -euo pipefail + owner="${GITHUB_REPOSITORY_OWNER,,}" + tag="${{ github.event.inputs.image_tag }}" + sha="$(git rev-parse --short HEAD)" + image="ghcr.io/${owner}/${{ matrix.image_suffix }}" + { + echo "image=${image}" + echo "tags=${image}:${tag},${image}:${sha}" + echo "sha=${sha}" + } >> "$GITHUB_OUTPUT" + echo "Will publish: ${image}:${tag} and ${image}:${sha}" + + - name: Build and push + if: steps.pick.outputs.run == 'true' + uses: docker/build-push-action@v6 + with: + context: . + file: ${{ matrix.dockerfile }} + push: ${{ github.event.inputs.dry_run != 'true' }} + tags: ${{ steps.meta.outputs.tags }} + platforms: linux/amd64 + provenance: false + sbom: false + cache-from: type=gha,scope=${{ matrix.name }} + cache-to: type=gha,mode=max,scope=${{ matrix.name }} + + - name: Summary + if: steps.pick.outputs.run == 'true' + shell: bash + run: | + { + echo "### ${{ matrix.name }}" + echo "" + echo "- image: \`${{ steps.meta.outputs.image }}\`" + echo "- tags: \`${{ steps.meta.outputs.tags }}\`" + echo "- dry_run: \`${{ github.event.inputs.dry_run }}\`" + echo "- sha: \`${{ steps.meta.outputs.sha }}\`" + } >> "$GITHUB_STEP_SUMMARY" From 14c97ba8db1cb4172e3276ba191c962be8a1cc11 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Mon, 20 Jul 2026 13:48:24 -0700 Subject: [PATCH 010/124] fix(proxy): make /cursor/chat/completions work with Cursor agent mode - delegate messages-shaped bodies to the standard chat completions handler - strip chat-only stream_options before the Responses pipeline - fix cursor_data_generator signature (request kwarg) and duck-type the stream gate so router-wrapped streams convert instead of leaking raw Responses events - convert custom_tool_call items and events to chat tool_calls in the streaming and non-streaming paths; remap streamed tool_call indices to 0-based sequential; accumulate raw and pydantic tool calls into one choice - normalize generic pydantic output items through the raw-dict handler --- .../transformation.py | 181 ++++++++----- .../proxy/response_api_endpoints/endpoints.py | 49 ++-- litellm/types/llms/openai.py | 4 + ...responses_transformation_transformation.py | 253 ++++++++++++++++++ .../response_api_endpoints/test_endpoints.py | 147 ++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 10 +- 6 files changed, 555 insertions(+), 89 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 89a44fcdeef..d1df75bde36 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -100,6 +100,32 @@ def _build_reasoning_item( } +def _tool_call_dict_from_output_item(item: dict[str, Any]) -> dict[str, Any]: + """Convert a ``function_call`` or ``custom_tool_call`` output item dict to a chat + completions tool_call dict. Custom (grammar/freeform) tool calls carry their raw + string payload in ``input`` rather than ``arguments``; both map to + ``function.arguments`` so chat clients (e.g. Cursor agent mode) receive them like + any other tool call. The single conversion rule shared by the non-streaming + accumulator and the streaming ``output_item.added`` branch.""" + from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, + ) + + is_custom = item.get("type") == "custom_tool_call" + arguments = (item.get("input") if is_custom else item.get("arguments")) or "" + name = item.get("name") or ("custom_tool" if is_custom else "") + tool_call_dict: dict[str, Any] = { + "id": LiteLLMCompletionResponsesConfig._tool_call_id_from_responses_item(item.get("id"), item.get("call_id")), + "function": {"name": name, "arguments": arguments}, + "type": "function", + } + provider_specific_fields = item.get("provider_specific_fields") + if isinstance(provider_specific_fields, dict) and provider_specific_fields: + tool_call_dict["provider_specific_fields"] = provider_specific_fields + tool_call_dict["function"]["provider_specific_fields"] = provider_specific_fields + return tool_call_dict + + def _reasoning_item_to_response_input( r_item: Union[ChatCompletionReasoningItem, Dict[str, Any]], ) -> Dict[str, Any]: @@ -176,36 +202,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): choice = Choices(message=msg, finish_reason="stop", index=index) return choice, index + 1 - # Handle function_call items (e.g., from GPT-5 Codex format) - if item_type == "function_call": - # Extract provider_specific_fields if present and pass through as-is - provider_specific_fields = item.get("provider_specific_fields") - if provider_specific_fields and not isinstance(provider_specific_fields, dict): - provider_specific_fields = ( - dict(provider_specific_fields) if hasattr(provider_specific_fields, "__dict__") else {} - ) - - tool_call_dict = { - "id": item.get("call_id") or item.get("id", ""), - "function": { - "name": item.get("name", ""), - "arguments": item.get("arguments", ""), - }, - "type": "function", - } - - # Pass through provider_specific_fields as-is if present - if provider_specific_fields: - tool_call_dict["provider_specific_fields"] = provider_specific_fields - # Also add to function's provider_specific_fields for consistency - tool_call_dict["function"]["provider_specific_fields"] = provider_specific_fields - - msg = Message( - content=None, - tool_calls=[tool_call_dict], - ) - choice = Choices(message=msg, finish_reason="tool_calls", index=index) - return choice, index + 1 + # function_call / custom_tool_call dicts are intercepted and accumulated by + # _convert_response_output_to_choices before this callback is reached # Unknown or unsupported type return None, index @@ -562,11 +560,21 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): accumulated_tool_calls.append(tool_call_dict) tool_call_index += 1 - elif isinstance(item, dict) and handle_raw_dict_callback is not None: - # Handle raw dict responses (e.g., from GPT-5 Codex) - choice, index = handle_raw_dict_callback(item=item, index=index) - if choice is not None: - choices.append(choice) + elif isinstance(item, (dict, BaseModel)): + # Raw dict items (e.g., from GPT-5 Codex) and pydantic items matching no + # openai SDK class above: typed ResponseCustomToolCall and litellm's own + # GenericResponseOutputItem from the completion bridge both land here + raw_item = item if isinstance(item, dict) else item.model_dump() + if raw_item.get("type") in ("function_call", "custom_tool_call"): + # Tool calls accumulate into the single trailing tool_calls choice + # like the typed branches above; a choice per call would hide every + # call after choices[0] from chat clients + accumulated_tool_calls.append(_tool_call_dict_from_output_item(raw_item)) + tool_call_index += 1 + elif handle_raw_dict_callback is not None: + choice, index = handle_raw_dict_callback(item=raw_item, index=index) + if choice is not None: + choices.append(choice) else: pass # don't fail request if item in list is not supported @@ -1078,6 +1086,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): def __init__(self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False): super().__init__(streaming_response, sync_stream, json_mode) self._chat_completion_id: str | None = None + self._tool_call_index_map: dict[int, int] = {} def _handle_string_chunk( self, str_line: Union[str, "BaseModel"] @@ -1096,15 +1105,35 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): return self.chunk_parser(json.loads(str_line)) + @staticmethod + def _sequential_tool_call_index( + tool_call_index_map: dict[int, int] | None, + output_index: int, + ) -> int: + """Chat-completions tool_call indices must be 0-based and sequential, but + Responses API ``output_index`` counts every output item (reasoning, + message, ...), so the first tool call of a reasoning model arrives at + output_index >= 1 and strict SSE accumulators (e.g. Cursor agent mode) + misplace it. When a per-stream map is provided, remap each distinct + output_index to the next sequential slot; without a map (stateless + callers), fall back to the raw output_index.""" + if tool_call_index_map is None: + return output_index + if output_index not in tool_call_index_map: + tool_call_index_map[output_index] = len(tool_call_index_map) # mutable-ok: per-stream accumulator state + return tool_call_index_map[output_index] + @staticmethod def translate_responses_chunk_to_openai_stream( parsed_chunk: Union[dict, BaseModel], + tool_call_index_map: dict[int, int] | None = None, ) -> "ModelResponseStream": """ Translate a Responses API streaming chunk to OpenAI chat completion streaming format. Args: parsed_chunk: Dict containing the Responses API event chunk + tool_call_index_map: Per-stream output_index -> sequential tool_call index map Returns: ModelResponseStream: OpenAI-formatted streaming chunk @@ -1165,7 +1194,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): function_chunk = ChatCompletionToolCallFunctionChunk( name=output_item.get("name", None), - arguments=parsed_chunk.get("arguments", ""), + arguments=output_item.get("arguments") or parsed_chunk.get("arguments") or "", ) if provider_specific_fields: @@ -1175,7 +1204,9 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): LiteLLMCompletionResponsesConfig, ) - tool_call_index = parsed_chunk.get("output_index", 0) + tool_call_index = OpenAiResponsesToChatCompletionStreamIterator._sequential_tool_call_index( + tool_call_index_map, parsed_chunk.get("output_index", 0) + ) tool_call_chunk = ChatCompletionToolCallChunk( id=LiteLLMCompletionResponsesConfig._tool_call_id_from_responses_item( output_item.get("id"), output_item.get("call_id") @@ -1198,10 +1229,41 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): ) ] ) - elif event_type == "response.function_call_arguments.delta": + if output_item.get("type") == "custom_tool_call": + tool_call_index = OpenAiResponsesToChatCompletionStreamIterator._sequential_tool_call_index( + tool_call_index_map, parsed_chunk.get("output_index", 0) + ) + converted = _tool_call_dict_from_output_item(output_item) + return ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta( + tool_calls=[ + ChatCompletionToolCallChunk( + id=converted["id"], + index=tool_call_index, + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=converted["function"]["name"], + arguments=converted["function"]["arguments"], + ), + ) + ] + ), + finish_reason=None, + ) + ] + ) + elif event_type in ( + ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA, + ResponsesAPIStreamEvents.CUSTOM_TOOL_CALL_INPUT_DELTA, + ): content_part: Optional[str] = parsed_chunk.get("delta", None) if content_part: - tool_call_index = parsed_chunk.get("output_index", 0) + tool_call_index = OpenAiResponsesToChatCompletionStreamIterator._sequential_tool_call_index( + tool_call_index_map, parsed_chunk.get("output_index", 0) + ) return ModelResponseStream( choices=[ StreamingChoices( @@ -1225,39 +1287,12 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): elif event_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE: # New output item added output_item = parsed_chunk.get("item", {}) - if output_item.get("type") == "function_call": - # Extract provider_specific_fields if present - provider_specific_fields = output_item.get("provider_specific_fields") - if provider_specific_fields and not isinstance(provider_specific_fields, dict): - provider_specific_fields = ( - dict(provider_specific_fields) if hasattr(provider_specific_fields, "__dict__") else {} - ) - - function_chunk = ChatCompletionToolCallFunctionChunk( - name=output_item.get("name", None), - arguments="", # responses API sends everything again, we don't - ) - - # Add provider_specific_fields to function if present - if provider_specific_fields: - function_chunk["provider_specific_fields"] = provider_specific_fields - - tool_call_index = parsed_chunk.get("output_index", 0) - tool_call_chunk = ChatCompletionToolCallChunk( - id=output_item.get("call_id"), - index=tool_call_index, - type="function", - function=function_chunk, - ) - - # Add provider_specific_fields if present - if provider_specific_fields: - tool_call_chunk.provider_specific_fields = provider_specific_fields # type: ignore - + if output_item.get("type") in ("function_call", "custom_tool_call"): # Do NOT emit finish_reason here — response.completed handles the terminal # finish_reason. Emitting "tool_calls" here would prematurely terminate # the stream before subsequent tool calls arrive (same fix as #17246 for - # the message-type branch). + # the message-type branch). The item's fields were already streamed via + # output_item.added and the argument delta events. return ModelResponseStream( choices=[ StreamingChoices( @@ -1316,7 +1351,9 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): output_items = response_data.get("output", []) if response_data else [] has_function_calls = any( - item.get("type") == "function_call" for item in output_items if isinstance(item, dict) + item.get("type") in ("function_call", "custom_tool_call") + for item in output_items + if isinstance(item, dict) ) finish_reason = "tool_calls" if has_function_calls else "stop" @@ -1386,7 +1423,9 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): """ verbose_logger.debug(f"Chat provider: transform_streaming_response called with chunk: {chunk}") return self._with_stream_scoped_id( - OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(chunk) + OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream( + chunk, tool_call_index_map=self._tool_call_index_map + ) ) def _with_stream_scoped_id(self, chunk: "ModelResponseStream") -> "ModelResponseStream": diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 05c36406f36..dcd7ba18e7f 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -294,11 +294,15 @@ async def cursor_chat_completions( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ - Cursor-specific endpoint that accepts Responses API input format but returns chat completions format. - - This endpoint handles requests from Cursor IDE which sends Responses API format (`input` field) - but expects chat completions format response (`choices`, `messages`, etc.). - + Cursor BYOK endpoint. Accepts both request shapes Cursor sends to its OpenAI-compatible + base URL and always answers in chat completions format. + + Cursor agent mode sends Responses API format bodies (`input`, flat tool defs, `reasoning`, + custom tools) to the chat/completions path while expecting chat completions responses; + those are routed through the Responses API pipeline and converted back. Genuine chat + completions bodies (`messages` present) are routed through the standard chat completions + pipeline untouched. + ```bash curl -X POST http://localhost:4000/cursor/chat/completions \ -H "Content-Type: application/json" \ @@ -317,6 +321,7 @@ async def cursor_chat_completions( from litellm.proxy.proxy_server import ( _read_request_body, async_data_generator, + chat_completion, general_settings, llm_router, proxy_config, @@ -328,20 +333,28 @@ async def cursor_chat_completions( user_temperature, version, ) - from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ModelResponse data = await _read_request_body(request=request) - # Convert 'messages' to 'input' for Responses API compatibility - # Cursor sends 'messages' but Responses API expects 'input' - if "messages" in data and "input" not in data: - data["input"] = data.pop("messages") + if "messages" in data: + # Genuine chat completions body (Cursor sends these for models whose BYOK it + # already fixed); delegate so behavior matches /chat/completions exactly + return await chat_completion( + request=request, + fastapi_response=fastapi_response, + model=None, + user_api_key_dict=user_api_key_dict, + ) + + # OpenAI's Responses API rejects chat-completions-only stream_options + # (Cursor sends include_usage); usage arrives via response.completed anyway + data.pop("stream_options", None) processor = ProxyBaseLLMRequestProcessing(data=data) - def cursor_data_generator(response, user_api_key_dict, request_data): + def cursor_data_generator(response, user_api_key_dict, request_data, request=None): """ Custom generator that transforms Responses API streaming chunks to chat completion chunks. @@ -349,17 +362,21 @@ async def cursor_chat_completions( to chat completion format that Cursor IDE expects. Args: - response: The streaming response (BaseResponsesAPIStreamingIterator or other) + response: The streaming Responses API event iterator (router-wrapped or not) user_api_key_dict: User API key authentication dict request_data: Request data containing model, logging_obj, etc. + request: The originating FastAPI request, forwarded for disconnect handling Returns: Async generator that yields SSE-formatted chat completion chunks """ - # If response is a BaseResponsesAPIStreamingIterator, transform it first - if isinstance(response, BaseResponsesAPIStreamingIterator): + # Any async-iterable here is a Responses API event stream needing conversion. + # Class-identity checks miss router-wrapped streams (e.g. + # HiddenParamsAsyncIteratorWrapper around LiteLLMCompletionStreamingIterator), + # which previously leaked raw Responses events to the client. + if hasattr(response, "__anext__"): # Transform Responses API iterator to chat completion iterator - # Cast to AsyncIterator[str] since BaseResponsesAPIStreamingIterator implements __aiter__/__anext__ + # Cast to AsyncIterator[str] since the stream implements __aiter__/__anext__ completion_stream = responses_api_bridge.transformation_handler.get_model_response_iterator( streaming_response=cast(AsyncIterator[str], response), sync_stream=False, @@ -378,12 +395,14 @@ async def cursor_chat_completions( response=streamwrapper, user_api_key_dict=user_api_key_dict, request_data=request_data, + request=request, ) # Otherwise, use the default generator return async_data_generator( response=response, user_api_key_dict=user_api_key_dict, request_data=request_data, + request=request, ) try: diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 314bb653196..0d064006412 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1405,6 +1405,10 @@ class ResponsesAPIStreamEvents(str, Enum): FUNCTION_CALL_ARGUMENTS_DELTA = "response.function_call_arguments.delta" FUNCTION_CALL_ARGUMENTS_DONE = "response.function_call_arguments.done" + # Custom tool call events (grammar/freeform tools, e.g. Cursor agent tools) + CUSTOM_TOOL_CALL_INPUT_DELTA = "response.custom_tool_call_input.delta" + CUSTOM_TOOL_CALL_INPUT_DONE = "response.custom_tool_call_input.done" + # File search events FILE_SEARCH_CALL_IN_PROGRESS = "response.file_search_call.in_progress" FILE_SEARCH_CALL_SEARCHING = "response.file_search_call.searching" diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index a111b932f2c..36c32d4b3a0 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -2962,3 +2962,256 @@ async def test_acompletion_bridge_normalizes_stream_options_on_the_wire( assert "stream_options" not in request_body else: assert request_body["stream_options"] == expected_wire_stream_options + + +def test_chunk_parser_custom_tool_call_stream_sequence(): + """Cursor agent mode drives grammar/freeform ``custom_tool_call`` items (e.g. its + ApplyPatch tool). The stream converter must surface them as chat-completions + tool_call deltas: the added event opens the call (id from ``call_id``, name, empty + arguments), each ``custom_tool_call_input.delta`` streams arguments, the done event + must NOT finish the stream, and ``response.completed`` must report + finish_reason="tool_calls". Before the fix every one of these events fell through + to an empty-content chunk and the completed event said "stop", so Cursor never saw + the tool call and agent mode stalled.""" + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + + iterator = OpenAiResponsesToChatCompletionStreamIterator( + streaming_response=None, sync_stream=True + ) + + added = iterator.chunk_parser( + { + "type": "response.output_item.added", + "output_index": 1, + "item": { + "type": "custom_tool_call", + "id": "ctc_1", + "call_id": "call_patch1", + "name": "ApplyPatch", + "input": "", + }, + } + ) + tool_call = added.choices[0].delta.tool_calls[0] + assert tool_call.id == "call_patch1" + assert tool_call.type == "function" + assert tool_call.function.name == "ApplyPatch" + assert tool_call.function.arguments == "" + assert tool_call.index == 0 + assert added.choices[0].finish_reason is None + + delta = iterator.chunk_parser( + { + "type": "response.custom_tool_call_input.delta", + "output_index": 1, + "delta": "*** Begin Patch", + } + ) + delta_tool_call = delta.choices[0].delta.tool_calls[0] + assert delta_tool_call.function.arguments == "*** Begin Patch" + assert delta_tool_call.index == 0 + assert delta.choices[0].finish_reason is None + + done = iterator.chunk_parser( + { + "type": "response.output_item.done", + "output_index": 1, + "item": { + "type": "custom_tool_call", + "call_id": "call_patch1", + "name": "ApplyPatch", + "input": "*** Begin Patch", + }, + } + ) + assert done.choices[0].finish_reason is None + + completed = iterator.chunk_parser( + { + "type": "response.completed", + "response": { + "output": [ + {"type": "reasoning", "id": "rs_1"}, + {"type": "custom_tool_call", "call_id": "call_patch1"}, + ], + "usage": {"input_tokens": 7, "output_tokens": 3, "total_tokens": 10}, + }, + } + ) + assert completed.choices[0].finish_reason == "tool_calls" + assert completed.usage is not None + assert completed.usage.total_tokens == 10 + + +def test_chunk_parser_remaps_tool_call_indices_sequentially(): + """Responses API output_index counts every output item, so a reasoning model's + first tool call arrives at output_index >= 1. Chat-completions clients accumulate + streamed tool_calls by index and expect the first call at 0; Cursor agent mode + misplaces calls when indices start above 0 (the community BYOK bridge assigns its + own sequential indices for the same reason). The iterator must remap each distinct + output_index to the next sequential slot and route argument deltas to the mapped + slot.""" + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + + iterator = OpenAiResponsesToChatCompletionStreamIterator( + streaming_response=None, sync_stream=True + ) + + first = iterator.chunk_parser( + { + "type": "response.output_item.added", + "output_index": 2, + "item": { + "type": "function_call", + "id": "fc_1", + "call_id": "call_read1", + "name": "read_file", + "arguments": "", + }, + } + ) + assert first.choices[0].delta.tool_calls[0].index == 0 + + first_args = iterator.chunk_parser( + { + "type": "response.function_call_arguments.delta", + "output_index": 2, + "delta": '{"path":', + } + ) + assert first_args.choices[0].delta.tool_calls[0].index == 0 + + second = iterator.chunk_parser( + { + "type": "response.output_item.added", + "output_index": 4, + "item": { + "type": "function_call", + "id": "fc_2", + "call_id": "call_grep1", + "name": "grep", + "arguments": "", + }, + } + ) + assert second.choices[0].delta.tool_calls[0].index == 1 + + second_args = iterator.chunk_parser( + { + "type": "response.function_call_arguments.delta", + "output_index": 4, + "delta": '{"pattern":', + } + ) + assert second_args.choices[0].delta.tool_calls[0].index == 1 + + +def test_convert_response_output_custom_tool_call_to_tool_calls_choice(): + """Non-streaming twin of the custom_tool_call fix: a typed ResponseCustomToolCall + output item must become a chat tool_call (arguments = the raw custom input string, + id = call_id) in a finish_reason="tool_calls" choice instead of being silently + dropped, which left Cursor agent mode with an empty assistant message.""" + from openai.types.responses import ResponseCustomToolCall + + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + item = ResponseCustomToolCall( + type="custom_tool_call", + id="ctc_9", + call_id="call_custom9", + name="ApplyPatch", + input="*** Begin Patch\n*** End Patch", + ) + + choices = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices([item]) + + assert len(choices) == 1 + choice = choices[0] + assert choice.finish_reason == "tool_calls" + tool_call = choice.message.tool_calls[0] + assert tool_call.id == "call_custom9" + assert tool_call.function.name == "ApplyPatch" + assert tool_call.function.arguments == "*** Begin Patch\n*** End Patch" + + +def test_convert_response_output_accumulates_raw_tool_calls_into_one_choice(): + """Raw dict and generic-pydantic tool-call items must accumulate into the single + trailing tool_calls choice exactly like typed items. Emitting one choice per tool + call (the old raw-dict behavior) hid every call after choices[0] from chat + clients, which read only the first choice; a multi-tool agent turn through the + completion bridge lost all but one call.""" + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + items = [ + { + "type": "function_call", + "id": "fc_1", + "call_id": "call_read42", + "name": "read_file", + "arguments": '{"path": "a.py"}', + }, + { + "type": "custom_tool_call", + "id": "ctc_1", + "call_id": "call_patch42", + "name": "ApplyPatch", + "input": "*** Begin Patch", + }, + ] + + choices = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices( + items, + handle_raw_dict_callback=handler._handle_raw_dict_response_item, + ) + + assert len(choices) == 1 + choice = choices[0] + assert choice.finish_reason == "tool_calls" + tool_calls = choice.message.tool_calls + assert len(tool_calls) == 2 + assert tool_calls[0].id == "call_read42" + assert tool_calls[0].function.name == "read_file" + assert tool_calls[0].function.arguments == '{"path": "a.py"}' + assert tool_calls[1].id == "call_patch42" + assert tool_calls[1].function.name == "ApplyPatch" + assert tool_calls[1].function.arguments == "*** Begin Patch" + + +def test_convert_response_output_generic_pydantic_message_item(): + """litellm's completion bridge (used for non-Responses-native providers behind the + router) emits GenericResponseOutputItem pydantic models rather than openai SDK + classes. The converter must normalize unrecognized pydantic items through the + raw-dict handler instead of dropping them; dropping them made transform_response + raise 'Unknown items in responses API response' on an otherwise-successful + completion (hit live via /cursor/chat/completions multi-turn tool round trips).""" + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + from litellm.types.responses.main import GenericResponseOutputItem, OutputText + + handler = LiteLLMResponsesTransformationHandler() + item = GenericResponseOutputItem( + type="message", + id="msg_generic1", + status="completed", + role="assistant", + content=[OutputText(type="output_text", text="42", annotations=[])], + ) + + choices = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices( + [item], + handle_raw_dict_callback=handler._handle_raw_dict_response_item, + ) + + assert len(choices) == 1 + assert choices[0].message.content == "42" + assert choices[0].finish_reason == "stop" diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index 07d1a9d14f9..9af1ef4766d 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -8,6 +8,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient +import litellm from litellm.proxy.proxy_server import app @@ -711,3 +712,149 @@ class TestManagedResponsesSameProvider: call_kwargs: dict = {} handler._inject_credentials(call_kwargs, model="vertex_ai/gemini-2.0-flash") assert "custom_llm_provider" not in call_kwargs + + +def _auth_override(): + from litellm.proxy._types import UserAPIKeyAuth + + return UserAPIKeyAuth(api_key="sk-test-cursor", user_id="cursor-user") + + +def test_cursor_chat_completions_messages_body_uses_chat_pipeline(): + """A genuine chat-completions body (``messages`` present; what Cursor sends for + models whose BYOK it already fixed) must run through the standard chat pipeline + untouched: multi-turn tool history (assistant tool_calls + role="tool" results) + and nested chat-format tool defs are valid there, while blindly renaming + ``messages`` to ``input`` (the pre-fix behavior) produced items the Responses API + rejects. Asserts acompletion is called with the exact messages and aresponses is + never touched.""" + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + import litellm.proxy.proxy_server as ps + + messages = [ + {"role": "user", "content": "read a file"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_hist1", + "type": "function", + "function": {"name": "read_file", "arguments": '{"path": "a.py"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_hist1", "content": "file contents"}, + {"role": "user", "content": "now summarize"}, + ] + + mock_router = MagicMock() + mock_router.acompletion = AsyncMock( + return_value=litellm.ModelResponse( + id="chatcmpl-cursor-1", + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "summary"}, + "finish_reason": "stop", + } + ], + model="gpt-4o", + ) + ) + mock_router.aresponses = AsyncMock() + mock_router.get_available_deployment = MagicMock(return_value=None) + + app.dependency_overrides[user_api_key_auth] = _auth_override + try: + with patch.object(ps, "llm_router", mock_router): + client = TestClient(app) + response = client.post( + "/cursor/chat/completions", + json={ + "model": "gpt-4o", + "messages": messages, + "tools": [ + { + "type": "function", + "function": {"name": "read_file", "parameters": {"type": "object"}}, + } + ], + }, + headers={"Authorization": "Bearer sk-test-cursor"}, + ) + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 200, response.text + body = response.json() + assert body["choices"][0]["message"]["content"] == "summary" + assert "output" not in body + + mock_router.acompletion.assert_called_once() + called_kwargs = mock_router.acompletion.call_args.kwargs + assert called_kwargs["messages"] == messages + assert "input" not in called_kwargs + mock_router.aresponses.assert_not_called() + + +def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_stream_options(): + """A Responses-shaped body (``input``, no ``messages``; what Cursor agent mode + sends) must run through the Responses pipeline with chat-completions output, and + ``stream_options`` (chat-completions-only; Cursor sends include_usage) must be + stripped before the Responses call since OpenAI's Responses API rejects it.""" + from openai.types.responses import ResponseOutputMessage, ResponseOutputText + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.types.llms.openai import ResponsesAPIResponse + + import litellm.proxy.proxy_server as ps + + mock_router = MagicMock() + mock_router.aresponses = AsyncMock( + return_value=ResponsesAPIResponse( + id="resp_cursor_agent1", + created_at=1234567890, + model="gpt-4o", + object="response", + output=[ + ResponseOutputMessage( + id="msg_agent1", + type="message", + role="assistant", + status="completed", + content=[ + ResponseOutputText(type="output_text", text="agent reply", annotations=[]) + ], + ) + ], + ) + ) + mock_router.acompletion = AsyncMock() + + app.dependency_overrides[user_api_key_auth] = _auth_override + try: + with patch.object(ps, "llm_router", mock_router): + client = TestClient(app) + response = client.post( + "/cursor/chat/completions", + json={ + "model": "gpt-4o", + "input": [{"role": "user", "content": "hello"}], + "stream_options": {"include_usage": True}, + }, + headers={"Authorization": "Bearer sk-test-cursor"}, + ) + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 200, response.text + body = response.json() + assert body["choices"][0]["message"]["content"] == "agent reply" + assert "output" not in body + + mock_router.aresponses.assert_called_once() + called_kwargs = mock_router.aresponses.call_args.kwargs + assert "stream_options" not in called_kwargs + mock_router.acompletion.assert_not_called() diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 109d638fb9c..4e0b9b88995 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -2621,10 +2621,14 @@ export interface paths { put?: never; /** * Cursor Chat Completions - * @description Cursor-specific endpoint that accepts Responses API input format but returns chat completions format. + * @description Cursor BYOK endpoint. Accepts both request shapes Cursor sends to its OpenAI-compatible + * base URL and always answers in chat completions format. * - * This endpoint handles requests from Cursor IDE which sends Responses API format (`input` field) - * but expects chat completions format response (`choices`, `messages`, etc.). + * Cursor agent mode sends Responses API format bodies (`input`, flat tool defs, `reasoning`, + * custom tools) to the chat/completions path while expecting chat completions responses; + * those are routed through the Responses API pipeline and converted back. Genuine chat + * completions bodies (`messages` present) are routed through the standard chat completions + * pipeline untouched. * * ```bash * curl -X POST http://localhost:4000/cursor/chat/completions -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{ From b080454d1f58efa233c22e3b363ee8b72205aabe Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Mon, 20 Jul 2026 14:00:08 -0700 Subject: [PATCH 011/124] refactor(responses): clarify output_item.added tool branches as if/elif chain --- .../litellm_responses_transformation/transformation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index d1df75bde36..eb4acb86674 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -1229,7 +1229,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): ) ] ) - if output_item.get("type") == "custom_tool_call": + elif output_item.get("type") == "custom_tool_call": tool_call_index = OpenAiResponsesToChatCompletionStreamIterator._sequential_tool_call_index( tool_call_index_map, parsed_chunk.get("output_index", 0) ) From af1b7f1347e56c2f00c2a16829d75eb7c52327ea Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Mon, 20 Jul 2026 14:18:09 -0700 Subject: [PATCH 012/124] fix(proxy): strip stream_options without mutating the cached request body --- .../proxy/response_api_endpoints/endpoints.py | 7 +++-- .../response_api_endpoints/test_endpoints.py | 27 +++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index dcd7ba18e7f..9601e2d4fde 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -349,8 +349,11 @@ async def cursor_chat_completions( ) # OpenAI's Responses API rejects chat-completions-only stream_options - # (Cursor sends include_usage); usage arrives via response.completed anyway - data.pop("stream_options", None) + # (Cursor sends include_usage); usage arrives via response.completed anyway. + # Rebuild rather than pop: _read_request_body can return the request-scope + # cached parsed-body dict itself, and removing keys from it corrupts the + # cache's key snapshot so later readers get an empty body + data = {key: value for key, value in data.items() if key != "stream_options"} processor = ProxyBaseLLMRequestProcessing(data=data) diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index 9af1ef4766d..0cc79658b6a 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -803,14 +803,30 @@ def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_s """A Responses-shaped body (``input``, no ``messages``; what Cursor agent mode sends) must run through the Responses pipeline with chat-completions output, and ``stream_options`` (chat-completions-only; Cursor sends include_usage) must be - stripped before the Responses call since OpenAI's Responses API rejects it.""" + stripped before the Responses call since OpenAI's Responses API rejects it. + Stripping must not mutate the dict _read_request_body returned: that can be the + request-scope cached parsed body itself, and removing a key from it corrupts the + cache's key snapshot so any later _read_request_body caller (spend tracking, + logging hooks) silently gets an empty body; a follow-up read must still see the + full original body.""" + import asyncio + from openai.types.responses import ResponseOutputMessage, ResponseOutputText from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.common_utils.http_parsing_utils import ( + _read_request_body as real_read_request_body, + ) from litellm.types.llms.openai import ResponsesAPIResponse import litellm.proxy.proxy_server as ps + captured_requests = [] + + async def capturing_read_request_body(request): + captured_requests.append(request) + return await real_read_request_body(request=request) + mock_router = MagicMock() mock_router.aresponses = AsyncMock( return_value=ResponsesAPIResponse( @@ -835,7 +851,9 @@ def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_s app.dependency_overrides[user_api_key_auth] = _auth_override try: - with patch.object(ps, "llm_router", mock_router): + with patch.object(ps, "llm_router", mock_router), patch.object( + ps, "_read_request_body", side_effect=capturing_read_request_body + ): client = TestClient(app) response = client.post( "/cursor/chat/completions", @@ -858,3 +876,8 @@ def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_s called_kwargs = mock_router.aresponses.call_args.kwargs assert "stream_options" not in called_kwargs mock_router.acompletion.assert_not_called() + + assert captured_requests + followup_body = asyncio.run(real_read_request_body(request=captured_requests[0])) + assert followup_body.get("stream_options") == {"include_usage": True} + assert followup_body.get("input") == [{"role": "user", "content": "hello"}] From 19c875fa82f35f1bb6c484ed7a5b1d8195d77ba3 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Mon, 20 Jul 2026 14:45:59 -0700 Subject: [PATCH 013/124] refactor(responses): route function_call added-events through the shared tool-call converter --- .../transformation.py | 57 ++++--------------- 1 file changed, 11 insertions(+), 46 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index eb4acb86674..9fd193b4eeb 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -120,7 +120,11 @@ def _tool_call_dict_from_output_item(item: dict[str, Any]) -> dict[str, Any]: "type": "function", } provider_specific_fields = item.get("provider_specific_fields") - if isinstance(provider_specific_fields, dict) and provider_specific_fields: + if provider_specific_fields and not isinstance(provider_specific_fields, dict): + provider_specific_fields = ( + dict(provider_specific_fields) if hasattr(provider_specific_fields, "__dict__") else None + ) + if provider_specific_fields: tool_call_dict["provider_specific_fields"] = provider_specific_fields tool_call_dict["function"]["provider_specific_fields"] = provider_specific_fields return tool_call_dict @@ -1184,39 +1188,26 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): elif event_type == "response.output_item.added": # New output item added output_item = parsed_chunk.get("item", {}) - if output_item.get("type") == "function_call": - # Extract provider_specific_fields if present - provider_specific_fields = output_item.get("provider_specific_fields") - if provider_specific_fields and not isinstance(provider_specific_fields, dict): - provider_specific_fields = ( - dict(provider_specific_fields) if hasattr(provider_specific_fields, "__dict__") else {} - ) + if output_item.get("type") in ("function_call", "custom_tool_call"): + converted = _tool_call_dict_from_output_item(output_item) + provider_specific_fields = converted.get("provider_specific_fields") function_chunk = ChatCompletionToolCallFunctionChunk( - name=output_item.get("name", None), - arguments=output_item.get("arguments") or parsed_chunk.get("arguments") or "", + name=converted["function"]["name"] or None, + arguments=converted["function"]["arguments"] or parsed_chunk.get("arguments") or "", ) - if provider_specific_fields: function_chunk["provider_specific_fields"] = provider_specific_fields - from litellm.responses.litellm_completion_transformation.transformation import ( - LiteLLMCompletionResponsesConfig, - ) - tool_call_index = OpenAiResponsesToChatCompletionStreamIterator._sequential_tool_call_index( tool_call_index_map, parsed_chunk.get("output_index", 0) ) tool_call_chunk = ChatCompletionToolCallChunk( - id=LiteLLMCompletionResponsesConfig._tool_call_id_from_responses_item( - output_item.get("id"), output_item.get("call_id") - ), + id=converted["id"], index=tool_call_index, type="function", function=function_chunk, ) - - # Add provider_specific_fields if present if provider_specific_fields: tool_call_chunk.provider_specific_fields = provider_specific_fields # type: ignore @@ -1229,32 +1220,6 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): ) ] ) - elif output_item.get("type") == "custom_tool_call": - tool_call_index = OpenAiResponsesToChatCompletionStreamIterator._sequential_tool_call_index( - tool_call_index_map, parsed_chunk.get("output_index", 0) - ) - converted = _tool_call_dict_from_output_item(output_item) - return ModelResponseStream( - choices=[ - StreamingChoices( - index=0, - delta=Delta( - tool_calls=[ - ChatCompletionToolCallChunk( - id=converted["id"], - index=tool_call_index, - type="function", - function=ChatCompletionToolCallFunctionChunk( - name=converted["function"]["name"], - arguments=converted["function"]["arguments"], - ), - ) - ] - ), - finish_reason=None, - ) - ] - ) elif event_type in ( ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA, ResponsesAPIStreamEvents.CUSTOM_TOOL_CALL_INPUT_DELTA, From 96916f29a60b005f7e75173ae3aa5c0f6adbafff Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Mon, 20 Jul 2026 16:36:57 -0700 Subject: [PATCH 014/124] feat(proxy): serve the OpenAI model list at /cursor/models for BYOK base URLs --- litellm/proxy/_types.py | 2 + .../proxy/response_api_endpoints/endpoints.py | 29 ++++++ .../response_api_endpoints/test_endpoints.py | 26 ++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 92 +++++++++++++++++++ 4 files changed, 149 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 23fe7af9994..f0cefb2d55b 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -287,6 +287,8 @@ class LiteLLMRoutes(enum.Enum): "/chat/completions", "/v1/chat/completions", "/cursor/chat/completions", + "/cursor/models", + "/cursor/v1/models", # completions "/engines/{model}/completions", "/openai/deployments/{model}/completions", diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 9601e2d4fde..80abdab8583 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -22,6 +22,8 @@ from litellm.types.responses.main import DeleteResponseResult router = APIRouter() +_user_api_key_auth_dep = Depends(user_api_key_auth) + @router.post( "/v1/responses", @@ -283,6 +285,33 @@ async def responses_api( ) +@router.get( + "/cursor/models", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], +) +@router.get( + "/cursor/v1/models", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], +) +async def cursor_model_list( + user_api_key_dict: UserAPIKeyAuth = _user_api_key_auth_dep, +): + """ + OpenAI-compatible model listing for the Cursor BYOK base URL. + + Clients pointed at `/cursor` as an OpenAI-compatible base URL resolve and + verify models via `GET {base}/models` (the OpenAI SDK contract). Without this + route those requests fall through to the Cursor Cloud Agents passthrough, which + demands a Cursor API key and 401s, so key verification silently fails before any + chat request is ever sent. Delegates to the standard `/v1/models` handler. + """ + from litellm.proxy.proxy_server import model_list + + return await model_list(user_api_key_dict=user_api_key_dict) + + @router.post( "/cursor/chat/completions", dependencies=[Depends(user_api_key_auth)], diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index 0cc79658b6a..c41de4a8e40 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -881,3 +881,29 @@ def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_s followup_body = asyncio.run(real_read_request_body(request=captured_requests[0])) assert followup_body.get("stream_options") == {"include_usage": True} assert followup_body.get("input") == [{"role": "user", "content": "hello"}] + + +def test_cursor_models_route_delegates_to_model_list(): + """Clients pointed at /cursor as an OpenAI-compatible base URL resolve and + verify keys via GET {base}/models (the OpenAI SDK contract). Without a dedicated + route those requests fall through to the Cursor Cloud Agents passthrough and 401 + for lack of a Cursor API key, so BYOK verification fails before any chat request + is sent. Both /cursor/models and /cursor/v1/models must serve the standard model + list instead.""" + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + import litellm.proxy.proxy_server as ps + + model_payload = {"data": [{"id": "gpt-5.6", "object": "model"}], "object": "list"} + + app.dependency_overrides[user_api_key_auth] = _auth_override + try: + with patch.object(ps, "model_list", AsyncMock(return_value=model_payload)) as mock_model_list: + client = TestClient(app) + for path in ("/cursor/models", "/cursor/v1/models"): + response = client.get(path, headers={"Authorization": "Bearer sk-test-cursor"}) + assert response.status_code == 200, f"{path}: {response.text}" + assert response.json() == model_payload + assert mock_model_list.call_count == 2 + finally: + app.dependency_overrides.pop(user_api_key_auth, None) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 4e0b9b88995..1bc8839976d 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -2645,6 +2645,58 @@ export interface paths { patch?: never; trace?: never; }; + "/cursor/models": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Cursor Model List + * @description OpenAI-compatible model listing for the Cursor BYOK base URL. + * + * Clients pointed at `/cursor` as an OpenAI-compatible base URL resolve and + * verify models via `GET {base}/models` (the OpenAI SDK contract). Without this + * route those requests fall through to the Cursor Cloud Agents passthrough, which + * demands a Cursor API key and 401s, so key verification silently fails before any + * chat request is ever sent. Delegates to the standard `/v1/models` handler. + */ + get: operations["cursor_model_list_cursor_models_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/cursor/v1/models": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Cursor Model List + * @description OpenAI-compatible model listing for the Cursor BYOK base URL. + * + * Clients pointed at `/cursor` as an OpenAI-compatible base URL resolve and + * verify models via `GET {base}/models` (the OpenAI SDK contract). Without this + * route those requests fall through to the Cursor Cloud Agents passthrough, which + * demands a Cursor API key and 401s, so key verification silently fails before any + * chat request is ever sent. Delegates to the standard `/v1/models` handler. + */ + get: operations["cursor_model_list_cursor_v1_models_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/cursor/{endpoint}": { parameters: { query?: never; @@ -38506,6 +38558,46 @@ export interface operations { }; }; }; + cursor_model_list_cursor_models_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; + cursor_model_list_cursor_v1_models_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; cursor_proxy_route_cursor__endpoint__get: { parameters: { query?: never; From b45c99f6c58a1a6f903f7b4ad64a6150f04becc0 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Tue, 21 Jul 2026 11:49:42 -0700 Subject: [PATCH 015/124] fix(litellm): support OpenAI chat completions custom tool calls end to end Cursor Ask mode sends chat bodies whose tools array mixes nested function tools with flat Responses-style custom tools; the /cursor messages arm now nests those before delegating, published via the request parsed-body cache. Core chat parsing gains first-class custom tool call types mirroring the openai SDK union: a single dict dispatch feeds the provider-dict sinks, Delta dispatch stops both stream re-parse sites from silently swallowing custom deltas, the chunk builder accumulates custom input for spend logs, function-assuming consumers (json-mode gate, multi_tool_use repair, helicone, lunary) skip custom entries, and the chat-to-responses bridge flattens nested custom tools to the Responses flat shape --- .../transformation.py | 12 ++ litellm/integrations/helicone.py | 7 +- litellm/integrations/lunary.py | 2 +- .../convert_dict_to_response.py | 25 ++-- .../streaming_chunk_builder_utils.py | 41 +++++- .../llms/openai/chat/gpt_transformation.py | 8 +- .../proxy/response_api_endpoints/endpoints.py | 27 +++- litellm/types/utils.py | 98 ++++++++++++-- ...responses_transformation_transformation.py | 34 +++++ ...responses_transformation_transformation.py | 1 + .../test_convert_dict_to_response.py | 104 +++++++++++++++ .../test_streaming_chunk_builder_utils.py | 35 +++++ .../test_streaming_handler.py | 99 ++++++++++++++ .../response_api_endpoints/test_endpoints.py | 123 ++++++++++++++++++ tests/test_litellm/types/test_types_utils.py | 89 +++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 29 ++++- 16 files changed, 704 insertions(+), 30 deletions(-) create mode 100644 tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 9fd193b4eeb..75cee42dd55 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -20,6 +20,7 @@ from typing import ( cast, ) +from openai.types.responses.custom_tool_param import CustomToolParam from openai.types.responses.tool_param import FunctionToolParam from pydantic import BaseModel @@ -894,6 +895,17 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): description=function_tool.get("description"), ) ) + elif tool.get("type") == "custom" and isinstance(tool.get("custom"), dict): + custom_payload = tool["custom"] + flat_custom: CustomToolParam = { + "type": "custom", + "name": custom_payload.get("name", ""), + } + if custom_payload.get("description") is not None: + flat_custom["description"] = custom_payload["description"] + if custom_payload.get("format") is not None: + flat_custom["format"] = custom_payload["format"] + responses_tools.append(flat_custom) else: responses_tools.append(tool) # type: ignore diff --git a/litellm/integrations/helicone.py b/litellm/integrations/helicone.py index 21e9479491e..4c7a606c16f 100644 --- a/litellm/integrations/helicone.py +++ b/litellm/integrations/helicone.py @@ -59,12 +59,15 @@ class HeliconeLogger: content = [] if "tool_calls" in message and message["tool_calls"]: for tool_call in message["tool_calls"]: + function = tool_call.get("function") + if not function: + continue content.append( { "type": "tool_use", "id": tool_call["id"], - "name": tool_call["function"]["name"], - "input": tool_call["function"]["arguments"], + "name": function["name"], + "input": function["arguments"], } ) elif "content" in message and message["content"]: diff --git a/litellm/integrations/lunary.py b/litellm/integrations/lunary.py index aaf5751cb79..448580f0b2d 100644 --- a/litellm/integrations/lunary.py +++ b/litellm/integrations/lunary.py @@ -31,7 +31,7 @@ def parse_tool_calls(tool_calls): return serialized - return [clean_tool_call(tool_call) for tool_call in tool_calls] + return [clean_tool_call(tool_call) for tool_call in tool_calls if getattr(tool_call, "function", None) is not None] def parse_messages(input): diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 47daf33824e..c5cfdea9ffe 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -19,6 +19,7 @@ from litellm.types.llms.openai import ( ) from litellm.types.utils import ( ChatCompletionDeltaToolCall, + ChatCompletionMessageCustomToolCall, ChatCompletionMessageToolCall, ChatCompletionRedactedThinkingBlock, Choices, @@ -43,6 +44,7 @@ from litellm.types.utils import ( TranscriptionUsageDurationObject, TranscriptionUsageTokensObject, Usage, + chat_completion_tool_call_from_dict, ) from .get_headers import get_response_headers @@ -369,7 +371,7 @@ from collections import defaultdict def _handle_invalid_parallel_tool_calls( - tool_calls: List[ChatCompletionMessageToolCall], + tool_calls: List[Union[ChatCompletionMessageToolCall, ChatCompletionMessageCustomToolCall]], ): """ Handle hallucinated parallel tool call from openai - https://community.openai.com/t/model-tries-to-call-unknown-function-multi-tool-use-parallel/490653 @@ -382,6 +384,8 @@ def _handle_invalid_parallel_tool_calls( try: replacements: Dict[int, List[ChatCompletionMessageToolCall]] = defaultdict(list) for i, tool_call in enumerate(tool_calls): + if isinstance(tool_call, ChatCompletionMessageCustomToolCall): + continue current_function = tool_call.function.name function_args = json.loads(tool_call.function.arguments) if current_function == "multi_tool_use.parallel": @@ -527,19 +531,20 @@ class LiteLLMResponseObjectHandler: def _should_convert_tool_call_to_json_mode( - tool_calls: Optional[Union[List[ChatCompletionMessageToolCall], List[DatabricksTool]]] = None, + tool_calls: Optional[ + Union[ + List[Union[ChatCompletionMessageToolCall, ChatCompletionMessageCustomToolCall]], + List[DatabricksTool], + ] + ] = None, convert_tool_call_to_json_mode: Optional[bool] = None, ) -> bool: """ Determine if tool calls should be converted to JSON mode """ - if ( - convert_tool_call_to_json_mode - and tool_calls is not None - and len(tool_calls) == 1 - and tool_calls[0]["function"]["name"] == RESPONSE_FORMAT_TOOL_NAME - ): - return True + if convert_tool_call_to_json_mode and tool_calls is not None and len(tool_calls) == 1: + function = tool_calls[0].get("function") + return function is not None and function["name"] == RESPONSE_FORMAT_TOOL_NAME return False @@ -647,7 +652,7 @@ def convert_to_model_response_object( if tool_calls is not None: _openai_tool_calls = [] for _tc in tool_calls: - _openai_tc = ChatCompletionMessageToolCall(**_tc) + _openai_tc = chat_completion_tool_call_from_dict(_tc) _openai_tool_calls.append(_openai_tc) fixed_tool_calls = _handle_invalid_parallel_tool_calls(_openai_tool_calls) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index d52d9849310..09bd55096e8 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -9,6 +9,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ( CacheCreationTokenDetails, ChatCompletionAudioResponse, + ChatCompletionCustomToolCallPayload, + ChatCompletionMessageCustomToolCall, ChatCompletionMessageToolCall, Choices, CompletionTokensDetails, @@ -202,8 +204,10 @@ class ChunkProcessor: response = self.update_model_response_with_hidden_params(model_response=response, chunk=chunk) return response - def get_combined_tool_content(self, tool_call_chunks: List[Dict[str, Any]]) -> List[ChatCompletionMessageToolCall]: - tool_calls_list: List[ChatCompletionMessageToolCall] = [] + def get_combined_tool_content( + self, tool_call_chunks: List[Dict[str, Any]] + ) -> List[Union[ChatCompletionMessageToolCall, ChatCompletionMessageCustomToolCall]]: + tool_calls_list: List[Union[ChatCompletionMessageToolCall, ChatCompletionMessageCustomToolCall]] = [] tool_call_map: Dict[int, Dict[str, Any]] = {} # Map to store tool calls by index for chunk in tool_call_chunks: @@ -219,12 +223,15 @@ class ChunkProcessor: # Check if tool_call has function (either as attribute or dict key) has_function = False + has_custom = False if isinstance(tool_call, dict): has_function = "function" in tool_call and tool_call["function"] is not None + has_custom = "custom" in tool_call and tool_call["custom"] is not None else: has_function = hasattr(tool_call, "function") and tool_call.function is not None + has_custom = getattr(tool_call, "custom", None) is not None - if not has_function: + if not has_function and not has_custom: continue # Get index (handle both dict and object) @@ -239,6 +246,8 @@ class ChunkProcessor: "name": None, "type": None, "arguments": [], + "custom_name": None, + "custom_input": [], "provider_specific_fields": None, } @@ -261,6 +270,13 @@ class ChunkProcessor: tool_call_map[index]["name"] = function.name if hasattr(function, "arguments") and function.arguments: tool_call_map[index]["arguments"].append(function.arguments) + + custom = tool_call.get("custom") + if isinstance(custom, dict): + if custom.get("name"): + tool_call_map[index]["custom_name"] = custom["name"] + if custom.get("input"): + tool_call_map[index]["custom_input"].append(custom["input"]) else: # tool_call is an object if hasattr(tool_call, "id") and tool_call.id: @@ -273,6 +289,13 @@ class ChunkProcessor: if hasattr(tool_call.function, "arguments") and tool_call.function.arguments: tool_call_map[index]["arguments"].append(tool_call.function.arguments) + custom = getattr(tool_call, "custom", None) + if custom is not None: + if getattr(custom, "name", None): + tool_call_map[index]["custom_name"] = custom.name + if getattr(custom, "input", None): + tool_call_map[index]["custom_input"].append(custom.input) + # Preserve provider_specific_fields from streaming chunks provider_fields = None if isinstance(tool_call, dict): @@ -299,7 +322,17 @@ class ChunkProcessor: # Convert the map to a list of tool calls for index in sorted(tool_call_map.keys()): tool_call_data = tool_call_map[index] - if tool_call_data["id"] and tool_call_data["name"]: + if tool_call_data["type"] == "custom" and tool_call_data["id"] and tool_call_data["custom_name"]: + tool_calls_list.append( + ChatCompletionMessageCustomToolCall( + id=tool_call_data["id"], + custom=ChatCompletionCustomToolCallPayload( + name=tool_call_data["custom_name"], + input="".join(tool_call_data["custom_input"]), + ), + ) + ) + elif tool_call_data["id"] and tool_call_data["name"]: combined_arguments = "".join(tool_call_data["arguments"]) or "{}" # Build function - provider_specific_fields should be on tool_call level, not function level diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index f2498c0a7e2..129a9b51d0d 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -50,12 +50,14 @@ from litellm.types.llms.openai import ( OpenAIMessageContentListBlock, ) from litellm.types.utils import ( + ChatCompletionMessageCustomToolCall, ChatCompletionMessageToolCall, Choices, Function, Message, ModelResponse, ModelResponseStream, + chat_completion_tool_call_from_dict, ) from litellm.utils import convert_to_model_response_object @@ -531,12 +533,14 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): for choice in choices: ## HANDLE JSON MODE - anthropic returns single function call] tool_calls = choice["message"].get("tool_calls", None) - new_tool_calls: Optional[List[ChatCompletionMessageToolCall]] = None + new_tool_calls: Optional[ + List[Union[ChatCompletionMessageToolCall, ChatCompletionMessageCustomToolCall]] + ] = None message_content = choice["message"].get("content", None) if tool_calls is not None: _openai_tool_calls = [] for _tc in tool_calls: - _openai_tc = ChatCompletionMessageToolCall(**_tc) # type: ignore + _openai_tc = chat_completion_tool_call_from_dict(dict(_tc)) _openai_tool_calls.append(_openai_tc) fixed_tool_calls = _handle_invalid_parallel_tool_calls(_openai_tool_calls) diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 80abdab8583..f9b2cc79f73 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -24,6 +24,23 @@ router = APIRouter() _user_api_key_auth_dep = Depends(user_api_key_auth) +_FLAT_CUSTOM_TOOL_KEYS = ("name", "description", "format") +_FLAT_FUNCTION_TOOL_KEYS = ("name", "description", "parameters", "strict") + + +def _nest_flat_chat_tool(tool: object) -> object: + if not isinstance(tool, dict) or "name" not in tool: + return tool + if tool.get("type") == "custom" and "custom" not in tool: + return {"type": "custom", "custom": {k: tool[k] for k in _FLAT_CUSTOM_TOOL_KEYS if k in tool}} + if tool.get("type") == "function" and "function" not in tool: + return {"type": "function", "function": {k: tool[k] for k in _FLAT_FUNCTION_TOOL_KEYS if k in tool}} + return tool + + +def _nest_flat_chat_tools(tools: list) -> list: + return [_nest_flat_chat_tool(tool) for tool in tools] + @router.post( "/v1/responses", @@ -330,7 +347,9 @@ async def cursor_chat_completions( custom tools) to the chat/completions path while expecting chat completions responses; those are routed through the Responses API pipeline and converted back. Genuine chat completions bodies (`messages` present) are routed through the standard chat completions - pipeline untouched. + pipeline, after nesting any flat Responses-style tool defs Cursor mixes into the chat + `tools` array (e.g. `{"type": "custom", "name": "ApplyPatch", ...}`) into the chat + completions shape OpenAI requires (`{"type": "custom", "custom": {...}}`). ```bash curl -X POST http://localhost:4000/cursor/chat/completions \ @@ -347,6 +366,7 @@ async def cursor_chat_completions( responses_api_bridge, ) from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.proxy.common_utils.http_parsing_utils import _safe_set_request_parsed_body from litellm.proxy.proxy_server import ( _read_request_body, async_data_generator, @@ -370,6 +390,11 @@ async def cursor_chat_completions( if "messages" in data: # Genuine chat completions body (Cursor sends these for models whose BYOK it # already fixed); delegate so behavior matches /chat/completions exactly + tools = data.get("tools") + if isinstance(tools, list): + nested_tools = _nest_flat_chat_tools(tools) + if nested_tools != tools: + _safe_set_request_parsed_body(request=request, parsed_body={**data, "tools": nested_tools}) return await chat_completion( request=request, fastapi_response=fastapi_response, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 18991f53e6f..a1e52f7584f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1084,6 +1084,71 @@ class ChatCompletionDeltaToolCall(OpenAIObject): setattr(self, key, value) +class ChatCompletionCustomToolCallPayload(OpenAIObject): + name: str + input: str + + def __contains__(self, key): + return hasattr(self, key) + + def get(self, key, default=None): + return getattr(self, key, default) + + def __getitem__(self, key): + return getattr(self, key) + + +class ChatCompletionDeltaCustomToolCallPayload(OpenAIObject): + name: str | None = None + input: str | None = None + + def __contains__(self, key): + return hasattr(self, key) + + def get(self, key, default=None): + return getattr(self, key, default) + + def __getitem__(self, key): + return getattr(self, key) + + +class ChatCompletionMessageCustomToolCall(OpenAIObject): + id: str + type: Literal["custom"] = "custom" + custom: ChatCompletionCustomToolCallPayload + + def __contains__(self, key): + return hasattr(self, key) + + def get(self, key, default=None): + return getattr(self, key, default) + + def __getitem__(self, key): + return getattr(self, key) + + def __setitem__(self, key, value): + setattr(self, key, value) + + +class ChatCompletionDeltaCustomToolCall(OpenAIObject): + id: str | None = None + type: str | None = None + custom: ChatCompletionDeltaCustomToolCallPayload + index: int + + def __contains__(self, key): + return hasattr(self, key) + + def get(self, key, default=None): + return getattr(self, key, default) + + def __getitem__(self, key): + return getattr(self, key) + + def __setitem__(self, key, value): + setattr(self, key, value) + + class ChatCompletionMessageToolCall(OpenAIObject): def __init__( self, @@ -1125,6 +1190,16 @@ class ChatCompletionMessageToolCall(OpenAIObject): setattr(self, key, value) +def chat_completion_tool_call_from_dict( + tool_call: dict, +) -> "ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall": + if tool_call.get("type") == "custom": + return ChatCompletionMessageCustomToolCall( + **{k: v for k, v in tool_call.items() if not (k == "function" and v is None)} + ) + return ChatCompletionMessageToolCall(**tool_call) + + from openai.types.chat.chat_completion_audio import ChatCompletionAudio @@ -1177,7 +1252,7 @@ def add_provider_specific_fields(object: BaseModel, provider_specific_fields: Op class Message(SafeAttributeModel, OpenAIObject): content: Optional[str] role: Literal["assistant", "user", "system", "tool", "function"] - tool_calls: Optional[List[ChatCompletionMessageToolCall]] + tool_calls: Optional[List[Union[ChatCompletionMessageToolCall, ChatCompletionMessageCustomToolCall]]] function_call: Optional[FunctionCall] audio: Optional[ChatCompletionAudioResponse] = None images: Optional[List[ImageURLListItem]] = None @@ -1208,7 +1283,7 @@ class Message(SafeAttributeModel, OpenAIObject): "function_call": (FunctionCall(**function_call) if function_call is not None else None), "tool_calls": ( [ - (ChatCompletionMessageToolCall(**tool_call) if isinstance(tool_call, dict) else tool_call) + (chat_completion_tool_call_from_dict(tool_call) if isinstance(tool_call, dict) else tool_call) for tool_call in tool_calls ] if tool_calls is not None and len(tool_calls) > 0 @@ -1301,7 +1376,7 @@ class Delta(SafeAttributeModel, OpenAIObject): content: Optional[str] role: Optional[str] function_call: Optional[FunctionCall] - tool_calls: Optional[List[ChatCompletionDeltaToolCall]] + tool_calls: Optional[List[Union[ChatCompletionDeltaToolCall, ChatCompletionDeltaCustomToolCall]]] audio: Optional[ChatCompletionAudioResponse] images: Optional[List[ImageURLListItem]] annotations: Optional[List[ChatCompletionAnnotation]] @@ -1339,17 +1414,24 @@ class Delta(SafeAttributeModel, OpenAIObject): function_call = FunctionCall(**function_call) if tool_calls is not None and isinstance(tool_calls, list): - coerced_tool_calls: List[ChatCompletionDeltaToolCall] = [] + coerced_tool_calls: List[Union[ChatCompletionDeltaToolCall, ChatCompletionDeltaCustomToolCall]] = [] current_index = 0 for tool_call in tool_calls: if isinstance(tool_call, dict): if tool_call.get("index", None) is None: tool_call["index"] = current_index current_index += 1 - if tool_call.get("type", None) is None: - tool_call["type"] = "function" - coerced_tool_calls.append(ChatCompletionDeltaToolCall(**tool_call)) - elif isinstance(tool_call, ChatCompletionDeltaToolCall): + if tool_call.get("type") == "custom" or "custom" in tool_call: + coerced_tool_calls.append( + ChatCompletionDeltaCustomToolCall( + **{k: v for k, v in tool_call.items() if not (k == "function" and v is None)} + ) + ) + else: + if tool_call.get("type", None) is None: + tool_call["type"] = "function" + coerced_tool_calls.append(ChatCompletionDeltaToolCall(**tool_call)) + elif isinstance(tool_call, (ChatCompletionDeltaToolCall, ChatCompletionDeltaCustomToolCall)): coerced_tool_calls.append(tool_call) tool_calls = coerced_tool_calls diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 36c32d4b3a0..767d1631649 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -3215,3 +3215,37 @@ def test_convert_response_output_generic_pydantic_message_item(): assert len(choices) == 1 assert choices[0].message.content == "42" assert choices[0].finish_reason == "stop" + + +def test_convert_tools_to_responses_format_flattens_nested_custom_tool(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + tools = [ + { + "type": "custom", + "custom": {"name": "ApplyPatch", "description": "V4A patch", "format": {"type": "text"}}, + }, + {"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}}, + ] + converted = handler._convert_tools_to_responses_format(tools) + assert converted[0] == { + "type": "custom", + "name": "ApplyPatch", + "description": "V4A patch", + "format": {"type": "text"}, + } + assert converted[1]["type"] == "function" + assert converted[1]["name"] == "f" + + +def test_convert_tools_to_responses_format_flattens_custom_tool_without_optional_keys(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + converted = handler._convert_tools_to_responses_format([{"type": "custom", "custom": {"name": "Minimal"}}]) + assert converted[0] == {"type": "custom", "name": "Minimal"} diff --git a/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py index 05bdc40112c..626e554d476 100644 --- a/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py @@ -257,3 +257,4 @@ def test_translate_responses_chunk_passthrough_chat_completion_chunk(): assert result.choices[0].delta.content == "Hi! How can I help?" assert result.choices[0].finish_reason is None + diff --git a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py new file mode 100644 index 00000000000..293e5de304f --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -0,0 +1,104 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../..")) + +from litellm.constants import RESPONSE_FORMAT_TOOL_NAME +from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + _handle_invalid_parallel_tool_calls, + _should_convert_tool_call_to_json_mode, + convert_to_model_response_object, +) +from litellm.types.utils import ( + ChatCompletionMessageCustomToolCall, + ChatCompletionMessageToolCall, + Function, + ModelResponse, +) + +OPENAI_CUSTOM_TOOL_CALL_RESPONSE = { + "id": "chatcmpl-abc", + "created": 1784657740, + "model": "gpt-5.6", + "object": "chat.completion", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_njxQ", + "type": "custom", + "custom": { + "name": "ApplyPatch", + "input": "*** Begin Patch\n*** Update File: main.py\n@@\n+def hello():\n+ print(\"Hello\")\n*** End Patch\n", + }, + } + ], + "refusal": None, + "annotations": [], + }, + } + ], + "usage": {"completion_tokens": 10, "prompt_tokens": 5, "total_tokens": 15}, +} + + +def test_convert_openai_custom_tool_call_response(): + result = convert_to_model_response_object( + response_object=OPENAI_CUSTOM_TOOL_CALL_RESPONSE, + model_response_object=ModelResponse(), + response_type="completion", + ) + tool_calls = result.choices[0].message.tool_calls + assert len(tool_calls) == 1 + assert isinstance(tool_calls[0], ChatCompletionMessageCustomToolCall) + dumped = tool_calls[0].model_dump() + assert dumped == OPENAI_CUSTOM_TOOL_CALL_RESPONSE["choices"][0]["message"]["tool_calls"][0] + assert result.choices[0].finish_reason == "tool_calls" + + +def test_should_convert_tool_call_to_json_mode_ignores_custom_tool_call(): + custom_tool_call = ChatCompletionMessageCustomToolCall( + id="call_c", + custom={"name": "ApplyPatch", "input": "patch"}, + ) + assert ( + _should_convert_tool_call_to_json_mode( + tool_calls=[custom_tool_call], + convert_tool_call_to_json_mode=True, + ) + is False + ) + + +def test_should_convert_tool_call_to_json_mode_still_matches_response_format_tool(): + response_format_call = ChatCompletionMessageToolCall( + id="call_f", + type="function", + function=Function(name=RESPONSE_FORMAT_TOOL_NAME, arguments='{"answer": 4}'), + ) + assert ( + _should_convert_tool_call_to_json_mode( + tool_calls=[response_format_call], + convert_tool_call_to_json_mode=True, + ) + is True + ) + + +def test_handle_invalid_parallel_tool_calls_skips_custom_tool_calls(): + custom_tool_call = ChatCompletionMessageCustomToolCall( + id="call_c", + custom={"name": "ApplyPatch", "input": "patch"}, + ) + function_tool_call = ChatCompletionMessageToolCall( + id="call_f", + type="function", + function=Function(name="get_weather", arguments='{"city": "SF"}'), + ) + result = _handle_invalid_parallel_tool_calls([custom_tool_call, function_tool_call]) + assert result == [custom_tool_call, function_tool_call] diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index be8c5a05601..197adf80f03 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -992,3 +992,38 @@ def test_cost_field_in_usage_chunks(): assert usage.cost == 0.00025 assert usage.prompt_tokens == 10 assert usage.completion_tokens == 5 + + +def test_get_combined_tool_content_custom_tool_call(): + from litellm.litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor + from litellm.types.utils import ChatCompletionMessageCustomToolCall + + processor = ChunkProcessor.__new__(ChunkProcessor) + tool_call_chunks = [ + { + "choices": [ + { + "delta": { + "tool_calls": [ + { + "index": 0, + "id": "call_TBs", + "type": "custom", + "custom": {"name": "ApplyPatch", "input": ""}, + } + ] + } + } + ] + }, + {"choices": [{"delta": {"tool_calls": [{"index": 0, "custom": {"input": "*** Begin Patch\n"}}]}}]}, + {"choices": [{"delta": {"tool_calls": [{"index": 0, "custom": {"input": "*** End Patch\n"}}]}}]}, + ] + combined = processor.get_combined_tool_content(tool_call_chunks) + assert len(combined) == 1 + assert isinstance(combined[0], ChatCompletionMessageCustomToolCall) + assert combined[0].model_dump() == { + "id": "call_TBs", + "type": "custom", + "custom": {"name": "ApplyPatch", "input": "*** Begin Patch\n*** End Patch\n"}, + } diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 514714136fd..48bc3709517 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -3355,3 +3355,102 @@ async def test_transport_read_error_before_finish_reason_raises(logging_obj: Log if chunk.choices and chunk.choices[0].finish_reason ] assert fabricated_finish_reasons == [] + + +def test_openai_custom_tool_call_stream_deltas_survive_conversion(logging_obj: Logging): + """ + Regression test: OpenAI chat completions custom tool calls stream as + delta.tool_calls entries with a `custom` payload and NO `function` key. + Delta() used to raise on those dicts and chunk_creator's except branch + replaced the choice with an empty Delta, silently dropping the entire + tool call from the client stream. + """ + from openai.types.chat.chat_completion_chunk import ChatCompletionChunk + + from litellm.types.utils import ChatCompletionDeltaCustomToolCall + + raw_chunks = [ + { + "id": "chatcmpl-custom", + "object": "chat.completion.chunk", + "created": 1784657671, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "delta": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "index": 0, + "id": "call_TBs", + "type": "custom", + "custom": {"name": "ApplyPatch", "input": ""}, + } + ], + }, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-custom", + "object": "chat.completion.chunk", + "created": 1784657671, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "delta": {"tool_calls": [{"index": 0, "custom": {"input": "*** Begin Patch\n"}}]}, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-custom", + "object": "chat.completion.chunk", + "created": 1784657671, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "delta": {"tool_calls": [{"index": 0, "custom": {"input": "*** End Patch\n"}}]}, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-custom", + "object": "chat.completion.chunk", + "created": 1784657671, + "model": "gpt-5.6", + "choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}], + }, + ] + sdk_chunks = [ChatCompletionChunk.construct(**raw) for raw in raw_chunks] + first_dumped = sdk_chunks[0].choices[0].model_dump() + assert first_dumped["delta"]["tool_calls"][0]["custom"] == {"name": "ApplyPatch", "input": ""} + + wrapper = CustomStreamWrapper( + completion_stream=iter(sdk_chunks), + model="gpt-5.6", + custom_llm_provider="openai", + logging_obj=logging_obj, + ) + + emitted = list(wrapper) + tool_call_deltas = [ + chunk.choices[0].delta.tool_calls[0] + for chunk in emitted + if chunk.choices and chunk.choices[0].delta and chunk.choices[0].delta.tool_calls + ] + assert len(tool_call_deltas) == 3 + assert isinstance(tool_call_deltas[0], ChatCompletionDeltaCustomToolCall) + assert tool_call_deltas[0].id == "call_TBs" + assert tool_call_deltas[0].type == "custom" + assert tool_call_deltas[0].custom.name == "ApplyPatch" + combined_input = "".join(tc.custom.input or "" for tc in tool_call_deltas) + assert combined_input == "*** Begin Patch\n*** End Patch\n" + finish_reasons = [chunk.choices[0].finish_reason for chunk in emitted if chunk.choices] + assert "tool_calls" in finish_reasons diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index c41de4a8e40..4ef338b35e4 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -907,3 +907,126 @@ def test_cursor_models_route_delegates_to_model_list(): assert mock_model_list.call_count == 2 finally: app.dependency_overrides.pop(user_api_key_auth, None) + + +class TestNestFlatChatTools: + def test_flat_custom_tool_is_nested(self): + from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tools + + result = _nest_flat_chat_tools( + [{"type": "custom", "name": "ApplyPatch", "description": "V4A patch", "format": {"type": "text"}}] + ) + assert result == [ + { + "type": "custom", + "custom": {"name": "ApplyPatch", "description": "V4A patch", "format": {"type": "text"}}, + } + ] + + def test_flat_function_tool_is_nested(self): + from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tools + + result = _nest_flat_chat_tools( + [{"type": "function", "name": "read_file", "description": "d", "parameters": {"type": "object"}}] + ) + assert result == [ + { + "type": "function", + "function": {"name": "read_file", "description": "d", "parameters": {"type": "object"}}, + } + ] + + def test_already_nested_and_unrecognized_tools_pass_through_unchanged(self): + from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tools + + tools = [ + {"type": "custom", "custom": {"name": "already_nested"}}, + {"type": "function", "function": {"name": "f", "parameters": {}}}, + {"type": "web_search"}, + {"type": "custom"}, + {"name": "typeless"}, + {}, + "junk", + None, + 42, + ] + assert _nest_flat_chat_tools(tools) == tools + + +class TestCursorMessagesArmToolNormalization: + @pytest.mark.asyncio + async def test_flat_custom_tool_nested_before_chat_completion_delegation(self): + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy._types import UserAPIKeyAuth + + seen = {} + + async def fake_chat_completion(request, fastapi_response, model, user_api_key_dict): + from litellm.proxy.common_utils.http_parsing_utils import _read_request_body + + seen["body"] = await _read_request_body(request=request) + return {"id": "chatcmpl-fake", "object": "chat.completion", "choices": []} + + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-1234") + try: + with patch("litellm.proxy.proxy_server.chat_completion", new=fake_chat_completion): + client = TestClient(app) + response = client.post( + "/cursor/chat/completions", + json={ + "model": "gpt-5.6", + "messages": [{"role": "user", "content": "use ApplyPatch"}], + "tools": [ + { + "type": "function", + "function": {"name": "read_file", "parameters": {"type": "object"}}, + }, + {"type": "custom", "name": "ApplyPatch", "description": "V4A patch"}, + ], + "tool_choice": "required", + }, + headers={"Authorization": "Bearer sk-1234"}, + ) + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 200 + assert seen["body"]["tools"] == [ + {"type": "function", "function": {"name": "read_file", "parameters": {"type": "object"}}}, + {"type": "custom", "custom": {"name": "ApplyPatch", "description": "V4A patch"}}, + ] + assert seen["body"]["messages"] == [{"role": "user", "content": "use ApplyPatch"}] + + @pytest.mark.asyncio + async def test_messages_body_without_flat_tools_leaves_parsed_body_cache_untouched(self): + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy._types import UserAPIKeyAuth + + seen = {} + + async def fake_chat_completion(request, fastapi_response, model, user_api_key_dict): + from litellm.proxy.common_utils.http_parsing_utils import _read_request_body + + seen["body"] = await _read_request_body(request=request) + return {"id": "chatcmpl-fake", "object": "chat.completion", "choices": []} + + body = { + "model": "gpt-5.6", + "messages": [{"role": "user", "content": "hi"}], + "tools": [{"type": "function", "function": {"name": "f", "parameters": {}}}], + } + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-1234") + try: + with patch("litellm.proxy.proxy_server.chat_completion", new=fake_chat_completion): + client = TestClient(app) + response = client.post( + "/cursor/chat/completions", + json=body, + headers={"Authorization": "Bearer sk-1234"}, + ) + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 200 + assert seen["body"]["tools"] == body["tools"] + assert seen["body"]["messages"] == body["messages"] diff --git a/tests/test_litellm/types/test_types_utils.py b/tests/test_litellm/types/test_types_utils.py index 320c46aed3b..4d08239360f 100644 --- a/tests/test_litellm/types/test_types_utils.py +++ b/tests/test_litellm/types/test_types_utils.py @@ -603,3 +603,92 @@ def test_delattr_fast_path_missing_attribute_is_noop(): del racy.x del racy.x +def test_chat_completion_tool_call_from_dict_custom(): + from litellm.types.utils import ( + ChatCompletionMessageCustomToolCall, + ChatCompletionMessageToolCall, + chat_completion_tool_call_from_dict, + ) + + custom_tc = { + "id": "call_njxQ", + "type": "custom", + "custom": {"name": "ApplyPatch", "input": "*** Begin Patch\n*** End Patch\n"}, + } + parsed = chat_completion_tool_call_from_dict(custom_tc) + assert isinstance(parsed, ChatCompletionMessageCustomToolCall) + assert parsed.model_dump() == custom_tc + + func_tc = {"id": "call_1", "type": "function", "function": {"name": "f", "arguments": "{}"}} + parsed_func = chat_completion_tool_call_from_dict(func_tc) + assert isinstance(parsed_func, ChatCompletionMessageToolCall) + assert "custom" not in parsed_func.model_dump() + + +def test_chat_completion_tool_call_from_dict_custom_strips_null_function(): + from litellm.types.utils import chat_completion_tool_call_from_dict + + sdk_shaped = { + "id": "call_x", + "type": "custom", + "function": None, + "custom": {"name": "ApplyPatch", "input": ""}, + } + parsed = chat_completion_tool_call_from_dict(sdk_shaped) + assert "function" not in parsed.model_dump() + + +def test_message_with_mixed_function_and_custom_tool_calls(): + from litellm.types.utils import ( + ChatCompletionMessageCustomToolCall, + ChatCompletionMessageToolCall, + Message, + ) + + message = Message( + content=None, + role="assistant", + tool_calls=[ + {"id": "call_c", "type": "custom", "custom": {"name": "ApplyPatch", "input": "patch"}}, + {"id": "call_f", "type": "function", "function": {"name": "f", "arguments": "{}"}}, + ], + ) + assert isinstance(message.tool_calls[0], ChatCompletionMessageCustomToolCall) + assert isinstance(message.tool_calls[1], ChatCompletionMessageToolCall) + dumped = message.model_dump()["tool_calls"] + assert dumped[0] == {"id": "call_c", "type": "custom", "custom": {"name": "ApplyPatch", "input": "patch"}} + assert "custom" not in dumped[1] + + +def test_delta_custom_tool_call_first_and_continuation_chunks(): + from litellm.types.utils import ChatCompletionDeltaCustomToolCall, Delta + + first_chunk_tc = { + "index": 0, + "id": "call_TBs", + "function": None, + "type": "custom", + "custom": {"name": "ApplyPatch", "input": ""}, + } + continuation_tc = {"index": 0, "id": None, "function": None, "type": None, "custom": {"input": "***"}} + + first_delta = Delta(role="assistant", tool_calls=[first_chunk_tc]) + assert isinstance(first_delta.tool_calls[0], ChatCompletionDeltaCustomToolCall) + first_dump = first_delta.model_dump()["tool_calls"][0] + assert first_dump["type"] == "custom" + assert first_dump["custom"] == {"name": "ApplyPatch", "input": ""} + assert "function" not in first_dump + + continuation_delta = Delta(tool_calls=[continuation_tc]) + cont_dump = continuation_delta.model_dump()["tool_calls"][0] + assert cont_dump["type"] is None + assert cont_dump["custom"]["input"] == "***" + assert "function" not in cont_dump + + +def test_delta_function_tool_call_unchanged_by_custom_support(): + from litellm.types.utils import ChatCompletionDeltaToolCall, Delta + + delta = Delta(tool_calls=[{"index": 0, "id": "c2", "type": "function", "function": {"name": "g", "arguments": ""}}]) + assert isinstance(delta.tool_calls[0], ChatCompletionDeltaToolCall) + assert "custom" not in delta.model_dump()["tool_calls"][0] diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 1bc8839976d..bf3cc75c796 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -2628,7 +2628,9 @@ export interface paths { * custom tools) to the chat/completions path while expecting chat completions responses; * those are routed through the Responses API pipeline and converted back. Genuine chat * completions bodies (`messages` present) are routed through the standard chat completions - * pipeline untouched. + * pipeline, after nesting any flat Responses-style tool defs Cursor mixes into the chat + * `tools` array (e.g. `{"type": "custom", "name": "ApplyPatch", ...}`) into the chat + * completions shape OpenAI requires (`{"type": "custom", "custom": {...}}`). * * ```bash * curl -X POST http://localhost:4000/cursor/chat/completions -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{ @@ -22195,6 +22197,15 @@ export interface components { */ type: "ephemeral"; }; + /** ChatCompletionCustomToolCallPayload */ + ChatCompletionCustomToolCallPayload: { + /** Input */ + input: string; + /** Name */ + name: string; + } & { + [key: string]: unknown; + }; /** ChatCompletionDeveloperMessage */ ChatCompletionDeveloperMessage: { cache_control?: components["schemas"]["ChatCompletionCachedContent"]; @@ -22281,6 +22292,20 @@ export interface components { /** Url */ url: string; }; + /** ChatCompletionMessageCustomToolCall */ + ChatCompletionMessageCustomToolCall: { + custom: components["schemas"]["ChatCompletionCustomToolCallPayload"]; + /** Id */ + id: string; + /** + * Type + * @default custom + * @constant + */ + type: "custom"; + } & { + [key: string]: unknown; + }; /** ChatCompletionMessageToolCall */ ChatCompletionMessageToolCall: { [key: string]: unknown; @@ -27909,7 +27934,7 @@ export interface components { /** Thinking Blocks */ thinking_blocks?: (components["schemas"]["ChatCompletionThinkingBlock"] | components["schemas"]["ChatCompletionRedactedThinkingBlock"])[] | null; /** Tool Calls */ - tool_calls: components["schemas"]["ChatCompletionMessageToolCall"][] | null; + tool_calls: (components["schemas"]["ChatCompletionMessageToolCall"] | components["schemas"]["ChatCompletionMessageCustomToolCall"])[] | null; } & { [key: string]: unknown; }; From b79b01e38afbe740530efb294c98e116d2bf9f6c Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Tue, 21 Jul 2026 13:59:23 -0700 Subject: [PATCH 016/124] fix(proxy): translate custom tool grammar formats and tool_choice across API surfaces Cursor's ApplyPatch is a grammar-constrained custom tool; the Responses surface carries the grammar flat while chat completions wraps the same fields in a grammar object, so the nested envelope from the previous commit still 400d at OpenAI (tools[N].custom.format.grammar). Adds a shared flat to nested format helper pair in prompt_templates/common_utils used by the cursor messages arm and the chat-to-responses bridge, nests flat Responses-style tool_choice objects on the cursor arm, flattens chat custom tool_choice on the chat-to-responses bridge, and maps custom tool_choice to function tool_choice on the responses-to-chat bridge to match that bridge's custom-to-function tool downgrade --- .../transformation.py | 29 +++--- .../prompt_templates/common_utils.py | 25 +++++ .../proxy/response_api_endpoints/endpoints.py | 28 +++++- .../transformation.py | 6 ++ ...responses_transformation_transformation.py | 48 ++++++++++ ...ore_utils_prompt_templates_common_utils.py | 45 +++++++++ .../response_api_endpoints/test_endpoints.py | 96 ++++++++++++++++++- .../test_litellm_completion_responses.py | 21 ++++ 8 files changed, 282 insertions(+), 16 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 75cee42dd55..e1842a62d56 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -155,17 +155,20 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): pass def _normalize_tool_choice_for_responses_api(self, tool_choice: Any) -> Any: - """Chat tool_choice uses function.name; Responses API expects top-level name.""" - if not isinstance(tool_choice, dict) or tool_choice.get("type") != "function": + """Chat tool_choice nests the name under function/custom; Responses API expects top-level name.""" + if not isinstance(tool_choice, dict): + return tool_choice + choice_type = tool_choice.get("type") + if choice_type not in ("function", "custom"): return tool_choice if isinstance(tool_choice.get("name"), str) and tool_choice.get("name"): - # Return only Responses shape so stray chat ``function`` key is not sent upstream. - return {"type": "function", "name": tool_choice["name"]} - fn = tool_choice.get("function") - if isinstance(fn, dict): - fn_name = fn.get("name") - if isinstance(fn_name, str) and fn_name: - return {"type": "function", "name": fn_name} + # Return only Responses shape so stray chat ``function``/``custom`` keys are not sent upstream. + return {"type": choice_type, "name": tool_choice["name"]} + nested = tool_choice.get(choice_type) + if isinstance(nested, dict): + nested_name = nested.get("name") + if isinstance(nested_name, str) and nested_name: + return {"type": choice_type, "name": nested_name} return tool_choice def _handle_raw_dict_response_item(self, item: Dict[str, Any], index: int) -> Tuple[Optional[Any], int]: @@ -896,6 +899,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) ) elif tool.get("type") == "custom" and isinstance(tool.get("custom"), dict): + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + convert_custom_tool_format_to_responses_shape, + ) + custom_payload = tool["custom"] flat_custom: CustomToolParam = { "type": "custom", @@ -903,8 +910,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): } if custom_payload.get("description") is not None: flat_custom["description"] = custom_payload["description"] - if custom_payload.get("format") is not None: - flat_custom["format"] = custom_payload["format"] + if isinstance(custom_payload.get("format"), dict): + flat_custom["format"] = convert_custom_tool_format_to_responses_shape(custom_payload["format"]) responses_tools.append(flat_custom) else: responses_tools.append(tool) # type: ignore diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index c43089950ee..3a7a710c6a9 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1252,6 +1252,31 @@ def is_function_call(optional_params: dict) -> bool: return False +def convert_custom_tool_format_to_chat_shape(format_obj: dict) -> dict: + """ + Responses API grammar formats are flat ({"type": "grammar", "definition", "syntax"}); + Chat Completions wraps the same fields in a "grammar" object. Text formats are + identical on both surfaces and pass through, as does anything unrecognized. + """ + if format_obj.get("type") == "grammar" and "grammar" not in format_obj: + return { + "type": "grammar", + "grammar": {k: format_obj[k] for k in ("definition", "syntax") if k in format_obj}, + } + return format_obj + + +def convert_custom_tool_format_to_responses_shape(format_obj: dict) -> dict: + """ + Inverse of convert_custom_tool_format_to_chat_shape: unwrap the Chat Completions + "grammar" object into the flat Responses API grammar shape. + """ + grammar = format_obj.get("grammar") + if format_obj.get("type") == "grammar" and isinstance(grammar, dict): + return {"type": "grammar", **{k: grammar[k] for k in ("definition", "syntax") if k in grammar}} + return format_obj + + def get_file_ids_from_messages(messages: List[AllMessageValues]) -> List[str]: """ Gets file ids from messages diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index f9b2cc79f73..7b64bcda7ce 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -29,10 +29,17 @@ _FLAT_FUNCTION_TOOL_KEYS = ("name", "description", "parameters", "strict") def _nest_flat_chat_tool(tool: object) -> object: + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + convert_custom_tool_format_to_chat_shape, + ) + if not isinstance(tool, dict) or "name" not in tool: return tool if tool.get("type") == "custom" and "custom" not in tool: - return {"type": "custom", "custom": {k: tool[k] for k in _FLAT_CUSTOM_TOOL_KEYS if k in tool}} + payload = {k: tool[k] for k in _FLAT_CUSTOM_TOOL_KEYS if k in tool} + if isinstance(payload.get("format"), dict): + payload = {**payload, "format": convert_custom_tool_format_to_chat_shape(payload["format"])} + return {"type": "custom", "custom": payload} if tool.get("type") == "function" and "function" not in tool: return {"type": "function", "function": {k: tool[k] for k in _FLAT_FUNCTION_TOOL_KEYS if k in tool}} return tool @@ -42,6 +49,16 @@ def _nest_flat_chat_tools(tools: list) -> list: return [_nest_flat_chat_tool(tool) for tool in tools] +def _nest_flat_chat_tool_choice(tool_choice: object) -> object: + if not isinstance(tool_choice, dict) or "name" not in tool_choice: + return tool_choice + if tool_choice.get("type") == "custom" and "custom" not in tool_choice: + return {"type": "custom", "custom": {"name": tool_choice["name"]}} + if tool_choice.get("type") == "function" and "function" not in tool_choice: + return {"type": "function", "function": {"name": tool_choice["name"]}} + return tool_choice + + @router.post( "/v1/responses", dependencies=[Depends(user_api_key_auth)], @@ -391,10 +408,17 @@ async def cursor_chat_completions( # Genuine chat completions body (Cursor sends these for models whose BYOK it # already fixed); delegate so behavior matches /chat/completions exactly tools = data.get("tools") + tool_choice = data.get("tool_choice") + normalized: dict = {} if isinstance(tools, list): nested_tools = _nest_flat_chat_tools(tools) if nested_tools != tools: - _safe_set_request_parsed_body(request=request, parsed_body={**data, "tools": nested_tools}) + normalized["tools"] = nested_tools + nested_tool_choice = _nest_flat_chat_tool_choice(tool_choice) + if nested_tool_choice != tool_choice: + normalized["tool_choice"] = nested_tool_choice + if normalized: + _safe_set_request_parsed_body(request=request, parsed_body={**data, **normalized}) return await chat_completion( request=request, fastapi_response=fastapi_response, diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 6b1ca3564e3..176274d236f 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -162,6 +162,12 @@ class LiteLLMCompletionResponsesConfig: if function_name: return {"type": "function", "function": {"name": function_name}} return "required" + elif tool_choice_type == "custom": + custom = tool_choice.get("custom") + custom_name = tool_choice.get("name") or (custom.get("name") if isinstance(custom, dict) else None) + if custom_name: + return {"type": "function", "function": {"name": custom_name}} + return "required" # Return as-is for unknown formats return tool_choice diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 767d1631649..64112ed43e8 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -2475,6 +2475,15 @@ def test_map_optional_params_tool_choice_chat_nested_to_responses_api(): {"type": "function", "name": "foo"}, ), ({"type": "required"}, {"type": "required"}), + ( + {"type": "custom", "custom": {"name": "ApplyPatch"}}, + {"type": "custom", "name": "ApplyPatch"}, + ), + ( + {"type": "custom", "name": "ApplyPatch"}, + {"type": "custom", "name": "ApplyPatch"}, + ), + ({"type": "custom"}, {"type": "custom"}), ], ) def test_normalize_tool_choice_for_responses_api(tool_choice, expected): @@ -3249,3 +3258,42 @@ def test_convert_tools_to_responses_format_flattens_custom_tool_without_optional handler = LiteLLMResponsesTransformationHandler() converted = handler._convert_tools_to_responses_format([{"type": "custom", "custom": {"name": "Minimal"}}]) assert converted[0] == {"type": "custom", "name": "Minimal"} + + +def test_convert_tools_to_responses_format_unwraps_nested_grammar_format(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + converted = handler._convert_tools_to_responses_format( + [ + { + "type": "custom", + "custom": { + "name": "ApplyPatch", + "format": { + "type": "grammar", + "grammar": {"definition": "start: patch", "syntax": "lark"}, + }, + }, + } + ] + ) + assert converted[0] == { + "type": "custom", + "name": "ApplyPatch", + "format": {"type": "grammar", "definition": "start: patch", "syntax": "lark"}, + } + + +def test_convert_tools_to_responses_format_text_format_passes_through(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + converted = handler._convert_tools_to_responses_format( + [{"type": "custom", "custom": {"name": "A", "format": {"type": "text"}}}] + ) + assert converted[0] == {"type": "custom", "name": "A", "format": {"type": "text"}} diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 1b1db634ed2..3728cc80323 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -721,3 +721,48 @@ class TestUnpackLegacyDefs: out = unpack_legacy_defs(schema) assert "components" not in out assert out["properties"]["r0"]["properties"]["p0"] == {"type": "string"} + + +class TestCustomToolFormatShapeConversion: + def test_flat_grammar_to_chat_shape(self): + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + convert_custom_tool_format_to_chat_shape, + ) + + assert convert_custom_tool_format_to_chat_shape( + {"type": "grammar", "definition": "start: patch", "syntax": "lark"} + ) == {"type": "grammar", "grammar": {"definition": "start: patch", "syntax": "lark"}} + + def test_nested_grammar_to_responses_shape(self): + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + convert_custom_tool_format_to_responses_shape, + ) + + assert convert_custom_tool_format_to_responses_shape( + {"type": "grammar", "grammar": {"definition": "start: patch", "syntax": "regex"}} + ) == {"type": "grammar", "definition": "start: patch", "syntax": "regex"} + + def test_both_directions_are_idempotent_and_pass_text_through(self): + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + convert_custom_tool_format_to_chat_shape, + convert_custom_tool_format_to_responses_shape, + ) + + flat = {"type": "grammar", "definition": "d", "syntax": "lark"} + nested = {"type": "grammar", "grammar": {"definition": "d", "syntax": "lark"}} + text = {"type": "text"} + assert convert_custom_tool_format_to_chat_shape(nested) == nested + assert convert_custom_tool_format_to_responses_shape(flat) == flat + assert convert_custom_tool_format_to_chat_shape(text) == text + assert convert_custom_tool_format_to_responses_shape(text) == text + assert convert_custom_tool_format_to_chat_shape(convert_custom_tool_format_to_responses_shape(nested)) == nested + + def test_unrecognized_formats_pass_through(self): + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + convert_custom_tool_format_to_chat_shape, + convert_custom_tool_format_to_responses_shape, + ) + + for weird in ({}, {"type": "grammar"}, {"type": "future_format", "x": 1}): + assert convert_custom_tool_format_to_chat_shape(dict(weird)) in (weird, {"type": "grammar", "grammar": {}}) + assert convert_custom_tool_format_to_responses_shape(dict(weird)) == weird diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index 4ef338b35e4..86fafa40811 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -981,9 +981,18 @@ class TestCursorMessagesArmToolNormalization: "type": "function", "function": {"name": "read_file", "parameters": {"type": "object"}}, }, - {"type": "custom", "name": "ApplyPatch", "description": "V4A patch"}, + { + "type": "custom", + "name": "ApplyPatch", + "description": "V4A patch", + "format": { + "type": "grammar", + "definition": "start: patch", + "syntax": "lark", + }, + }, ], - "tool_choice": "required", + "tool_choice": {"type": "custom", "name": "ApplyPatch"}, }, headers={"Authorization": "Bearer sk-1234"}, ) @@ -993,8 +1002,19 @@ class TestCursorMessagesArmToolNormalization: assert response.status_code == 200 assert seen["body"]["tools"] == [ {"type": "function", "function": {"name": "read_file", "parameters": {"type": "object"}}}, - {"type": "custom", "custom": {"name": "ApplyPatch", "description": "V4A patch"}}, + { + "type": "custom", + "custom": { + "name": "ApplyPatch", + "description": "V4A patch", + "format": { + "type": "grammar", + "grammar": {"definition": "start: patch", "syntax": "lark"}, + }, + }, + }, ] + assert seen["body"]["tool_choice"] == {"type": "custom", "custom": {"name": "ApplyPatch"}} assert seen["body"]["messages"] == [{"role": "user", "content": "use ApplyPatch"}] @pytest.mark.asyncio @@ -1030,3 +1050,73 @@ class TestCursorMessagesArmToolNormalization: assert response.status_code == 200 assert seen["body"]["tools"] == body["tools"] assert seen["body"]["messages"] == body["messages"] + + +class TestNestFlatChatToolGrammarFormat: + def test_flat_grammar_format_is_wrapped_for_chat(self): + from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tools + + result = _nest_flat_chat_tools( + [ + { + "type": "custom", + "name": "ApplyPatch", + "description": "V4A patch", + "format": {"type": "grammar", "definition": "start: patch", "syntax": "lark"}, + } + ] + ) + assert result == [ + { + "type": "custom", + "custom": { + "name": "ApplyPatch", + "description": "V4A patch", + "format": { + "type": "grammar", + "grammar": {"definition": "start: patch", "syntax": "lark"}, + }, + }, + } + ] + + def test_flat_text_format_is_copied_unchanged(self): + from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tools + + result = _nest_flat_chat_tools( + [{"type": "custom", "name": "A", "format": {"type": "text"}}] + ) + assert result == [{"type": "custom", "custom": {"name": "A", "format": {"type": "text"}}}] + + +class TestNestFlatChatToolChoice: + def test_flat_custom_tool_choice_is_nested(self): + from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tool_choice + + assert _nest_flat_chat_tool_choice({"type": "custom", "name": "ApplyPatch"}) == { + "type": "custom", + "custom": {"name": "ApplyPatch"}, + } + + def test_flat_function_tool_choice_is_nested(self): + from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tool_choice + + assert _nest_flat_chat_tool_choice({"type": "function", "name": "f"}) == { + "type": "function", + "function": {"name": "f"}, + } + + def test_non_flat_tool_choice_values_pass_through(self): + from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tool_choice + + for unchanged in ( + "auto", + "required", + None, + {"type": "custom", "custom": {"name": "x"}}, + {"type": "function", "function": {"name": "f"}}, + {"type": "auto"}, + {"name": "typeless"}, + 42, + ): + assert _nest_flat_chat_tool_choice(unchanged) == unchanged diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index d8e3f495ced..3f3f51f0d3f 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -959,6 +959,27 @@ class TestToolChoiceTransformation: ) assert result == {"type": "function", "function": {"name": "get_weather"}} + def test_transform_tool_choice_custom_follows_function_downgrade(self): + """ + This bridge downgrades custom tools to function tools + (convert_custom_tool_to_function_tool), so a custom tool_choice must become a + function tool_choice naming the same tool or it references a tool type absent + from the converted request. + """ + flat = LiteLLMCompletionResponsesConfig._transform_tool_choice( + {"type": "custom", "name": "ApplyPatch"} + ) + assert flat == {"type": "function", "function": {"name": "ApplyPatch"}} + + nested = LiteLLMCompletionResponsesConfig._transform_tool_choice( + {"type": "custom", "custom": {"name": "ApplyPatch"}} + ) + assert nested == {"type": "function", "function": {"name": "ApplyPatch"}} + + def test_transform_tool_choice_custom_without_name_falls_back_to_required(self): + result = LiteLLMCompletionResponsesConfig._transform_tool_choice({"type": "custom"}) + assert result == "required" + def test_transform_tool_choice_function_without_name_falls_back_to_required(self): """A function-type dict with no name still falls back to required""" result = LiteLLMCompletionResponsesConfig._transform_tool_choice( From ebe48d67de3e10f42a46bf19b1700331b72c1e14 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Tue, 21 Jul 2026 14:28:03 -0700 Subject: [PATCH 017/124] fix(proxy): normalize each tool shape level independently on the Cursor messages arm Live Cursor Ask-mode captures show the shape dialects mix PER LEVEL: the tool envelope arrives chat-nested while the grammar format inside it is still Responses-flat, so a normalizer that pattern-matches whole-tool templates misses every hybrid. The cursor arm now normalizes the envelope level and the format level independently and idempotently, making it total over the envelope x format matrix; a parametrized 8-cell test pins every combination. The reference BYOK bridge was checked and forwards chat bodies verbatim, so there is no prior art for these hybrids --- .../proxy/response_api_endpoints/endpoints.py | 27 +++++--- .../response_api_endpoints/test_endpoints.py | 69 ++++++++++++++----- ui/litellm-dashboard/src/lib/http/schema.d.ts | 10 ++- 3 files changed, 76 insertions(+), 30 deletions(-) diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 7b64bcda7ce..a980f85d406 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -33,14 +33,21 @@ def _nest_flat_chat_tool(tool: object) -> object: convert_custom_tool_format_to_chat_shape, ) - if not isinstance(tool, dict) or "name" not in tool: + if not isinstance(tool, dict): return tool - if tool.get("type") == "custom" and "custom" not in tool: - payload = {k: tool[k] for k in _FLAT_CUSTOM_TOOL_KEYS if k in tool} + if tool.get("type") == "custom": + if isinstance(tool.get("custom"), dict): + envelope = tool + payload = tool["custom"] + elif "name" in tool: + envelope = {"type": "custom"} + payload = {k: tool[k] for k in _FLAT_CUSTOM_TOOL_KEYS if k in tool} + else: + return tool if isinstance(payload.get("format"), dict): payload = {**payload, "format": convert_custom_tool_format_to_chat_shape(payload["format"])} - return {"type": "custom", "custom": payload} - if tool.get("type") == "function" and "function" not in tool: + return {**envelope, "custom": payload} + if tool.get("type") == "function" and "function" not in tool and "name" in tool: return {"type": "function", "function": {k: tool[k] for k in _FLAT_FUNCTION_TOOL_KEYS if k in tool}} return tool @@ -364,9 +371,13 @@ async def cursor_chat_completions( custom tools) to the chat/completions path while expecting chat completions responses; those are routed through the Responses API pipeline and converted back. Genuine chat completions bodies (`messages` present) are routed through the standard chat completions - pipeline, after nesting any flat Responses-style tool defs Cursor mixes into the chat - `tools` array (e.g. `{"type": "custom", "name": "ApplyPatch", ...}`) into the chat - completions shape OpenAI requires (`{"type": "custom", "custom": {...}}`). + pipeline, after normalizing each level of the `tools` array and `tool_choice` to the chat + completions shapes OpenAI requires. Cursor mixes Responses API shapes into chat bodies + per level, independently: a flat tool def (`{"type": "custom", "name": "ApplyPatch", ...}`) + gets nested under `custom`, and a flat grammar format + (`{"type": "grammar", "definition", "syntax"}`) gets wrapped as + `{"type": "grammar", "grammar": {...}}` wherever it appears, including inside tool defs + Cursor already sent pre-nested. ```bash curl -X POST http://localhost:4000/cursor/chat/completions \ diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index 86fafa40811..b8699d6ef8c 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -1052,26 +1052,56 @@ class TestCursorMessagesArmToolNormalization: assert seen["body"]["messages"] == body["messages"] -class TestNestFlatChatToolGrammarFormat: - def test_flat_grammar_format_is_wrapped_for_chat(self): +class TestNestFlatChatToolShapeMatrix: + """ + Cursor mixes Responses API shapes into chat bodies PER LEVEL, independently + (live-captured: a pre-nested custom envelope carrying a flat grammar format). + Every cell of envelope x format must land on the canonical chat shape. + """ + + FLAT_GRAMMAR = {"type": "grammar", "definition": "start: patch", "syntax": "lark"} + NESTED_GRAMMAR = {"type": "grammar", "grammar": {"definition": "start: patch", "syntax": "lark"}} + TEXT = {"type": "text"} + + @pytest.mark.parametrize("envelope", ["flat", "nested"]) + @pytest.mark.parametrize("format_shape", ["absent", "text", "flat_grammar", "nested_grammar"]) + def test_every_envelope_and_format_combination_lands_canonical(self, envelope, format_shape): from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tools - result = _nest_flat_chat_tools( - [ - { - "type": "custom", - "name": "ApplyPatch", - "description": "V4A patch", - "format": {"type": "grammar", "definition": "start: patch", "syntax": "lark"}, - } - ] - ) - assert result == [ + format_value = { + "absent": None, + "text": self.TEXT, + "flat_grammar": self.FLAT_GRAMMAR, + "nested_grammar": self.NESTED_GRAMMAR, + }[format_shape] + payload = {"name": "ApplyPatch", "description": "V4A patch"} + if format_value is not None: + payload["format"] = format_value + tool = {"type": "custom", "custom": payload} if envelope == "nested" else {"type": "custom", **payload} + + canonical_payload = {"name": "ApplyPatch", "description": "V4A patch"} + if format_shape in ("flat_grammar", "nested_grammar"): + canonical_payload["format"] = self.NESTED_GRAMMAR + elif format_shape == "text": + canonical_payload["format"] = self.TEXT + + assert _nest_flat_chat_tools([tool]) == [{"type": "custom", "custom": canonical_payload}] + + def test_nested_envelope_with_flat_grammar_matches_live_cursor_capture(self): + from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tools + + cursor_tool = { + "type": "custom", + "custom": { + "name": "ApplyPatch", + "format": {"type": "grammar", "definition": "start: patch", "syntax": "lark"}, + }, + } + assert _nest_flat_chat_tools([cursor_tool]) == [ { "type": "custom", "custom": { "name": "ApplyPatch", - "description": "V4A patch", "format": { "type": "grammar", "grammar": {"definition": "start: patch", "syntax": "lark"}, @@ -1080,13 +1110,14 @@ class TestNestFlatChatToolGrammarFormat: } ] - def test_flat_text_format_is_copied_unchanged(self): + def test_canonical_nested_tool_is_returned_equal(self): from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tools - result = _nest_flat_chat_tools( - [{"type": "custom", "name": "A", "format": {"type": "text"}}] - ) - assert result == [{"type": "custom", "custom": {"name": "A", "format": {"type": "text"}}}] + canonical = { + "type": "custom", + "custom": {"name": "A", "format": {"type": "grammar", "grammar": {"definition": "d", "syntax": "lark"}}}, + } + assert _nest_flat_chat_tools([canonical]) == [canonical] class TestNestFlatChatToolChoice: diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index bf3cc75c796..94f633c676e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -2628,9 +2628,13 @@ export interface paths { * custom tools) to the chat/completions path while expecting chat completions responses; * those are routed through the Responses API pipeline and converted back. Genuine chat * completions bodies (`messages` present) are routed through the standard chat completions - * pipeline, after nesting any flat Responses-style tool defs Cursor mixes into the chat - * `tools` array (e.g. `{"type": "custom", "name": "ApplyPatch", ...}`) into the chat - * completions shape OpenAI requires (`{"type": "custom", "custom": {...}}`). + * pipeline, after normalizing each level of the `tools` array and `tool_choice` to the chat + * completions shapes OpenAI requires. Cursor mixes Responses API shapes into chat bodies + * per level, independently: a flat tool def (`{"type": "custom", "name": "ApplyPatch", ...}`) + * gets nested under `custom`, and a flat grammar format + * (`{"type": "grammar", "definition", "syntax"}`) gets wrapped as + * `{"type": "grammar", "grammar": {...}}` wherever it appears, including inside tool defs + * Cursor already sent pre-nested. * * ```bash * curl -X POST http://localhost:4000/cursor/chat/completions -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{ From 6d102ea5599a1ecf9c8fb823a88a18fc8131cd0c Mon Sep 17 00:00:00 2001 From: Tin Date: Tue, 21 Jul 2026 16:11:35 -0700 Subject: [PATCH 018/124] fix(litellm): bridge gpt-5.4+ chat requests with tools when reasoning defaults on OpenAI enables reasoning by default for gpt-5.4+ (unset reasoning_effort means medium server-side) and Chat Completions rejects function tools whenever reasoning is on, so a tools request without an explicit reasoning_effort 400d instead of auto-bridging to the Responses API; the bridge heuristic now treats unset effort as reasoning-active and honors the documented escape hatch by keeping explicit "none" on chat completions. The cursor input arm also gains the mirror of the messages-arm normalization: chat-nested tool envelopes, grammar formats, and object tool_choice flatten to the Responses dialect before dispatch --- litellm/main.py | 13 +- .../proxy/response_api_endpoints/endpoints.py | 49 +++++++ .../response_api_endpoints/test_endpoints.py | 136 ++++++++++++++++++ tests/test_litellm/test_main.py | 72 +++++++++- 4 files changed, 262 insertions(+), 8 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index acdec7385da..43d021ebe8b 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1022,7 +1022,12 @@ def responses_api_bridge_check( # ``reasoningSummary`` in ``extra_body``) must be bridged; Chat Completions rejects # those keys. # - # - gpt-5.4+: tools + reasoning_effort (original) or any reasoning-summary alias. + # - gpt-5.4+: function tools with reasoning active must be bridged. OpenAI enables + # reasoning by default for these models (unset reasoning_effort means medium + # server-side), and Chat Completions rejects tools whenever reasoning is on + # ("Function tools with reasoning_effort are not supported ... use /v1/responses + # or set reasoning_effort to 'none'"), so only an explicit ``"none"`` keeps the + # request chat-servable. # - Older GPT-5 names (e.g. ``gpt-5``, ``gpt-5.1``): bridge only when a reasoning # summary alias is present with ``reasoning_effort`` (tools alone stay on chat). if ( @@ -1030,8 +1035,10 @@ def responses_api_bridge_check( and model_info.get("mode") != "responses" and OpenAIGPT5Config.is_model_gpt_5_model(model) and not OpenAIGPT5Config.is_model_gpt_5_search_model(model) - and reasoning_effort is not None - and (reasoning_summary is not None or (OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) and tools)) + and ( + (reasoning_effort is not None and reasoning_summary is not None) + or (OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) and tools and reasoning_effort != "none") + ) ): model_info["mode"] = "responses" model = model.replace("responses/", "") diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index a980f85d406..8a678601284 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -66,6 +66,47 @@ def _nest_flat_chat_tool_choice(tool_choice: object) -> object: return tool_choice +def _flatten_chat_tool_for_responses(tool: object) -> object: + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + convert_custom_tool_format_to_responses_shape, + ) + + if not isinstance(tool, dict): + return tool + if tool.get("type") == "custom": + if isinstance(tool.get("custom"), dict): + payload = {k: tool["custom"][k] for k in _FLAT_CUSTOM_TOOL_KEYS if k in tool["custom"]} + elif "name" in tool: + payload = {k: tool[k] for k in _FLAT_CUSTOM_TOOL_KEYS if k in tool} + else: + return tool + if isinstance(payload.get("format"), dict): + payload = {**payload, "format": convert_custom_tool_format_to_responses_shape(payload["format"])} + return {"type": "custom", **payload} + if tool.get("type") == "function" and isinstance(tool.get("function"), dict): + return { + "type": "function", + **{k: tool["function"][k] for k in _FLAT_FUNCTION_TOOL_KEYS if k in tool["function"]}, + } + return tool + + +def _flatten_chat_tools_for_responses(tools: list) -> list: + return [_flatten_chat_tool_for_responses(tool) for tool in tools] + + +def _flatten_chat_tool_choice_for_responses(tool_choice: object) -> object: + if not isinstance(tool_choice, dict): + return tool_choice + choice_type = tool_choice.get("type") + if choice_type not in ("custom", "function"): + return tool_choice + nested = tool_choice.get(choice_type) + if isinstance(nested, dict) and isinstance(nested.get("name"), str): + return {"type": choice_type, "name": nested["name"]} + return tool_choice + + @router.post( "/v1/responses", dependencies=[Depends(user_api_key_auth)], @@ -444,6 +485,14 @@ async def cursor_chat_completions( # cache's key snapshot so later readers get an empty body data = {key: value for key, value in data.items() if key != "stream_options"} + tools = data.get("tools") + if isinstance(tools, list): + data = {**data, "tools": _flatten_chat_tools_for_responses(tools)} + tool_choice = data.get("tool_choice") + flattened_tool_choice = _flatten_chat_tool_choice_for_responses(tool_choice) + if flattened_tool_choice != tool_choice: + data = {**data, "tool_choice": flattened_tool_choice} + processor = ProxyBaseLLMRequestProcessing(data=data) def cursor_data_generator(response, user_api_key_dict, request_data, request=None): diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index b8699d6ef8c..04f027ac4bb 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -1151,3 +1151,139 @@ class TestNestFlatChatToolChoice: 42, ): assert _nest_flat_chat_tool_choice(unchanged) == unchanged + + +class TestFlattenChatToolsForResponsesInputArm: + """ + Mirror of TestNestFlatChatToolShapeMatrix for the input arm: chat-nested shapes in a + Responses-shaped body must flatten to the Responses dialect, per level, idempotently. + """ + + FLAT_GRAMMAR = {"type": "grammar", "definition": "start: patch", "syntax": "lark"} + NESTED_GRAMMAR = {"type": "grammar", "grammar": {"definition": "start: patch", "syntax": "lark"}} + + @pytest.mark.parametrize("envelope", ["flat", "nested"]) + @pytest.mark.parametrize("format_shape", ["absent", "text", "flat_grammar", "nested_grammar"]) + def test_every_envelope_and_format_combination_lands_flat(self, envelope, format_shape): + from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tools_for_responses + + format_value = { + "absent": None, + "text": {"type": "text"}, + "flat_grammar": self.FLAT_GRAMMAR, + "nested_grammar": self.NESTED_GRAMMAR, + }[format_shape] + payload = {"name": "ApplyPatch", "description": "V4A patch"} + if format_value is not None: + payload["format"] = format_value + tool = {"type": "custom", "custom": payload} if envelope == "nested" else {"type": "custom", **payload} + + canonical = {"type": "custom", "name": "ApplyPatch", "description": "V4A patch"} + if format_shape in ("flat_grammar", "nested_grammar"): + canonical["format"] = self.FLAT_GRAMMAR + elif format_shape == "text": + canonical["format"] = {"type": "text"} + + assert _flatten_chat_tools_for_responses([tool]) == [canonical] + + def test_nested_function_tool_is_flattened_and_flat_passes_through(self): + from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tools_for_responses + + nested = {"type": "function", "function": {"name": "read_file", "parameters": {"type": "object"}}} + flat = {"type": "function", "name": "read_file", "parameters": {"type": "object"}} + assert _flatten_chat_tools_for_responses([nested]) == [flat] + assert _flatten_chat_tools_for_responses([flat]) == [flat] + + def test_unrecognized_entries_pass_through(self): + from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tools_for_responses + + entries = [{"type": "web_search"}, {"type": "custom"}, "junk", None, {}] + assert _flatten_chat_tools_for_responses(entries) == entries + + +class TestFlattenChatToolChoiceForResponsesInputArm: + def test_nested_custom_and_function_tool_choice_flatten(self): + from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tool_choice_for_responses + + assert _flatten_chat_tool_choice_for_responses({"type": "custom", "custom": {"name": "ApplyPatch"}}) == { + "type": "custom", + "name": "ApplyPatch", + } + assert _flatten_chat_tool_choice_for_responses({"type": "function", "function": {"name": "f"}}) == { + "type": "function", + "name": "f", + } + + def test_flat_and_string_tool_choice_pass_through(self): + from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tool_choice_for_responses + + for unchanged in ("auto", "required", None, {"type": "custom", "name": "x"}, {"type": "auto"}, 42): + assert _flatten_chat_tool_choice_for_responses(unchanged) == unchanged + + +class TestCursorInputArmFlattening: + @pytest.mark.asyncio + async def test_nested_chat_shapes_in_input_body_reach_aresponses_flattened(self): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from openai.types.responses import ResponseOutputMessage, ResponseOutputText + + from litellm.types.llms.openai import ResponsesAPIResponse + + mock_response = ResponsesAPIResponse( + id="resp_flat123", + created_at=1234567890, + model="gpt-5.6", + object="response", + output=[ + ResponseOutputMessage( + id="msg_flat123", + type="message", + role="assistant", + status="completed", + content=[ResponseOutputText(type="output_text", text="ok", annotations=[])], + ) + ], + ) + + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-1234") + try: + with patch("litellm.proxy.proxy_server.llm_router") as mock_router: + mock_router.aresponses = AsyncMock(return_value=mock_response) + client = TestClient(app) + response = client.post( + "/cursor/chat/completions", + json={ + "model": "gpt-5.6", + "input": [{"role": "user", "content": "use ApplyPatch"}], + "tools": [ + { + "type": "custom", + "custom": { + "name": "ApplyPatch", + "format": { + "type": "grammar", + "grammar": {"definition": "start: patch", "syntax": "lark"}, + }, + }, + }, + {"type": "function", "name": "read_file", "parameters": {"type": "object"}}, + ], + "tool_choice": {"type": "custom", "custom": {"name": "ApplyPatch"}}, + }, + headers={"Authorization": "Bearer sk-1234"}, + ) + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 200 + call_kwargs = mock_router.aresponses.call_args.kwargs + assert call_kwargs["tools"] == [ + { + "type": "custom", + "name": "ApplyPatch", + "format": {"type": "grammar", "definition": "start: patch", "syntax": "lark"}, + }, + {"type": "function", "name": "read_file", "parameters": {"type": "object"}}, + ] + assert call_kwargs["tool_choice"] == {"type": "custom", "name": "ApplyPatch"} diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 4611aafa3c1..b4f94177f7d 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -810,8 +810,12 @@ def test_responses_api_bridge_check_azure_gpt_5_4_tools_plus_reasoning_routes_to assert model_info.get("mode") == "responses" -def test_responses_api_bridge_check_azure_gpt_5_4_tools_without_reasoning_stays_chat(): - """Azure gpt-5.4 with tools only should not be force-routed to Responses API.""" +def test_responses_api_bridge_check_azure_gpt_5_4_tools_with_default_reasoning_routes_to_responses(): + """ + Azure gpt-5.4 with tools and UNSET reasoning_effort must bridge: OpenAI enables + reasoning by default for gpt-5.4+, and Chat Completions rejects function tools + whenever reasoning is on. + """ from litellm.main import responses_api_bridge_check with patch("litellm.main._get_model_info_helper") as mock_get_model_info: @@ -824,11 +828,15 @@ def test_responses_api_bridge_check_azure_gpt_5_4_tools_without_reasoning_stays_ ) assert model == "gpt-5.4" - assert model_info.get("mode") != "responses" + assert model_info.get("mode") == "responses" -def test_responses_api_bridge_check_gpt_5_4_tools_without_reasoning_stays_chat(): - """gpt-5.4 with tools only should not be force-routed to Responses API.""" +def test_responses_api_bridge_check_gpt_5_4_tools_with_default_reasoning_routes_to_responses(): + """ + gpt-5.4 with tools and UNSET reasoning_effort must bridge: OpenAI enables reasoning + by default for gpt-5.4+, and Chat Completions rejects function tools whenever + reasoning is on ("use /v1/responses or set reasoning_effort to 'none'"). + """ from litellm.main import responses_api_bridge_check with patch("litellm.main._get_model_info_helper") as mock_get_model_info: @@ -841,6 +849,60 @@ def test_responses_api_bridge_check_gpt_5_4_tools_without_reasoning_stays_chat() ) assert model == "gpt-5.4" + assert model_info.get("mode") == "responses" + + +def test_responses_api_bridge_check_gpt_5_4_tools_with_reasoning_none_stays_chat(): + """ + Explicit reasoning_effort "none" is OpenAI's documented escape hatch that keeps + function tools servable on Chat Completions; the bridge must not fire. + """ + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5.4", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort="none", + ) + + assert model == "gpt-5.4" + assert model_info.get("mode") != "responses" + + +def test_responses_api_bridge_check_reasoning_none_with_summary_still_routes_to_responses(): + """A reasoning summary is Responses-only regardless of effort value.""" + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5.4", + custom_llm_provider="openai", + reasoning_effort="none", + reasoning_summary="detailed", + ) + + assert model == "gpt-5.4" + assert model_info.get("mode") == "responses" + + +def test_responses_api_bridge_check_older_gpt_5_tools_without_reasoning_stays_chat(): + """Pre-5.4 GPT-5 names keep the old boundary: tools alone never bridge.""" + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5.1", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort=None, + ) + + assert model == "gpt-5.1" assert model_info.get("mode") != "responses" From 7276f44b1db7ccab847549acd298230fa74ad243 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Tue, 21 Jul 2026 18:10:16 -0700 Subject: [PATCH 019/124] fix(litellm): gate the gpt-5.4+ responses bridge on function tools specifically OpenAI's chat completions rejection applies to function tools only; custom (grammar) tools are served natively with reasoning on, live-proven by a 200 on a custom-only gpt-5.6 chat request. Gating on any truthy tools needlessly bridged custom-only requests, and the bridge maps custom tool calls back function-shaped, so the native chat custom tool_call surface added earlier in this PR was bypassed exactly where chat serves it natively. The gate now checks for a function-type tool in either the nested chat or flat Responses def shape; the same coarseness existed on the explicit-effort arm before this PR and is fixed by the shared leg --- litellm/main.py | 20 +++++++---- tests/test_litellm/test_main.py | 59 +++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 6 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 43d021ebe8b..8a9e0e37ace 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1022,14 +1022,20 @@ def responses_api_bridge_check( # ``reasoningSummary`` in ``extra_body``) must be bridged; Chat Completions rejects # those keys. # - # - gpt-5.4+: function tools with reasoning active must be bridged. OpenAI enables + # - gpt-5.4+: FUNCTION tools with reasoning active must be bridged. OpenAI enables # reasoning by default for these models (unset reasoning_effort means medium - # server-side), and Chat Completions rejects tools whenever reasoning is on - # ("Function tools with reasoning_effort are not supported ... use /v1/responses - # or set reasoning_effort to 'none'"), so only an explicit ``"none"`` keeps the - # request chat-servable. + # server-side), and Chat Completions rejects function tools whenever reasoning is + # on ("Function tools with reasoning_effort are not supported ... use + # /v1/responses or set reasoning_effort to 'none'"), so only an explicit + # ``"none"`` keeps the request chat-servable. Custom (grammar) tools are served + # natively by Chat Completions with reasoning on, so custom-only requests stay on + # chat and keep their native custom tool_call response shape. # - Older GPT-5 names (e.g. ``gpt-5``, ``gpt-5.1``): bridge only when a reasoning # summary alias is present with ``reasoning_effort`` (tools alone stay on chat). + has_function_tool = any( + (tool.get("type") == "function" if isinstance(tool, dict) else getattr(tool, "type", None) == "function") + for tool in (tools or []) + ) if ( custom_llm_provider in ("openai", "azure") and model_info.get("mode") != "responses" @@ -1037,7 +1043,9 @@ def responses_api_bridge_check( and not OpenAIGPT5Config.is_model_gpt_5_search_model(model) and ( (reasoning_effort is not None and reasoning_summary is not None) - or (OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) and tools and reasoning_effort != "none") + or ( + OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) and has_function_tool and reasoning_effort != "none" + ) ) ): model_info["mode"] = "responses" diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index b4f94177f7d..4a0ec04bed1 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -889,6 +889,65 @@ def test_responses_api_bridge_check_reasoning_none_with_summary_still_routes_to_ assert model_info.get("mode") == "responses" +def test_responses_api_bridge_check_gpt_5_4_custom_tools_only_stays_chat(): + """ + Chat Completions serves custom (grammar) tools natively with reasoning on; only + FUNCTION tools trigger the OpenAI rejection. Custom-only requests must stay on chat + so responses keep the native custom tool_call shape instead of the bridge's + function-shaped mapping. + """ + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5.6", + custom_llm_provider="openai", + tools=[{"type": "custom", "custom": {"name": "ApplyPatch", "description": "V4A patch"}}], + reasoning_effort=None, + ) + + assert model == "gpt-5.6" + assert model_info.get("mode") != "responses" + + +def test_responses_api_bridge_check_gpt_5_4_mixed_function_and_custom_tools_routes_to_responses(): + """One function tool in the mix is enough to make chat unservable with reasoning on.""" + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5.6", + custom_llm_provider="openai", + tools=[ + {"type": "custom", "custom": {"name": "ApplyPatch"}}, + {"type": "function", "function": {"name": "shell"}}, + ], + reasoning_effort=None, + ) + + assert model == "gpt-5.6" + assert model_info.get("mode") == "responses" + + +def test_responses_api_bridge_check_gpt_5_4_flat_function_tool_routes_to_responses(): + """Responses-style flat function tool defs still count as function tools.""" + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5.6", + custom_llm_provider="openai", + tools=[{"type": "function", "name": "shell", "parameters": {"type": "object"}}], + reasoning_effort=None, + ) + + assert model == "gpt-5.6" + assert model_info.get("mode") == "responses" + + def test_responses_api_bridge_check_older_gpt_5_tools_without_reasoning_stays_chat(): """Pre-5.4 GPT-5 names keep the old boundary: tools alone never bridge.""" from litellm.main import responses_api_bridge_check From bbba450301344ee8b4f4981a32dfe7fc63d10f9b Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Tue, 21 Jul 2026 20:13:02 -0700 Subject: [PATCH 020/124] fix(litellm): honor dict-form reasoning_effort in the bridge escape hatch and serialize custom tool calls in helicone and lunary logs The bridge gate compared reasoning_effort against the string "none", so litellm's dict form ({"effort": "none"}) wrongly bridged; the gate now reads the effort value from either form and treats a summary inside the dict as Responses-only regardless of effort. Helicone and lunary previously skipped custom tool calls entirely; both now serialize them (helicone as a tool_use block from the custom payload, lunary with the custom name and input in its function fields, keeping type custom), with new mapped tests for both integrations --- litellm/integrations/helicone.py | 29 +++++++---- litellm/integrations/lunary.py | 16 +++++- litellm/main.py | 8 +-- .../integrations/test_helicone.py | 51 +++++++++++++++++++ .../test_litellm/integrations/test_lunary.py | 40 +++++++++++++++ tests/test_litellm/test_main.py | 50 ++++++++++++++++++ 6 files changed, 180 insertions(+), 14 deletions(-) create mode 100644 tests/test_litellm/integrations/test_helicone.py create mode 100644 tests/test_litellm/integrations/test_lunary.py diff --git a/litellm/integrations/helicone.py b/litellm/integrations/helicone.py index 4c7a606c16f..c9346f7e6cf 100644 --- a/litellm/integrations/helicone.py +++ b/litellm/integrations/helicone.py @@ -60,16 +60,25 @@ class HeliconeLogger: if "tool_calls" in message and message["tool_calls"]: for tool_call in message["tool_calls"]: function = tool_call.get("function") - if not function: - continue - content.append( - { - "type": "tool_use", - "id": tool_call["id"], - "name": function["name"], - "input": function["arguments"], - } - ) + custom = tool_call.get("custom") + if function: + content.append( + { + "type": "tool_use", + "id": tool_call["id"], + "name": function["name"], + "input": function["arguments"], + } + ) + elif custom: + content.append( + { + "type": "tool_use", + "id": tool_call["id"], + "name": custom["name"], + "input": custom["input"], + } + ) elif "content" in message and message["content"]: content = [{"type": "text", "text": message["content"]}] diff --git a/litellm/integrations/lunary.py b/litellm/integrations/lunary.py index 448580f0b2d..94cb5bab8fe 100644 --- a/litellm/integrations/lunary.py +++ b/litellm/integrations/lunary.py @@ -20,6 +20,16 @@ def parse_tool_calls(tool_calls): return None def clean_tool_call(tool_call): + custom = getattr(tool_call, "custom", None) + if custom is not None: + return { + "type": tool_call.type, + "id": tool_call.id, + "function": { + "name": custom.name, + "arguments": custom.input, + }, + } serialized = { "type": tool_call.type, "id": tool_call.id, @@ -31,7 +41,11 @@ def parse_tool_calls(tool_calls): return serialized - return [clean_tool_call(tool_call) for tool_call in tool_calls if getattr(tool_call, "function", None) is not None] + return [ + clean_tool_call(tool_call) + for tool_call in tool_calls + if getattr(tool_call, "function", None) is not None or getattr(tool_call, "custom", None) is not None + ] def parse_messages(input): diff --git a/litellm/main.py b/litellm/main.py index 8a9e0e37ace..b6c6b44a6f7 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1036,6 +1036,10 @@ def responses_api_bridge_check( (tool.get("type") == "function" if isinstance(tool, dict) else getattr(tool, "type", None) == "function") for tool in (tools or []) ) + if isinstance(reasoning_effort, dict): + reasoning_active = reasoning_effort.get("effort") != "none" or reasoning_effort.get("summary") is not None + else: + reasoning_active = reasoning_effort != "none" if ( custom_llm_provider in ("openai", "azure") and model_info.get("mode") != "responses" @@ -1043,9 +1047,7 @@ def responses_api_bridge_check( and not OpenAIGPT5Config.is_model_gpt_5_search_model(model) and ( (reasoning_effort is not None and reasoning_summary is not None) - or ( - OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) and has_function_tool and reasoning_effort != "none" - ) + or (OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) and has_function_tool and reasoning_active) ) ): model_info["mode"] = "responses" diff --git a/tests/test_litellm/integrations/test_helicone.py b/tests/test_litellm/integrations/test_helicone.py new file mode 100644 index 00000000000..eeef6fadbe5 --- /dev/null +++ b/tests/test_litellm/integrations/test_helicone.py @@ -0,0 +1,51 @@ +import os +import sys +import types + +sys.path.insert(0, os.path.abspath("../../..")) + +from litellm.integrations.helicone import HeliconeLogger + + +def _claude_mapping(messages, response_obj): + logger = HeliconeLogger.__new__(HeliconeLogger) + return logger.claude_mapping(model="gpt-5.6", messages=messages, response_obj=response_obj) + + +def test_claude_mapping_serializes_custom_tool_calls(monkeypatch): + try: + import anthropic # noqa: F401 + except ImportError: + stub = types.ModuleType("anthropic") + stub.HUMAN_PROMPT = "\n\nHuman:" + stub.AI_PROMPT = "\n\nAssistant:" + monkeypatch.setitem(sys.modules, "anthropic", stub) + response_obj = { + "id": "chatcmpl-1", + "choices": [ + { + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_c", + "type": "custom", + "custom": {"name": "ApplyPatch", "input": "*** Begin Patch"}, + }, + { + "id": "call_f", + "type": "function", + "function": {"name": "read_file", "arguments": '{"path": "a.py"}'}, + }, + ], + }, + "finish_reason": "tool_calls", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 2}, + } + mapped = _claude_mapping([{"role": "user", "content": "hi"}], response_obj) + tool_use_blocks = [b for b in mapped["content"] if b["type"] == "tool_use"] + assert {"type": "tool_use", "id": "call_c", "name": "ApplyPatch", "input": "*** Begin Patch"} in tool_use_blocks + assert {"type": "tool_use", "id": "call_f", "name": "read_file", "input": '{"path": "a.py"}'} in tool_use_blocks diff --git a/tests/test_litellm/integrations/test_lunary.py b/tests/test_litellm/integrations/test_lunary.py new file mode 100644 index 00000000000..0a1ec100594 --- /dev/null +++ b/tests/test_litellm/integrations/test_lunary.py @@ -0,0 +1,40 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath("../../..")) + +from litellm.integrations.lunary import parse_tool_calls +from litellm.types.utils import ( + ChatCompletionMessageCustomToolCall, + ChatCompletionMessageToolCall, + Function, +) + + +def test_parse_tool_calls_serializes_custom_tool_calls(): + custom_call = ChatCompletionMessageCustomToolCall( + id="call_c", + custom={"name": "ApplyPatch", "input": "*** Begin Patch"}, + ) + function_call = ChatCompletionMessageToolCall( + id="call_f", + type="function", + function=Function(name="read_file", arguments='{"path": "a.py"}'), + ) + parsed = parse_tool_calls([custom_call, function_call]) + assert parsed == [ + { + "type": "custom", + "id": "call_c", + "function": {"name": "ApplyPatch", "arguments": "*** Begin Patch"}, + }, + { + "type": "function", + "id": "call_f", + "function": {"name": "read_file", "arguments": '{"path": "a.py"}'}, + }, + ] + + +def test_parse_tool_calls_none_passthrough(): + assert parse_tool_calls(None) is None diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 4a0ec04bed1..60558760f8e 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -948,6 +948,56 @@ def test_responses_api_bridge_check_gpt_5_4_flat_function_tool_routes_to_respons assert model_info.get("mode") == "responses" +def test_responses_api_bridge_check_dict_effort_none_stays_chat(): + """The escape hatch must honor litellm's dict form: {"effort": "none"} means reasoning off.""" + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5.6", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort={"effort": "none"}, + ) + + assert model == "gpt-5.6" + assert model_info.get("mode") != "responses" + + +def test_responses_api_bridge_check_dict_effort_active_routes_to_responses(): + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5.6", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort={"effort": "low"}, + ) + + assert model == "gpt-5.6" + assert model_info.get("mode") == "responses" + + +def test_responses_api_bridge_check_dict_effort_none_with_summary_routes_to_responses(): + """A summary inside the dict form is Responses-only even when effort is none.""" + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5.6", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort={"effort": "none", "summary": "concise"}, + ) + + assert model == "gpt-5.6" + assert model_info.get("mode") == "responses" + + def test_responses_api_bridge_check_older_gpt_5_tools_without_reasoning_stays_chat(): """Pre-5.4 GPT-5 names keep the old boundary: tools alone never bridge.""" from litellm.main import responses_api_bridge_check From e9d16bc35cb7e1a754114a1977cdcab441c9f39b Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Tue, 21 Jul 2026 20:51:21 -0700 Subject: [PATCH 021/124] fix(litellm): make the responses bridge and cursor routing total over the surfaces they now serve Three gaps from the bridge becoming a mainstream path for chat traffic. The chat to responses message converter only mapped function tool_calls, so history carrying the native custom tool calls this PR introduced raised "tool call not supported" on follow-up turns; custom entries now map to custom_tool_call items and their results to custom_tool_call_output. The stream translator returned an empty delta for output_item.done on tool items, which left the responses guardrail handler's tool extraction permanently empty (dead on staging too, where the built chunk was discarded); stateless callers now receive the complete tool call while per-stream callers keep the suppressed delta that prevents client-side duplication. Cursor routing keyed on the presence of a messages key, so a null or empty stub next to a real agent-mode input array picked the chat arm; routing now keys on messages content --- .../transformation.py | 56 ++++++++-- .../proxy/response_api_endpoints/endpoints.py | 13 ++- ...responses_transformation_transformation.py | 103 ++++++++++++++++++ .../response_api_endpoints/test_endpoints.py | 59 ++++++++++ 4 files changed, 222 insertions(+), 9 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index e1842a62d56..768bf6c3e66 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -221,6 +221,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) -> Tuple[List[Any], Optional[str]]: input_items: List[Any] = [] instructions: Optional[str] = None + custom_tool_call_ids: set = set() for msg in messages: role = msg.get("role") @@ -266,18 +267,28 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): else: # Fallback: convert unexpected types to input_text tool_output = [{"type": "input_text", "text": str(content)}] - input_items.append( - { - "type": "function_call_output", - "call_id": tool_call_id, - "output": tool_output, - } - ) + if tool_call_id in custom_tool_call_ids: + input_items.append( + { + "type": "custom_tool_call_output", + "call_id": tool_call_id, + "output": content if isinstance(content, str) else tool_output, + } + ) + else: + input_items.append( + { + "type": "function_call_output", + "call_id": tool_call_id, + "output": tool_output, + } + ) elif role == "assistant" and tool_calls and isinstance(tool_calls, list): for r_item in _get_reasoning_items(msg): input_items.append(_reasoning_item_to_response_input(r_item)) for tool_call in tool_calls: function = tool_call.get("function") + custom = tool_call.get("custom") if function: input_tool_call: Dict[str, Any] = { "type": "function_call", @@ -288,6 +299,16 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if "arguments" in function: input_tool_call["arguments"] = function["arguments"] input_items.append(input_tool_call) + elif isinstance(custom, dict): + custom_tool_call_ids.add(tool_call["id"]) + input_items.append( + { + "type": "custom_tool_call", + "call_id": tool_call["id"], + "name": custom.get("name", ""), + "input": custom.get("input", ""), + } + ) else: raise ValueError(f"tool call not supported: {tool_call}") elif content is not None: @@ -1272,6 +1293,27 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): # New output item added output_item = parsed_chunk.get("item", {}) if output_item.get("type") in ("function_call", "custom_tool_call"): + if tool_call_index_map is None: + # Stateless callers (the responses guardrail handler extracting + # tool calls from a buffered output_item.done) get the complete + # tool call; per-stream callers already received it via + # output_item.added and the argument delta events + return ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta( + tool_calls=[ + { + **_tool_call_dict_from_output_item(dict(output_item)), + "index": parsed_chunk.get("output_index", 0), + } + ] + ), + finish_reason=None, + ) + ] + ) # Do NOT emit finish_reason here — response.completed handles the terminal # finish_reason. Emitting "tool_calls" here would prematurely terminate # the stream before subsequent tool calls arrive (same fix as #17246 for diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 8a678601284..8ee742c0d69 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -95,6 +95,13 @@ def _flatten_chat_tools_for_responses(tools: list) -> list: return [_flatten_chat_tool_for_responses(tool) for tool in tools] +def _is_chat_completions_body(data: dict) -> bool: + messages = data.get("messages") + if isinstance(messages, list) and len(messages) > 0: + return True + return "messages" in data and "input" not in data + + def _flatten_chat_tool_choice_for_responses(tool_choice: object) -> object: if not isinstance(tool_choice, dict): return tool_choice @@ -456,9 +463,11 @@ async def cursor_chat_completions( data = await _read_request_body(request=request) - if "messages" in data: + if _is_chat_completions_body(data): # Genuine chat completions body (Cursor sends these for models whose BYOK it - # already fixed); delegate so behavior matches /chat/completions exactly + # already fixed); delegate so behavior matches /chat/completions exactly. + # Keyed on messages CONTENT, not key presence: Cursor can send a null or + # empty messages stub alongside a real agent-mode input array tools = data.get("tools") tool_choice = data.get("tool_choice") normalized: dict = {} diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 64112ed43e8..b8bd5c951ee 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -3297,3 +3297,106 @@ def test_convert_tools_to_responses_format_text_format_passes_through(): [{"type": "custom", "custom": {"name": "A", "format": {"type": "text"}}}] ) assert converted[0] == {"type": "custom", "name": "A", "format": {"type": "text"}} + + +def test_convert_chat_completion_messages_maps_custom_tool_call_history(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + input_items, instructions = handler.convert_chat_completion_messages_to_responses_api( + [ + {"role": "user", "content": "use ApplyPatch"}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_c", + "type": "custom", + "custom": {"name": "ApplyPatch", "input": "*** Begin Patch"}, + }, + { + "id": "call_f", + "type": "function", + "function": {"name": "shell", "arguments": '{"cmd": "ls"}'}, + }, + ], + }, + {"role": "tool", "tool_call_id": "call_c", "content": "patch applied"}, + {"role": "tool", "tool_call_id": "call_f", "content": "a.py"}, + ] + ) + assert { + "type": "custom_tool_call", + "call_id": "call_c", + "name": "ApplyPatch", + "input": "*** Begin Patch", + } in input_items + assert {"type": "custom_tool_call_output", "call_id": "call_c", "output": "patch applied"} in input_items + assert {"type": "function_call", "call_id": "call_f", "name": "shell", "arguments": '{"cmd": "ls"}'} in input_items + assert { + "type": "function_call_output", + "call_id": "call_f", + "output": [{"type": "input_text", "text": "a.py"}], + } in input_items + + +def test_convert_chat_completion_messages_still_rejects_unknown_tool_call_shape(): + import pytest + + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + with pytest.raises(ValueError, match="tool call not supported"): + handler.convert_chat_completion_messages_to_responses_api( + [{"role": "assistant", "tool_calls": [{"id": "call_x", "type": "mystery"}]}] + ) + + +def test_output_item_done_stateless_emits_complete_tool_call(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + + for item, expected_name, expected_args in ( + ( + {"type": "function_call", "call_id": "call_f", "name": "shell", "arguments": '{"cmd": "ls"}'}, + "shell", + '{"cmd": "ls"}', + ), + ( + {"type": "custom_tool_call", "call_id": "call_c", "name": "ApplyPatch", "input": "*** Begin Patch"}, + "ApplyPatch", + "*** Begin Patch", + ), + ): + chunk = OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream( + {"type": "response.output_item.done", "output_index": 2, "item": item} + ) + tool_calls = chunk.choices[0].delta.tool_calls + assert tool_calls is not None and len(tool_calls) == 1 + assert tool_calls[0].id == item["call_id"] + assert tool_calls[0].function.name == expected_name + assert tool_calls[0].function.arguments == expected_args + assert tool_calls[0].index == 2 + assert chunk.choices[0].finish_reason is None + + +def test_output_item_done_with_stream_map_keeps_empty_delta(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + + chunk = OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream( + { + "type": "response.output_item.done", + "output_index": 0, + "item": {"type": "custom_tool_call", "call_id": "call_c", "name": "ApplyPatch", "input": "x"}, + }, + tool_call_index_map={0: 0}, + ) + assert chunk.choices[0].delta.tool_calls is None + assert chunk.choices[0].finish_reason is None diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index 04f027ac4bb..6a1e0d0a494 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -1287,3 +1287,62 @@ class TestCursorInputArmFlattening: {"type": "function", "name": "read_file", "parameters": {"type": "object"}}, ] assert call_kwargs["tool_choice"] == {"type": "custom", "name": "ApplyPatch"} + + +class TestChatCompletionsBodyDetection: + def test_routing_matrix(self): + from litellm.proxy.response_api_endpoints.endpoints import _is_chat_completions_body + + assert _is_chat_completions_body({"messages": [{"role": "user", "content": "hi"}]}) is True + assert _is_chat_completions_body({"messages": [{"role": "user", "content": "hi"}], "input": []}) is True + assert _is_chat_completions_body({"messages": None, "input": [{"role": "user", "content": "hi"}]}) is False + assert _is_chat_completions_body({"messages": [], "input": [{"role": "user", "content": "hi"}]}) is False + assert _is_chat_completions_body({"messages": None}) is True + assert _is_chat_completions_body({"messages": []}) is True + assert _is_chat_completions_body({"input": [{"role": "user", "content": "hi"}]}) is False + assert _is_chat_completions_body({}) is False + + @pytest.mark.asyncio + async def test_null_messages_stub_with_input_reaches_responses_arm(self): + from openai.types.responses import ResponseOutputMessage, ResponseOutputText + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.types.llms.openai import ResponsesAPIResponse + + mock_response = ResponsesAPIResponse( + id="resp_stub1", + created_at=1234567890, + model="gpt-5.6", + object="response", + output=[ + ResponseOutputMessage( + id="msg_stub1", + type="message", + role="assistant", + status="completed", + content=[ResponseOutputText(type="output_text", text="ok", annotations=[])], + ) + ], + ) + + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-1234") + try: + with patch("litellm.proxy.proxy_server.llm_router") as mock_router: + mock_router.aresponses = AsyncMock(return_value=mock_response) + client = TestClient(app) + response = client.post( + "/cursor/chat/completions", + json={ + "model": "gpt-5.6", + "messages": None, + "input": [{"role": "user", "content": "hello"}], + }, + headers={"Authorization": "Bearer sk-1234"}, + ) + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 200 + assert mock_router.aresponses.call_args is not None + assert mock_router.aresponses.call_args.kwargs["input"] == [{"role": "user", "content": "hello"}] From a5ba1caac5c312ff2189b1200d1eda54b5cac8e6 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Tue, 21 Jul 2026 21:06:41 -0700 Subject: [PATCH 022/124] test(helicone): stub the anthropic module unconditionally An import probe proves nothing about the real SDK: it may be absent (it lives in the proxy-runtime extra) and the tests/test_litellm/llms/anthropic test package can shadow it once collection puts that path on sys.path, which made the test order-sensitive across collection sets --- tests/test_litellm/integrations/test_helicone.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/test_litellm/integrations/test_helicone.py b/tests/test_litellm/integrations/test_helicone.py index eeef6fadbe5..da07fa1a9bf 100644 --- a/tests/test_litellm/integrations/test_helicone.py +++ b/tests/test_litellm/integrations/test_helicone.py @@ -13,13 +13,15 @@ def _claude_mapping(messages, response_obj): def test_claude_mapping_serializes_custom_tool_calls(monkeypatch): - try: - import anthropic # noqa: F401 - except ImportError: - stub = types.ModuleType("anthropic") - stub.HUMAN_PROMPT = "\n\nHuman:" - stub.AI_PROMPT = "\n\nAssistant:" - monkeypatch.setitem(sys.modules, "anthropic", stub) + """ + Stub the anthropic module unconditionally: the SDK may be absent (it lives in the + proxy-runtime extra), and the tests/test_litellm/llms/anthropic test package can + shadow it on sys.path, so an import probe proves nothing about the real SDK. + """ + stub = types.ModuleType("anthropic") + stub.HUMAN_PROMPT = "\n\nHuman:" + stub.AI_PROMPT = "\n\nAssistant:" + monkeypatch.setitem(sys.modules, "anthropic", stub) response_obj = { "id": "chatcmpl-1", "choices": [ From d516a72c0587fe813d7fdce39e5741fd74f2f660 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Tue, 21 Jul 2026 21:47:51 -0700 Subject: [PATCH 023/124] fix(litellm): scope the unset-effort responses bridge to constraint-enforcing endpoints Chat-only OpenAI-compatible backends registered under the openai provider with custom api_base and gpt-5.4+ model names served tools-without-reasoning fine and have no /responses route, so the unset-effort arm added for real OpenAI would have silently rerouted previously working deployments. The arm now fires only when api_base is unset (default OpenAI endpoint) or the provider is azure; an explicit reasoning_effort keeps its pre-existing bridging behavior on any api_base. Flagged lines also modernized to PEP 604 --- .../convert_dict_to_response.py | 9 +-- .../llms/openai/chat/gpt_transformation.py | 4 +- litellm/main.py | 17 +++++- tests/test_litellm/test_main.py | 58 +++++++++++++++++++ 4 files changed, 78 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index c5cfdea9ffe..1b23db87264 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -531,12 +531,9 @@ class LiteLLMResponseObjectHandler: def _should_convert_tool_call_to_json_mode( - tool_calls: Optional[ - Union[ - List[Union[ChatCompletionMessageToolCall, ChatCompletionMessageCustomToolCall]], - List[DatabricksTool], - ] - ] = None, + tool_calls: ( + list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall] | list[DatabricksTool] | None + ) = None, convert_tool_call_to_json_mode: Optional[bool] = None, ) -> bool: """ diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 129a9b51d0d..e4492a8aba6 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -533,9 +533,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): for choice in choices: ## HANDLE JSON MODE - anthropic returns single function call] tool_calls = choice["message"].get("tool_calls", None) - new_tool_calls: Optional[ - List[Union[ChatCompletionMessageToolCall, ChatCompletionMessageCustomToolCall]] - ] = None + new_tool_calls: list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall] | None = None message_content = choice["message"].get("content", None) if tool_calls is not None: _openai_tool_calls = [] diff --git a/litellm/main.py b/litellm/main.py index b6c6b44a6f7..d008d976130 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -986,6 +986,7 @@ def responses_api_bridge_check( tools: Optional[List[Any]] = None, reasoning_effort: Optional[Any] = None, reasoning_summary: Optional[Any] = None, + api_base: str | None = None, ) -> Tuple[dict, str]: model_info: Dict[str, Any] = {} @@ -1030,6 +1031,12 @@ def responses_api_bridge_check( # ``"none"`` keeps the request chat-servable. Custom (grammar) tools are served # natively by Chat Completions with reasoning on, so custom-only requests stay on # chat and keep their native custom tool_call response shape. + # - The UNSET-effort arm only fires against endpoints known to enforce that + # constraint (the default OpenAI endpoint, or Azure OpenAI where api_base is + # always set): chat-only OpenAI-compatible backends registered under the openai + # provider with a custom api_base and gpt-5.4+ model names serve tools without + # reasoning fine and have no /responses route, so they keep pre-existing + # behavior (bridge only on an explicit reasoning_effort). # - Older GPT-5 names (e.g. ``gpt-5``, ``gpt-5.1``): bridge only when a reasoning # summary alias is present with ``reasoning_effort`` (tools alone stay on chat). has_function_tool = any( @@ -1040,6 +1047,7 @@ def responses_api_bridge_check( reasoning_active = reasoning_effort.get("effort") != "none" or reasoning_effort.get("summary") is not None else: reasoning_active = reasoning_effort != "none" + on_constraint_enforcing_endpoint = custom_llm_provider == "azure" or api_base is None if ( custom_llm_provider in ("openai", "azure") and model_info.get("mode") != "responses" @@ -1047,7 +1055,12 @@ def responses_api_bridge_check( and not OpenAIGPT5Config.is_model_gpt_5_search_model(model) and ( (reasoning_effort is not None and reasoning_summary is not None) - or (OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) and has_function_tool and reasoning_active) + or ( + OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) + and has_function_tool + and reasoning_active + and (reasoning_effort is not None or on_constraint_enforcing_endpoint) + ) ) ): model_info["mode"] = "responses" @@ -5173,6 +5186,7 @@ def completion( # type: ignore model=model, custom_llm_provider=custom_llm_provider, web_search_options=web_search_options, + api_base=api_base, ) if not _should_allow_input_examples(custom_llm_provider=custom_llm_provider, model=model): @@ -5412,6 +5426,7 @@ def completion( # type: ignore tools=tools, reasoning_effort=reasoning_effort, reasoning_summary=_reasoning_summary_for_bridge, + api_base=api_base, ) # Use base_model (the true underlying model) for Azure model-type diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 60558760f8e..f72d2b5e23b 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -998,6 +998,64 @@ def test_responses_api_bridge_check_dict_effort_none_with_summary_routes_to_resp assert model_info.get("mode") == "responses" +def test_responses_api_bridge_check_custom_api_base_with_unset_effort_stays_chat(): + """ + Chat-only OpenAI-compatible backends registered under the openai provider with a + custom api_base and gpt-5.4+ model names serve tools-without-reasoning fine and + have no /responses route; the unset-effort arm must not reroute them. + """ + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5.6", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort=None, + api_base="http://vllm.internal:8000/v1", + ) + + assert model == "gpt-5.6" + assert model_info.get("mode") != "responses" + + +def test_responses_api_bridge_check_custom_api_base_with_explicit_effort_still_routes(): + """Explicit reasoning_effort keeps its pre-existing bridging behavior on any api_base.""" + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5.6", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort="high", + api_base="http://vllm.internal:8000/v1", + ) + + assert model == "gpt-5.6" + assert model_info.get("mode") == "responses" + + +def test_responses_api_bridge_check_azure_with_api_base_and_unset_effort_routes(): + """Azure OpenAI always sets api_base and does enforce the constraint; keep bridging.""" + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5.4", + custom_llm_provider="azure", + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort=None, + api_base="https://myresource.openai.azure.com", + ) + + assert model == "gpt-5.4" + assert model_info.get("mode") == "responses" + + def test_responses_api_bridge_check_older_gpt_5_tools_without_reasoning_stays_chat(): """Pre-5.4 GPT-5 names keep the old boundary: tools alone never bridge.""" from litellm.main import responses_api_bridge_check From cc00650fecfd9b3bb1b44806a5cf8dc71e043dcf Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Wed, 22 Jul 2026 15:46:55 -0700 Subject: [PATCH 024/124] fix(litellm): treat a blank api_base as the default OpenAI endpoint in the bridge gate A blank api_base (empty or whitespace) resolves to the default OpenAI base downstream but is not None, so the constraint-enforcing-endpoint check misclassified it as a custom backend and skipped the unset-effort auto-bridge, leaving gpt-5.4+ function-tool requests to 400 at OpenAI. The check now treats None, empty, and whitespace api_base alike; a real custom base still opts out. Verified with get_llm_provider, which passes a blank api_base through while resolving the provider to openai --- litellm/main.py | 5 ++++- tests/test_litellm/test_main.py | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index d008d976130..4aa6bf9a19b 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1047,7 +1047,10 @@ def responses_api_bridge_check( reasoning_active = reasoning_effort.get("effort") != "none" or reasoning_effort.get("summary") is not None else: reasoning_active = reasoning_effort != "none" - on_constraint_enforcing_endpoint = custom_llm_provider == "azure" or api_base is None + # A blank api_base (None, "", or whitespace) is not a custom endpoint: it resolves + # to the default OpenAI base downstream, which does enforce the reasoning+tools + # constraint. Azure always targets an OpenAI-constraint endpoint regardless. + on_constraint_enforcing_endpoint = custom_llm_provider == "azure" or not (api_base and api_base.strip()) if ( custom_llm_provider in ("openai", "azure") and model_info.get("mode") != "responses" diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index f72d2b5e23b..057d11e1ecd 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -998,6 +998,29 @@ def test_responses_api_bridge_check_dict_effort_none_with_summary_routes_to_resp assert model_info.get("mode") == "responses" +@pytest.mark.parametrize("blank_api_base", [None, "", " ", "\t"]) +def test_responses_api_bridge_check_blank_api_base_is_default_openai(blank_api_base): + """ + A blank api_base (None, empty, or whitespace) resolves to the default OpenAI + endpoint downstream, which enforces the reasoning+tools constraint, so gpt-5.4+ + function-tool requests with unset reasoning_effort must still auto-bridge. + """ + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5.6", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort=None, + api_base=blank_api_base, + ) + + assert model == "gpt-5.6" + assert model_info.get("mode") == "responses" + + def test_responses_api_bridge_check_custom_api_base_with_unset_effort_stays_chat(): """ Chat-only OpenAI-compatible backends registered under the openai provider with a From 56cc475c801db630077737bee23556bf0fe55d53 Mon Sep 17 00:00:00 2001 From: tin Date: Thu, 23 Jul 2026 00:46:13 +0000 Subject: [PATCH 025/124] refactor(cursor): trim LOC in cursor byok tool normalization - share one _CustomToolCallAccess mixin across the 4 new custom-tool classes instead of hand-rolling dict access on each - inline the single-use _nest_flat_chat_tools / _flatten_chat_tools_for_responses list wrappers at their call sites - drop _nest_flat_chat_tool_choice: it rewrote object-form chat tool_choice into {type,custom:{name}}, a shape OpenAI rejects; real Cursor never sends tool_choice on the messages arm, so pass it through unchanged --- .../proxy/response_api_endpoints/endpoints.py | 26 +--- litellm/types/utils.py | 64 +++------- .../response_api_endpoints/test_endpoints.py | 115 ++++++------------ 3 files changed, 58 insertions(+), 147 deletions(-) diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 8ee742c0d69..a3c1100eec3 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -52,20 +52,6 @@ def _nest_flat_chat_tool(tool: object) -> object: return tool -def _nest_flat_chat_tools(tools: list) -> list: - return [_nest_flat_chat_tool(tool) for tool in tools] - - -def _nest_flat_chat_tool_choice(tool_choice: object) -> object: - if not isinstance(tool_choice, dict) or "name" not in tool_choice: - return tool_choice - if tool_choice.get("type") == "custom" and "custom" not in tool_choice: - return {"type": "custom", "custom": {"name": tool_choice["name"]}} - if tool_choice.get("type") == "function" and "function" not in tool_choice: - return {"type": "function", "function": {"name": tool_choice["name"]}} - return tool_choice - - def _flatten_chat_tool_for_responses(tool: object) -> object: from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_custom_tool_format_to_responses_shape, @@ -91,10 +77,6 @@ def _flatten_chat_tool_for_responses(tool: object) -> object: return tool -def _flatten_chat_tools_for_responses(tools: list) -> list: - return [_flatten_chat_tool_for_responses(tool) for tool in tools] - - def _is_chat_completions_body(data: dict) -> bool: messages = data.get("messages") if isinstance(messages, list) and len(messages) > 0: @@ -469,15 +451,11 @@ async def cursor_chat_completions( # Keyed on messages CONTENT, not key presence: Cursor can send a null or # empty messages stub alongside a real agent-mode input array tools = data.get("tools") - tool_choice = data.get("tool_choice") normalized: dict = {} if isinstance(tools, list): - nested_tools = _nest_flat_chat_tools(tools) + nested_tools = [_nest_flat_chat_tool(tool) for tool in tools] if nested_tools != tools: normalized["tools"] = nested_tools - nested_tool_choice = _nest_flat_chat_tool_choice(tool_choice) - if nested_tool_choice != tool_choice: - normalized["tool_choice"] = nested_tool_choice if normalized: _safe_set_request_parsed_body(request=request, parsed_body={**data, **normalized}) return await chat_completion( @@ -496,7 +474,7 @@ async def cursor_chat_completions( tools = data.get("tools") if isinstance(tools, list): - data = {**data, "tools": _flatten_chat_tools_for_responses(tools)} + data = {**data, "tools": [_flatten_chat_tool_for_responses(tool) for tool in tools]} tool_choice = data.get("tool_choice") flattened_tool_choice = _flatten_chat_tool_choice_for_responses(tool_choice) if flattened_tool_choice != tool_choice: diff --git a/litellm/types/utils.py b/litellm/types/utils.py index a1e52f7584f..7ff12b617e3 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1084,70 +1084,42 @@ class ChatCompletionDeltaToolCall(OpenAIObject): setattr(self, key, value) -class ChatCompletionCustomToolCallPayload(OpenAIObject): +class _CustomToolCallAccess(OpenAIObject): + def __contains__(self, key): + return hasattr(self, key) + + def get(self, key, default=None): + return getattr(self, key, default) + + def __getitem__(self, key): + return getattr(self, key) + + def __setitem__(self, key, value): + setattr(self, key, value) + + +class ChatCompletionCustomToolCallPayload(_CustomToolCallAccess): name: str input: str - def __contains__(self, key): - return hasattr(self, key) - def get(self, key, default=None): - return getattr(self, key, default) - - def __getitem__(self, key): - return getattr(self, key) - - -class ChatCompletionDeltaCustomToolCallPayload(OpenAIObject): +class ChatCompletionDeltaCustomToolCallPayload(_CustomToolCallAccess): name: str | None = None input: str | None = None - def __contains__(self, key): - return hasattr(self, key) - def get(self, key, default=None): - return getattr(self, key, default) - - def __getitem__(self, key): - return getattr(self, key) - - -class ChatCompletionMessageCustomToolCall(OpenAIObject): +class ChatCompletionMessageCustomToolCall(_CustomToolCallAccess): id: str type: Literal["custom"] = "custom" custom: ChatCompletionCustomToolCallPayload - def __contains__(self, key): - return hasattr(self, key) - def get(self, key, default=None): - return getattr(self, key, default) - - def __getitem__(self, key): - return getattr(self, key) - - def __setitem__(self, key, value): - setattr(self, key, value) - - -class ChatCompletionDeltaCustomToolCall(OpenAIObject): +class ChatCompletionDeltaCustomToolCall(_CustomToolCallAccess): id: str | None = None type: str | None = None custom: ChatCompletionDeltaCustomToolCallPayload index: int - def __contains__(self, key): - return hasattr(self, key) - - def get(self, key, default=None): - return getattr(self, key, default) - - def __getitem__(self, key): - return getattr(self, key) - - def __setitem__(self, key, value): - setattr(self, key, value) - class ChatCompletionMessageToolCall(OpenAIObject): def __init__( diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index 6a1e0d0a494..88e7bbc84d4 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -911,33 +911,29 @@ def test_cursor_models_route_delegates_to_model_list(): class TestNestFlatChatTools: def test_flat_custom_tool_is_nested(self): - from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tools + from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tool - result = _nest_flat_chat_tools( - [{"type": "custom", "name": "ApplyPatch", "description": "V4A patch", "format": {"type": "text"}}] + result = _nest_flat_chat_tool( + {"type": "custom", "name": "ApplyPatch", "description": "V4A patch", "format": {"type": "text"}} ) - assert result == [ - { - "type": "custom", - "custom": {"name": "ApplyPatch", "description": "V4A patch", "format": {"type": "text"}}, - } - ] + assert result == { + "type": "custom", + "custom": {"name": "ApplyPatch", "description": "V4A patch", "format": {"type": "text"}}, + } def test_flat_function_tool_is_nested(self): - from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tools + from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tool - result = _nest_flat_chat_tools( - [{"type": "function", "name": "read_file", "description": "d", "parameters": {"type": "object"}}] + result = _nest_flat_chat_tool( + {"type": "function", "name": "read_file", "description": "d", "parameters": {"type": "object"}} ) - assert result == [ - { - "type": "function", - "function": {"name": "read_file", "description": "d", "parameters": {"type": "object"}}, - } - ] + assert result == { + "type": "function", + "function": {"name": "read_file", "description": "d", "parameters": {"type": "object"}}, + } def test_already_nested_and_unrecognized_tools_pass_through_unchanged(self): - from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tools + from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tool tools = [ {"type": "custom", "custom": {"name": "already_nested"}}, @@ -950,7 +946,7 @@ class TestNestFlatChatTools: None, 42, ] - assert _nest_flat_chat_tools(tools) == tools + assert [_nest_flat_chat_tool(tool) for tool in tools] == tools class TestCursorMessagesArmToolNormalization: @@ -1014,7 +1010,7 @@ class TestCursorMessagesArmToolNormalization: }, }, ] - assert seen["body"]["tool_choice"] == {"type": "custom", "custom": {"name": "ApplyPatch"}} + assert seen["body"]["tool_choice"] == {"type": "custom", "name": "ApplyPatch"} assert seen["body"]["messages"] == [{"role": "user", "content": "use ApplyPatch"}] @pytest.mark.asyncio @@ -1066,7 +1062,7 @@ class TestNestFlatChatToolShapeMatrix: @pytest.mark.parametrize("envelope", ["flat", "nested"]) @pytest.mark.parametrize("format_shape", ["absent", "text", "flat_grammar", "nested_grammar"]) def test_every_envelope_and_format_combination_lands_canonical(self, envelope, format_shape): - from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tools + from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tool format_value = { "absent": None, @@ -1085,10 +1081,10 @@ class TestNestFlatChatToolShapeMatrix: elif format_shape == "text": canonical_payload["format"] = self.TEXT - assert _nest_flat_chat_tools([tool]) == [{"type": "custom", "custom": canonical_payload}] + assert _nest_flat_chat_tool(tool) == {"type": "custom", "custom": canonical_payload} def test_nested_envelope_with_flat_grammar_matches_live_cursor_capture(self): - from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tools + from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tool cursor_tool = { "type": "custom", @@ -1097,60 +1093,25 @@ class TestNestFlatChatToolShapeMatrix: "format": {"type": "grammar", "definition": "start: patch", "syntax": "lark"}, }, } - assert _nest_flat_chat_tools([cursor_tool]) == [ - { - "type": "custom", - "custom": { - "name": "ApplyPatch", - "format": { - "type": "grammar", - "grammar": {"definition": "start: patch", "syntax": "lark"}, - }, + assert _nest_flat_chat_tool(cursor_tool) == { + "type": "custom", + "custom": { + "name": "ApplyPatch", + "format": { + "type": "grammar", + "grammar": {"definition": "start: patch", "syntax": "lark"}, }, - } - ] + }, + } def test_canonical_nested_tool_is_returned_equal(self): - from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tools + from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tool canonical = { "type": "custom", "custom": {"name": "A", "format": {"type": "grammar", "grammar": {"definition": "d", "syntax": "lark"}}}, } - assert _nest_flat_chat_tools([canonical]) == [canonical] - - -class TestNestFlatChatToolChoice: - def test_flat_custom_tool_choice_is_nested(self): - from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tool_choice - - assert _nest_flat_chat_tool_choice({"type": "custom", "name": "ApplyPatch"}) == { - "type": "custom", - "custom": {"name": "ApplyPatch"}, - } - - def test_flat_function_tool_choice_is_nested(self): - from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tool_choice - - assert _nest_flat_chat_tool_choice({"type": "function", "name": "f"}) == { - "type": "function", - "function": {"name": "f"}, - } - - def test_non_flat_tool_choice_values_pass_through(self): - from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tool_choice - - for unchanged in ( - "auto", - "required", - None, - {"type": "custom", "custom": {"name": "x"}}, - {"type": "function", "function": {"name": "f"}}, - {"type": "auto"}, - {"name": "typeless"}, - 42, - ): - assert _nest_flat_chat_tool_choice(unchanged) == unchanged + assert _nest_flat_chat_tool(canonical) == canonical class TestFlattenChatToolsForResponsesInputArm: @@ -1165,7 +1126,7 @@ class TestFlattenChatToolsForResponsesInputArm: @pytest.mark.parametrize("envelope", ["flat", "nested"]) @pytest.mark.parametrize("format_shape", ["absent", "text", "flat_grammar", "nested_grammar"]) def test_every_envelope_and_format_combination_lands_flat(self, envelope, format_shape): - from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tools_for_responses + from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tool_for_responses format_value = { "absent": None, @@ -1184,21 +1145,21 @@ class TestFlattenChatToolsForResponsesInputArm: elif format_shape == "text": canonical["format"] = {"type": "text"} - assert _flatten_chat_tools_for_responses([tool]) == [canonical] + assert _flatten_chat_tool_for_responses(tool) == canonical def test_nested_function_tool_is_flattened_and_flat_passes_through(self): - from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tools_for_responses + from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tool_for_responses nested = {"type": "function", "function": {"name": "read_file", "parameters": {"type": "object"}}} flat = {"type": "function", "name": "read_file", "parameters": {"type": "object"}} - assert _flatten_chat_tools_for_responses([nested]) == [flat] - assert _flatten_chat_tools_for_responses([flat]) == [flat] + assert _flatten_chat_tool_for_responses(nested) == flat + assert _flatten_chat_tool_for_responses(flat) == flat def test_unrecognized_entries_pass_through(self): - from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tools_for_responses + from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tool_for_responses entries = [{"type": "web_search"}, {"type": "custom"}, "junk", None, {}] - assert _flatten_chat_tools_for_responses(entries) == entries + assert [_flatten_chat_tool_for_responses(entry) for entry in entries] == entries class TestFlattenChatToolChoiceForResponsesInputArm: From c7c656e8a9132fd583a053ee9cf3d6da95535f4b Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Wed, 22 Jul 2026 23:51:19 -0700 Subject: [PATCH 026/124] fix(cursor): convert tools and tool_choice through one envelope rule Chat Completions nests a named tool_choice under its tool type while the Responses API keeps it flat; ChatCompletionNamedToolChoiceParam and ChatCompletionNamedToolChoiceCustomParam both mark the nested key required. The messages arm normalized tool definitions but forwarded tool_choice at whatever level Cursor sent it, so a flat {"type": "custom", "name": "ApplyPatch"} reached OpenAI unchanged and was rejected while the tool defs beside it nested correctly A tool definition and a named tool_choice carry the same envelope, so both now convert through a single _convert_tool_envelope, and _normalize_tool_dialect moves tools and tool_choice together on each arm. That covers all four cells of {tool def, tool_choice} x {to chat, to responses} and removes the shape where one field can be converted while the other is missed, replacing three helpers with two and cutting 24 lines Also restores the end-to-end assertion that a flat tool_choice reaches chat_completion nested, which had been flipped to pin the passthrough behavior --- .../proxy/response_api_endpoints/endpoints.py | 109 ++++------ .../response_api_endpoints/test_endpoints.py | 195 +++++++++--------- 2 files changed, 140 insertions(+), 164 deletions(-) diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index a3c1100eec3..b67352f2057 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -24,57 +24,47 @@ router = APIRouter() _user_api_key_auth_dep = Depends(user_api_key_auth) -_FLAT_CUSTOM_TOOL_KEYS = ("name", "description", "format") -_FLAT_FUNCTION_TOOL_KEYS = ("name", "description", "parameters", "strict") +_TOOL_PAYLOAD_KEYS = { + "custom": ("name", "description", "format"), + "function": ("name", "description", "parameters", "strict"), +} -def _nest_flat_chat_tool(tool: object) -> object: +def _convert_tool_envelope(obj: object, *, to_chat: bool) -> object: from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_custom_tool_format_to_chat_shape, - ) - - if not isinstance(tool, dict): - return tool - if tool.get("type") == "custom": - if isinstance(tool.get("custom"), dict): - envelope = tool - payload = tool["custom"] - elif "name" in tool: - envelope = {"type": "custom"} - payload = {k: tool[k] for k in _FLAT_CUSTOM_TOOL_KEYS if k in tool} - else: - return tool - if isinstance(payload.get("format"), dict): - payload = {**payload, "format": convert_custom_tool_format_to_chat_shape(payload["format"])} - return {**envelope, "custom": payload} - if tool.get("type") == "function" and "function" not in tool and "name" in tool: - return {"type": "function", "function": {k: tool[k] for k in _FLAT_FUNCTION_TOOL_KEYS if k in tool}} - return tool - - -def _flatten_chat_tool_for_responses(tool: object) -> object: - from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_custom_tool_format_to_responses_shape, ) - if not isinstance(tool, dict): - return tool - if tool.get("type") == "custom": - if isinstance(tool.get("custom"), dict): - payload = {k: tool["custom"][k] for k in _FLAT_CUSTOM_TOOL_KEYS if k in tool["custom"]} - elif "name" in tool: - payload = {k: tool[k] for k in _FLAT_CUSTOM_TOOL_KEYS if k in tool} - else: - return tool - if isinstance(payload.get("format"), dict): - payload = {**payload, "format": convert_custom_tool_format_to_responses_shape(payload["format"])} - return {"type": "custom", **payload} - if tool.get("type") == "function" and isinstance(tool.get("function"), dict): - return { - "type": "function", - **{k: tool["function"][k] for k in _FLAT_FUNCTION_TOOL_KEYS if k in tool["function"]}, - } - return tool + if not isinstance(obj, dict): + return obj + tool_type = obj.get("type") + payload_keys = _TOOL_PAYLOAD_KEYS.get(tool_type) + if payload_keys is None: + return obj + nested = obj.get(tool_type) + source = nested if isinstance(nested, dict) else obj + if source is obj and "name" not in obj: + return obj + payload = {key: source[key] for key in payload_keys if key in source} + if isinstance(payload.get("format"), dict): + convert = convert_custom_tool_format_to_chat_shape if to_chat else convert_custom_tool_format_to_responses_shape + payload = {**payload, "format": convert(payload["format"])} + return {"type": tool_type, tool_type: payload} if to_chat else {"type": tool_type, **payload} + + +def _normalize_tool_dialect(data: dict, *, to_chat: bool) -> dict: + converted: dict = {} + tools = data.get("tools") + if isinstance(tools, list): + normalized_tools = [_convert_tool_envelope(tool, to_chat=to_chat) for tool in tools] + if normalized_tools != tools: + converted["tools"] = normalized_tools + tool_choice = data.get("tool_choice") + normalized_choice = _convert_tool_envelope(tool_choice, to_chat=to_chat) + if normalized_choice != tool_choice: + converted["tool_choice"] = normalized_choice + return {**data, **converted} if converted else data def _is_chat_completions_body(data: dict) -> bool: @@ -84,18 +74,6 @@ def _is_chat_completions_body(data: dict) -> bool: return "messages" in data and "input" not in data -def _flatten_chat_tool_choice_for_responses(tool_choice: object) -> object: - if not isinstance(tool_choice, dict): - return tool_choice - choice_type = tool_choice.get("type") - if choice_type not in ("custom", "function"): - return tool_choice - nested = tool_choice.get(choice_type) - if isinstance(nested, dict) and isinstance(nested.get("name"), str): - return {"type": choice_type, "name": nested["name"]} - return tool_choice - - @router.post( "/v1/responses", dependencies=[Depends(user_api_key_auth)], @@ -450,14 +428,9 @@ async def cursor_chat_completions( # already fixed); delegate so behavior matches /chat/completions exactly. # Keyed on messages CONTENT, not key presence: Cursor can send a null or # empty messages stub alongside a real agent-mode input array - tools = data.get("tools") - normalized: dict = {} - if isinstance(tools, list): - nested_tools = [_nest_flat_chat_tool(tool) for tool in tools] - if nested_tools != tools: - normalized["tools"] = nested_tools - if normalized: - _safe_set_request_parsed_body(request=request, parsed_body={**data, **normalized}) + normalized = _normalize_tool_dialect(data, to_chat=True) + if normalized is not data: + _safe_set_request_parsed_body(request=request, parsed_body=normalized) return await chat_completion( request=request, fastapi_response=fastapi_response, @@ -472,13 +445,7 @@ async def cursor_chat_completions( # cache's key snapshot so later readers get an empty body data = {key: value for key, value in data.items() if key != "stream_options"} - tools = data.get("tools") - if isinstance(tools, list): - data = {**data, "tools": [_flatten_chat_tool_for_responses(tool) for tool in tools]} - tool_choice = data.get("tool_choice") - flattened_tool_choice = _flatten_chat_tool_choice_for_responses(tool_choice) - if flattened_tool_choice != tool_choice: - data = {**data, "tool_choice": flattened_tool_choice} + data = _normalize_tool_dialect(data, to_chat=False) processor = ProxyBaseLLMRequestProcessing(data=data) diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index 88e7bbc84d4..fa5a16a9f3b 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -911,10 +911,11 @@ def test_cursor_models_route_delegates_to_model_list(): class TestNestFlatChatTools: def test_flat_custom_tool_is_nested(self): - from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tool + from litellm.proxy.response_api_endpoints.endpoints import _convert_tool_envelope - result = _nest_flat_chat_tool( - {"type": "custom", "name": "ApplyPatch", "description": "V4A patch", "format": {"type": "text"}} + result = _convert_tool_envelope( + {"type": "custom", "name": "ApplyPatch", "description": "V4A patch", "format": {"type": "text"}}, + to_chat=True, ) assert result == { "type": "custom", @@ -922,10 +923,11 @@ class TestNestFlatChatTools: } def test_flat_function_tool_is_nested(self): - from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tool + from litellm.proxy.response_api_endpoints.endpoints import _convert_tool_envelope - result = _nest_flat_chat_tool( - {"type": "function", "name": "read_file", "description": "d", "parameters": {"type": "object"}} + result = _convert_tool_envelope( + {"type": "function", "name": "read_file", "description": "d", "parameters": {"type": "object"}}, + to_chat=True, ) assert result == { "type": "function", @@ -933,7 +935,7 @@ class TestNestFlatChatTools: } def test_already_nested_and_unrecognized_tools_pass_through_unchanged(self): - from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tool + from litellm.proxy.response_api_endpoints.endpoints import _convert_tool_envelope tools = [ {"type": "custom", "custom": {"name": "already_nested"}}, @@ -946,7 +948,7 @@ class TestNestFlatChatTools: None, 42, ] - assert [_nest_flat_chat_tool(tool) for tool in tools] == tools + assert [_convert_tool_envelope(tool, to_chat=True) for tool in tools] == tools class TestCursorMessagesArmToolNormalization: @@ -1010,7 +1012,7 @@ class TestCursorMessagesArmToolNormalization: }, }, ] - assert seen["body"]["tool_choice"] == {"type": "custom", "name": "ApplyPatch"} + assert seen["body"]["tool_choice"] == {"type": "custom", "custom": {"name": "ApplyPatch"}} assert seen["body"]["messages"] == [{"role": "user", "content": "use ApplyPatch"}] @pytest.mark.asyncio @@ -1048,21 +1050,23 @@ class TestCursorMessagesArmToolNormalization: assert seen["body"]["messages"] == body["messages"] -class TestNestFlatChatToolShapeMatrix: +class TestToolEnvelopeConversionMatrix: """ Cursor mixes Responses API shapes into chat bodies PER LEVEL, independently (live-captured: a pre-nested custom envelope carrying a flat grammar format). - Every cell of envelope x format must land on the canonical chat shape. + Tool definitions and tool_choice share one envelope rule, so every cell of + direction x envelope x format must land on that direction's canonical shape. """ FLAT_GRAMMAR = {"type": "grammar", "definition": "start: patch", "syntax": "lark"} NESTED_GRAMMAR = {"type": "grammar", "grammar": {"definition": "start: patch", "syntax": "lark"}} TEXT = {"type": "text"} + @pytest.mark.parametrize("to_chat", [True, False]) @pytest.mark.parametrize("envelope", ["flat", "nested"]) @pytest.mark.parametrize("format_shape", ["absent", "text", "flat_grammar", "nested_grammar"]) - def test_every_envelope_and_format_combination_lands_canonical(self, envelope, format_shape): - from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tool + def test_every_direction_envelope_and_format_lands_canonical(self, to_chat, envelope, format_shape): + from litellm.proxy.response_api_endpoints.endpoints import _convert_tool_envelope format_value = { "absent": None, @@ -1077,109 +1081,114 @@ class TestNestFlatChatToolShapeMatrix: canonical_payload = {"name": "ApplyPatch", "description": "V4A patch"} if format_shape in ("flat_grammar", "nested_grammar"): - canonical_payload["format"] = self.NESTED_GRAMMAR + canonical_payload["format"] = self.NESTED_GRAMMAR if to_chat else self.FLAT_GRAMMAR elif format_shape == "text": canonical_payload["format"] = self.TEXT + expected = ( + {"type": "custom", "custom": canonical_payload} if to_chat else {"type": "custom", **canonical_payload} + ) - assert _nest_flat_chat_tool(tool) == {"type": "custom", "custom": canonical_payload} + assert _convert_tool_envelope(tool, to_chat=to_chat) == expected def test_nested_envelope_with_flat_grammar_matches_live_cursor_capture(self): - from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tool + from litellm.proxy.response_api_endpoints.endpoints import _convert_tool_envelope - cursor_tool = { + cursor_tool = {"type": "custom", "custom": {"name": "ApplyPatch", "format": self.FLAT_GRAMMAR}} + assert _convert_tool_envelope(cursor_tool, to_chat=True) == { "type": "custom", - "custom": { - "name": "ApplyPatch", - "format": {"type": "grammar", "definition": "start: patch", "syntax": "lark"}, - }, - } - assert _nest_flat_chat_tool(cursor_tool) == { - "type": "custom", - "custom": { - "name": "ApplyPatch", - "format": { - "type": "grammar", - "grammar": {"definition": "start: patch", "syntax": "lark"}, - }, - }, + "custom": {"name": "ApplyPatch", "format": self.NESTED_GRAMMAR}, } - def test_canonical_nested_tool_is_returned_equal(self): - from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tool + @pytest.mark.parametrize("to_chat", [True, False]) + def test_conversion_is_idempotent(self, to_chat): + from litellm.proxy.response_api_endpoints.endpoints import _convert_tool_envelope - canonical = { - "type": "custom", - "custom": {"name": "A", "format": {"type": "grammar", "grammar": {"definition": "d", "syntax": "lark"}}}, - } - assert _nest_flat_chat_tool(canonical) == canonical + once = _convert_tool_envelope({"type": "custom", "name": "A", "format": self.FLAT_GRAMMAR}, to_chat=to_chat) + assert _convert_tool_envelope(once, to_chat=to_chat) == once - -class TestFlattenChatToolsForResponsesInputArm: - """ - Mirror of TestNestFlatChatToolShapeMatrix for the input arm: chat-nested shapes in a - Responses-shaped body must flatten to the Responses dialect, per level, idempotently. - """ - - FLAT_GRAMMAR = {"type": "grammar", "definition": "start: patch", "syntax": "lark"} - NESTED_GRAMMAR = {"type": "grammar", "grammar": {"definition": "start: patch", "syntax": "lark"}} - - @pytest.mark.parametrize("envelope", ["flat", "nested"]) - @pytest.mark.parametrize("format_shape", ["absent", "text", "flat_grammar", "nested_grammar"]) - def test_every_envelope_and_format_combination_lands_flat(self, envelope, format_shape): - from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tool_for_responses - - format_value = { - "absent": None, - "text": {"type": "text"}, - "flat_grammar": self.FLAT_GRAMMAR, - "nested_grammar": self.NESTED_GRAMMAR, - }[format_shape] - payload = {"name": "ApplyPatch", "description": "V4A patch"} - if format_value is not None: - payload["format"] = format_value - tool = {"type": "custom", "custom": payload} if envelope == "nested" else {"type": "custom", **payload} - - canonical = {"type": "custom", "name": "ApplyPatch", "description": "V4A patch"} - if format_shape in ("flat_grammar", "nested_grammar"): - canonical["format"] = self.FLAT_GRAMMAR - elif format_shape == "text": - canonical["format"] = {"type": "text"} - - assert _flatten_chat_tool_for_responses(tool) == canonical - - def test_nested_function_tool_is_flattened_and_flat_passes_through(self): - from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tool_for_responses + def test_nested_function_tool_flattens_and_flat_passes_through(self): + from litellm.proxy.response_api_endpoints.endpoints import _convert_tool_envelope nested = {"type": "function", "function": {"name": "read_file", "parameters": {"type": "object"}}} flat = {"type": "function", "name": "read_file", "parameters": {"type": "object"}} - assert _flatten_chat_tool_for_responses(nested) == flat - assert _flatten_chat_tool_for_responses(flat) == flat + assert _convert_tool_envelope(nested, to_chat=False) == flat + assert _convert_tool_envelope(flat, to_chat=False) == flat - def test_unrecognized_entries_pass_through(self): - from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tool_for_responses + @pytest.mark.parametrize("to_chat", [True, False]) + def test_unrecognized_entries_pass_through(self, to_chat): + from litellm.proxy.response_api_endpoints.endpoints import _convert_tool_envelope - entries = [{"type": "web_search"}, {"type": "custom"}, "junk", None, {}] - assert [_flatten_chat_tool_for_responses(entry) for entry in entries] == entries + entries = [{"type": "web_search"}, {"type": "custom"}, "junk", None, {}, 42, {"type": "auto"}] + assert [_convert_tool_envelope(entry, to_chat=to_chat) for entry in entries] == entries -class TestFlattenChatToolChoiceForResponsesInputArm: - def test_nested_custom_and_function_tool_choice_flatten(self): - from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tool_choice_for_responses +class TestToolChoiceSharesTheToolEnvelopeRule: + """ + tool_choice carries the same {"type": T, T: {...}} chat envelope as a tool + definition, so it converts through the same function in both directions. + OpenAI requires the nested key on chat (SDK ChatCompletionNamedToolChoiceParam + and ChatCompletionNamedToolChoiceCustomParam both mark it Required). + """ - assert _flatten_chat_tool_choice_for_responses({"type": "custom", "custom": {"name": "ApplyPatch"}}) == { - "type": "custom", + @pytest.mark.parametrize("choice_type", ["custom", "function"]) + def test_flat_tool_choice_is_nested_for_chat(self, choice_type): + from litellm.proxy.response_api_endpoints.endpoints import _convert_tool_envelope + + assert _convert_tool_envelope({"type": choice_type, "name": "ApplyPatch"}, to_chat=True) == { + "type": choice_type, + choice_type: {"name": "ApplyPatch"}, + } + + @pytest.mark.parametrize("choice_type", ["custom", "function"]) + def test_nested_tool_choice_is_flattened_for_responses(self, choice_type): + from litellm.proxy.response_api_endpoints.endpoints import _convert_tool_envelope + + assert _convert_tool_envelope({"type": choice_type, choice_type: {"name": "ApplyPatch"}}, to_chat=False) == { + "type": choice_type, "name": "ApplyPatch", } - assert _flatten_chat_tool_choice_for_responses({"type": "function", "function": {"name": "f"}}) == { - "type": "function", - "name": "f", - } - def test_flat_and_string_tool_choice_pass_through(self): - from litellm.proxy.response_api_endpoints.endpoints import _flatten_chat_tool_choice_for_responses + @pytest.mark.parametrize("to_chat", [True, False]) + def test_sentinel_and_malformed_tool_choice_pass_through(self, to_chat): + from litellm.proxy.response_api_endpoints.endpoints import _convert_tool_envelope - for unchanged in ("auto", "required", None, {"type": "custom", "name": "x"}, {"type": "auto"}, 42): - assert _flatten_chat_tool_choice_for_responses(unchanged) == unchanged + for unchanged in ("auto", "required", "none", None, {"type": "auto"}, 42): + assert _convert_tool_envelope(unchanged, to_chat=to_chat) == unchanged + + +class TestNormalizeToolDialectCoversBothFields: + """ + The regression that motivated one normalizer: tools were converted while + tool_choice was left flat, so OpenAI rejected the request. Both fields move + together in a single call, on both arms. + """ + + @pytest.mark.parametrize("to_chat", [True, False]) + def test_tools_and_tool_choice_convert_together(self, to_chat): + from litellm.proxy.response_api_endpoints.endpoints import _normalize_tool_dialect + + flat = {"type": "custom", "name": "ApplyPatch"} + nested = {"type": "custom", "custom": {"name": "ApplyPatch"}} + source = flat if to_chat else nested + expected = nested if to_chat else flat + + out = _normalize_tool_dialect({"messages": [], "tools": [source], "tool_choice": source}, to_chat=to_chat) + assert out["tools"] == [expected] + assert out["tool_choice"] == expected + + def test_body_needing_no_conversion_is_returned_by_identity(self): + from litellm.proxy.response_api_endpoints.endpoints import _normalize_tool_dialect + + data = {"messages": [], "tools": [{"type": "function", "function": {"name": "f"}}], "tool_choice": "auto"} + assert _normalize_tool_dialect(data, to_chat=True) is data + + def test_absent_tool_fields_are_not_invented(self): + from litellm.proxy.response_api_endpoints.endpoints import _normalize_tool_dialect + + data = {"messages": [{"role": "user", "content": "hi"}]} + result = _normalize_tool_dialect(data, to_chat=True) + assert result == data + assert "tools" not in result and "tool_choice" not in result class TestCursorInputArmFlattening: From 4139f548da9f3a72b7c9dc327aaaad730472fdbe Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Sat, 25 Jul 2026 10:45:41 -0700 Subject: [PATCH 027/124] fix(bridge): resolve the effective OpenAI base once, shared by gate and chat handler The gpt-5.4+ responses-bridge gate classified the endpoint from the call-level api_base alone, while the OpenAI chat handler resolves arg > global > env > default. A custom base configured via litellm.api_base or OPENAI_BASE_URL/ OPENAI_API_BASE was therefore invisible to the gate: it read blank as the default OpenAI endpoint and bridged a request the custom backend has no /responses route for. Extract that resolution into one _resolve_openai_api_base() and have both the gate and _complete_custom_openai() call it, so the gate can never classify an endpoint the request won't hit. The gate compares the resolved base against the default (import litellm seeds OPENAI_BASE_URL to the default, so "override is non-None" is not a safe custom-endpoint signal); whitespace collapses to the default as before. reasoning_effort="none" remains the escape hatch. --- litellm/main.py | 39 ++++++++++++++++++------- tests/test_litellm/test_main.py | 52 +++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 11 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 4aa6bf9a19b..fbb43dd41fa 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -979,6 +979,23 @@ def mock_completion( raise Exception("Mock completion response failed - {}".format(e)) +_OPENAI_DEFAULT_API_BASE = "https://api.openai.com/v1" + + +def _resolve_openai_api_base(api_base: str | None) -> str: + """Effective OpenAI base a chat request will hit: arg > global > env > default. The bridge gate + and the ``_complete_custom_openai`` chat handler MUST resolve this identically, or a custom base + set via ``litellm.api_base`` or ``OPENAI_BASE_URL``/``OPENAI_API_BASE`` is invisible to the gate, + which then misreads it as the default OpenAI endpoint and bridges a request the backend can't serve.""" + return ( + api_base + or litellm.api_base + or get_secret_str("OPENAI_BASE_URL") + or get_secret_str("OPENAI_API_BASE") + or _OPENAI_DEFAULT_API_BASE + ) + + def responses_api_bridge_check( model: str, custom_llm_provider: str, @@ -1047,10 +1064,15 @@ def responses_api_bridge_check( reasoning_active = reasoning_effort.get("effort") != "none" or reasoning_effort.get("summary") is not None else: reasoning_active = reasoning_effort != "none" - # A blank api_base (None, "", or whitespace) is not a custom endpoint: it resolves - # to the default OpenAI base downstream, which does enforce the reasoning+tools - # constraint. Azure always targets an OpenAI-constraint endpoint regardless. - on_constraint_enforcing_endpoint = custom_llm_provider == "azure" or not (api_base and api_base.strip()) + # The reasoning+tools constraint is enforced only by the real OpenAI endpoint (and Azure OpenAI). + # Resolve the effective base arg>global>env>default exactly as the chat handler does, so a custom + # base set via litellm.api_base or OPENAI_BASE_URL/OPENAI_API_BASE isn't misread as the default and + # bridged to a /responses route it lacks. A whitespace-only base collapses to the default too. + resolved_api_base = _resolve_openai_api_base(api_base) + on_constraint_enforcing_endpoint = custom_llm_provider == "azure" or resolved_api_base.strip() in ( + "", + _OPENAI_DEFAULT_API_BASE, + ) if ( custom_llm_provider in ("openai", "azure") and model_info.get("mode") != "responses" @@ -2396,13 +2418,8 @@ def _complete_custom_openai( stream = ctx.stream timeout = ctx.timeout - api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there - or litellm.api_base - or get_secret("OPENAI_BASE_URL") - or get_secret("OPENAI_API_BASE") - or "https://api.openai.com/v1" - ) + # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there + api_base = _resolve_openai_api_base(api_base) organization = ( organization or litellm.organization diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 057d11e1ecd..9e160370048 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -1043,6 +1043,58 @@ def test_responses_api_bridge_check_custom_api_base_with_unset_effort_stays_chat assert model_info.get("mode") != "responses" +def test_responses_api_bridge_check_custom_api_base_via_global_with_unset_effort_stays_chat(monkeypatch): + """ + A custom base set through the litellm.api_base global (not the call arg) is resolved the + same way the chat handler resolves it, so the unset-effort arm must not reroute a chat-only + backend to a /responses route it lacks. Regression guard: the gate previously inspected only + the call-level api_base and bridged these requests. + """ + import litellm + from litellm.main import responses_api_bridge_check + + monkeypatch.setattr(litellm, "api_base", "http://vllm.internal:8000/v1") + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5.6", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort=None, + api_base=None, + ) + + assert model == "gpt-5.6" + assert model_info.get("mode") != "responses" + + +@pytest.mark.parametrize("env_var", ["OPENAI_BASE_URL", "OPENAI_API_BASE"]) +def test_responses_api_bridge_check_custom_api_base_via_env_with_unset_effort_stays_chat(monkeypatch, env_var): + """ + A custom base set via OPENAI_BASE_URL/OPENAI_API_BASE env is resolved identically to the chat + handler, so the unset-effort arm leaves the request on chat instead of bridging it. + """ + import litellm + from litellm.main import responses_api_bridge_check + + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setenv(env_var, "http://vllm.internal:8000/v1") + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5.6", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort=None, + api_base=None, + ) + + assert model == "gpt-5.6" + assert model_info.get("mode") != "responses" + + def test_responses_api_bridge_check_custom_api_base_with_explicit_effort_still_routes(): """Explicit reasoning_effort keeps its pre-existing bridging behavior on any api_base.""" from litellm.main import responses_api_bridge_check From c27f1b7b6d274d6dfe6bbc123ee3c5c1b163e83a Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Mon, 27 Jul 2026 11:59:34 -0700 Subject: [PATCH 028/124] fix(tools): classify custom tool calls by one shared rule and make envelope payload extraction total A chat tool-call dict was classified custom-vs-function with four different spellings: the non-streaming parser required type == "custom", the streaming Delta coercion also accepted a custom payload without type, and the stream assembler required a type that later chunks never carry. The same payload could be a custom tool call mid-stream, a TypeError on the completed message, and silently dropped from the assembled message. is_custom_tool_call_dict() is now the single discriminator (explicit custom type, or a custom payload present) used by both parsers, and the assembler classifies from the accumulated custom payload, matching how the deltas it consumes were classified. The tool envelope converter picked one exclusive payload source: the nested dict when present, else the top level. An empty nested envelope therefore shadowed top-level fields and the normalized tool lost its name. Payload extraction is now total over both locations, nested first, and an envelope with no name anywhere passes through unchanged instead of being emitted stripped. --- .../streaming_chunk_builder_utils.py | 2 +- .../proxy/response_api_endpoints/endpoints.py | 10 +++-- litellm/types/utils.py | 10 +++-- .../test_streaming_chunk_builder_utils.py | 37 ++++++++++++++++ .../response_api_endpoints/test_endpoints.py | 24 +++++++++++ tests/test_litellm/types/test_types_utils.py | 42 +++++++++++++++++++ 6 files changed, 118 insertions(+), 7 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 09bd55096e8..f4f1b6fca0d 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -322,7 +322,7 @@ class ChunkProcessor: # Convert the map to a list of tool calls for index in sorted(tool_call_map.keys()): tool_call_data = tool_call_map[index] - if tool_call_data["type"] == "custom" and tool_call_data["id"] and tool_call_data["custom_name"]: + if tool_call_data["id"] and tool_call_data["custom_name"]: tool_calls_list.append( ChatCompletionMessageCustomToolCall( id=tool_call_data["id"], diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index b67352f2057..3a2c57aa2ba 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -43,10 +43,14 @@ def _convert_tool_envelope(obj: object, *, to_chat: bool) -> object: if payload_keys is None: return obj nested = obj.get(tool_type) - source = nested if isinstance(nested, dict) else obj - if source is obj and "name" not in obj: + nested_source = nested if isinstance(nested, dict) else {} + payload = { + key: nested_source[key] if key in nested_source else obj[key] + for key in payload_keys + if key in nested_source or key in obj + } + if "name" not in payload: return obj - payload = {key: source[key] for key in payload_keys if key in source} if isinstance(payload.get("format"), dict): convert = convert_custom_tool_format_to_chat_shape if to_chat else convert_custom_tool_format_to_responses_shape payload = {**payload, "format": convert(payload["format"])} diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 7ff12b617e3..404725ec61b 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1162,12 +1162,16 @@ class ChatCompletionMessageToolCall(OpenAIObject): setattr(self, key, value) +def is_custom_tool_call_dict(tool_call: dict) -> bool: + return tool_call.get("type") == "custom" or tool_call.get("custom") is not None + + def chat_completion_tool_call_from_dict( tool_call: dict, ) -> "ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall": - if tool_call.get("type") == "custom": + if is_custom_tool_call_dict(tool_call): return ChatCompletionMessageCustomToolCall( - **{k: v for k, v in tool_call.items() if not (k == "function" and v is None)} + **{k: v for k, v in tool_call.items() if not (k in ("function", "type") and v is None)} ) return ChatCompletionMessageToolCall(**tool_call) @@ -1393,7 +1397,7 @@ class Delta(SafeAttributeModel, OpenAIObject): if tool_call.get("index", None) is None: tool_call["index"] = current_index current_index += 1 - if tool_call.get("type") == "custom" or "custom" in tool_call: + if is_custom_tool_call_dict(tool_call): coerced_tool_calls.append( ChatCompletionDeltaCustomToolCall( **{k: v for k, v in tool_call.items() if not (k == "function" and v is None)} diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index 197adf80f03..2db5461702a 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -1027,3 +1027,40 @@ def test_get_combined_tool_content_custom_tool_call(): "type": "custom", "custom": {"name": "ApplyPatch", "input": "*** Begin Patch\n*** End Patch\n"}, } + + +def test_get_combined_tool_content_custom_tool_call_without_type_field(): + """Delta coercion classifies a tool-call chunk as custom from its ``custom`` payload + alone (``type`` may never arrive on any chunk). The assembler must use the same + evidence; requiring ``type == "custom"`` dropped the whole tool call from the + combined message (it matched neither the custom nor the function branch).""" + from litellm.litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor + from litellm.types.utils import ChatCompletionMessageCustomToolCall + + processor = ChunkProcessor.__new__(ChunkProcessor) + tool_call_chunks = [ + { + "choices": [ + { + "delta": { + "tool_calls": [ + { + "index": 0, + "id": "call_TBs", + "custom": {"name": "ApplyPatch", "input": "*** Begin"}, + } + ] + } + } + ] + }, + {"choices": [{"delta": {"tool_calls": [{"index": 0, "custom": {"input": " Patch"}}]}}]}, + ] + combined = processor.get_combined_tool_content(tool_call_chunks) + assert len(combined) == 1 + assert isinstance(combined[0], ChatCompletionMessageCustomToolCall) + assert combined[0].model_dump() == { + "id": "call_TBs", + "type": "custom", + "custom": {"name": "ApplyPatch", "input": "*** Begin Patch"}, + } diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index fa5a16a9f3b..00ac8ca386a 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -1121,6 +1121,30 @@ class TestToolEnvelopeConversionMatrix: entries = [{"type": "web_search"}, {"type": "custom"}, "junk", None, {}, 42, {"type": "auto"}] assert [_convert_tool_envelope(entry, to_chat=to_chat) for entry in entries] == entries + @pytest.mark.parametrize("to_chat", [True, False]) + def test_empty_nested_envelope_falls_back_to_top_level_payload(self, to_chat): + """An empty nested envelope must not shadow payload fields that sit at the top + level; treating the empty dict as the sole payload source dropped the name.""" + from litellm.proxy.response_api_endpoints.endpoints import _convert_tool_envelope + + hybrid = {"type": "custom", "custom": {}, "name": "ApplyPatch", "format": self.TEXT} + expected_payload = {"name": "ApplyPatch", "format": self.TEXT} + expected = {"type": "custom", "custom": expected_payload} if to_chat else {"type": "custom", **expected_payload} + assert _convert_tool_envelope(hybrid, to_chat=to_chat) == expected + + def test_nested_payload_wins_over_stray_top_level_fields(self): + from litellm.proxy.response_api_endpoints.endpoints import _convert_tool_envelope + + tool = {"type": "custom", "custom": {"name": "NestedName"}, "name": "TopName"} + assert _convert_tool_envelope(tool, to_chat=False) == {"type": "custom", "name": "NestedName"} + + @pytest.mark.parametrize("to_chat", [True, False]) + def test_nameless_envelope_passes_through_unchanged(self, to_chat): + from litellm.proxy.response_api_endpoints.endpoints import _convert_tool_envelope + + nameless = {"type": "custom", "custom": {}, "description": "no name anywhere"} + assert _convert_tool_envelope(nameless, to_chat=to_chat) == nameless + class TestToolChoiceSharesTheToolEnvelopeRule: """ diff --git a/tests/test_litellm/types/test_types_utils.py b/tests/test_litellm/types/test_types_utils.py index 4d08239360f..a446f820870 100644 --- a/tests/test_litellm/types/test_types_utils.py +++ b/tests/test_litellm/types/test_types_utils.py @@ -638,6 +638,48 @@ def test_chat_completion_tool_call_from_dict_custom_strips_null_function(): assert "function" not in parsed.model_dump() +def test_chat_completion_tool_call_from_dict_typeless_custom_payload(): + """A tool-call dict can carry a ``custom`` payload with ``type`` absent or None + (e.g. rebuilt from streaming deltas, where only the first chunk has ``type``). + Classifying on ``type == "custom"`` alone sent these to the function branch, + which raised TypeError (missing ``function``) on a payload the streaming path + accepts as custom.""" + from litellm.types.utils import ChatCompletionMessageCustomToolCall, chat_completion_tool_call_from_dict + + typeless = {"id": "call_1", "custom": {"name": "ApplyPatch", "input": "*** Begin Patch"}} + parsed = chat_completion_tool_call_from_dict(typeless) + assert isinstance(parsed, ChatCompletionMessageCustomToolCall) + assert parsed.type == "custom" + assert parsed.custom.name == "ApplyPatch" + + null_typed = {"id": "call_2", "type": None, "custom": {"name": "f", "input": "{}"}} + assert isinstance(chat_completion_tool_call_from_dict(null_typed), ChatCompletionMessageCustomToolCall) + + +def test_custom_tool_call_classification_agrees_across_streaming_and_non_streaming(): + """The streaming Delta coercion and the non-streaming from_dict parser must + classify the same tool-call dict identically, or a provider payload becomes a + custom tool call mid-stream and something else on the completed message.""" + from litellm.types.utils import ( + ChatCompletionDeltaCustomToolCall, + ChatCompletionMessageCustomToolCall, + Delta, + chat_completion_tool_call_from_dict, + ) + + tool_calls = [ + {"id": "c1", "type": "custom", "custom": {"name": "ApplyPatch", "input": ""}}, + {"id": "c2", "custom": {"name": "ApplyPatch", "input": "x"}}, + {"id": "c3", "type": "function", "function": {"name": "g", "arguments": "{}"}}, + ] + for tool_call in tool_calls: + message_parsed = chat_completion_tool_call_from_dict(dict(tool_call)) + delta_parsed = Delta(tool_calls=[dict(tool_call, index=0)]).tool_calls[0] + assert isinstance(message_parsed, ChatCompletionMessageCustomToolCall) == isinstance( + delta_parsed, ChatCompletionDeltaCustomToolCall + ) + + def test_message_with_mixed_function_and_custom_tool_calls(): from litellm.types.utils import ( ChatCompletionMessageCustomToolCall, From 4d43080a74e7e9d5ed61e616e2a5f08bb9da7301 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Sat, 1 Aug 2026 12:34:25 -0700 Subject: [PATCH 029/124] fix(pricing): apply OpenAI's gpt-5.6 terra/luna cut to Azure cost map OpenAI cut Terra 20% and Luna 80% on 2026-07-30; openai and bedrock_mantle entries already match. Azure global and us/eu data-zone terra/luna rows still used the pre-cut rates, so spend tracking over-billed those Azure deployments. Sol is unchanged. Cache-read, priority, and long-context fields scale with the same multipliers already used for azure gpt-5.6. --- ...odel_prices_and_context_window_backup.json | 120 +++++++++--------- model_prices_and_context_window.json | 120 +++++++++--------- .../llm_cost_calc/test_llm_cost_calc_utils.py | 8 +- 3 files changed, 124 insertions(+), 124 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 07f04136bd0..ebb1290ca57 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6425,23 +6425,23 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.6-terra": { - "cache_read_input_token_cost": 2.5e-07, - "cache_read_input_token_cost_above_272k_tokens": 5e-07, - "cache_read_input_token_cost_priority": 5e-07, - "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, - "input_cost_per_token": 2.5e-06, - "input_cost_per_token_above_272k_tokens": 5e-06, - "input_cost_per_token_priority": 5e-06, - "input_cost_per_token_above_272k_tokens_priority": 1e-05, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "cache_read_input_token_cost_priority": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_272k_tokens": 4e-06, + "input_cost_per_token_priority": 4e-06, + "input_cost_per_token_above_272k_tokens_priority": 8e-06, "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.5e-05, - "output_cost_per_token_above_272k_tokens": 2.25e-05, - "output_cost_per_token_priority": 3e-05, - "output_cost_per_token_above_272k_tokens_priority": 4.5e-05, + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_above_272k_tokens": 1.8e-05, + "output_cost_per_token_priority": 2.4e-05, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6470,23 +6470,23 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.6-luna": { - "cache_read_input_token_cost": 1e-07, - "cache_read_input_token_cost_above_272k_tokens": 2e-07, - "cache_read_input_token_cost_priority": 2e-07, - "cache_read_input_token_cost_above_272k_tokens_priority": 4e-07, - "input_cost_per_token": 1e-06, - "input_cost_per_token_above_272k_tokens": 2e-06, - "input_cost_per_token_priority": 2e-06, - "input_cost_per_token_above_272k_tokens_priority": 4e-06, + "cache_read_input_token_cost": 2e-08, + "cache_read_input_token_cost_above_272k_tokens": 4e-08, + "cache_read_input_token_cost_priority": 4e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, + "input_cost_per_token": 2e-07, + "input_cost_per_token_above_272k_tokens": 4e-07, + "input_cost_per_token_priority": 4e-07, + "input_cost_per_token_above_272k_tokens_priority": 8e-07, "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 6e-06, - "output_cost_per_token_above_272k_tokens": 9e-06, - "output_cost_per_token_priority": 1.2e-05, - "output_cost_per_token_above_272k_tokens_priority": 1.8e-05, + "output_cost_per_token": 1.2e-06, + "output_cost_per_token_above_272k_tokens": 1.8e-06, + "output_cost_per_token_priority": 2.4e-06, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6599,20 +6599,20 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6-terra": { - "cache_read_input_token_cost": 2.75e-07, - "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, - "cache_read_input_token_cost_priority": 6.875e-07, - "input_cost_per_token": 2.75e-06, - "input_cost_per_token_above_272k_tokens": 5.5e-06, - "input_cost_per_token_priority": 6.875e-06, + "cache_read_input_token_cost": 2.2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4.4e-07, + "cache_read_input_token_cost_priority": 5.5e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_above_272k_tokens": 4.4e-06, + "input_cost_per_token_priority": 5.5e-06, "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.65e-05, - "output_cost_per_token_above_272k_tokens": 2.475e-05, - "output_cost_per_token_priority": 4.125e-05, + "output_cost_per_token": 1.32e-05, + "output_cost_per_token_above_272k_tokens": 1.98e-05, + "output_cost_per_token_priority": 3.3e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6641,20 +6641,20 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6-luna": { - "cache_read_input_token_cost": 1.1e-07, - "cache_read_input_token_cost_above_272k_tokens": 2.2e-07, - "cache_read_input_token_cost_priority": 2.75e-07, - "input_cost_per_token": 1.1e-06, - "input_cost_per_token_above_272k_tokens": 2.2e-06, - "input_cost_per_token_priority": 2.75e-06, + "cache_read_input_token_cost": 2.2e-08, + "cache_read_input_token_cost_above_272k_tokens": 4.4e-08, + "cache_read_input_token_cost_priority": 5.5e-08, + "input_cost_per_token": 2.2e-07, + "input_cost_per_token_above_272k_tokens": 4.4e-07, + "input_cost_per_token_priority": 5.5e-07, "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 6.6e-06, - "output_cost_per_token_above_272k_tokens": 9.9e-06, - "output_cost_per_token_priority": 1.65e-05, + "output_cost_per_token": 1.32e-06, + "output_cost_per_token_above_272k_tokens": 1.98e-06, + "output_cost_per_token_priority": 3.3e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6767,20 +6767,20 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6-terra": { - "cache_read_input_token_cost": 2.75e-07, - "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, - "cache_read_input_token_cost_priority": 6.875e-07, - "input_cost_per_token": 2.75e-06, - "input_cost_per_token_above_272k_tokens": 5.5e-06, - "input_cost_per_token_priority": 6.875e-06, + "cache_read_input_token_cost": 2.2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4.4e-07, + "cache_read_input_token_cost_priority": 5.5e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_above_272k_tokens": 4.4e-06, + "input_cost_per_token_priority": 5.5e-06, "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.65e-05, - "output_cost_per_token_above_272k_tokens": 2.475e-05, - "output_cost_per_token_priority": 4.125e-05, + "output_cost_per_token": 1.32e-05, + "output_cost_per_token_above_272k_tokens": 1.98e-05, + "output_cost_per_token_priority": 3.3e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6809,20 +6809,20 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6-luna": { - "cache_read_input_token_cost": 1.1e-07, - "cache_read_input_token_cost_above_272k_tokens": 2.2e-07, - "cache_read_input_token_cost_priority": 2.75e-07, - "input_cost_per_token": 1.1e-06, - "input_cost_per_token_above_272k_tokens": 2.2e-06, - "input_cost_per_token_priority": 2.75e-06, + "cache_read_input_token_cost": 2.2e-08, + "cache_read_input_token_cost_above_272k_tokens": 4.4e-08, + "cache_read_input_token_cost_priority": 5.5e-08, + "input_cost_per_token": 2.2e-07, + "input_cost_per_token_above_272k_tokens": 4.4e-07, + "input_cost_per_token_priority": 5.5e-07, "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 6.6e-06, - "output_cost_per_token_above_272k_tokens": 9.9e-06, - "output_cost_per_token_priority": 1.65e-05, + "output_cost_per_token": 1.32e-06, + "output_cost_per_token_above_272k_tokens": 1.98e-06, + "output_cost_per_token_priority": 3.3e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 346f613ea3e..f07f245433d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6425,23 +6425,23 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.6-terra": { - "cache_read_input_token_cost": 2.5e-07, - "cache_read_input_token_cost_above_272k_tokens": 5e-07, - "cache_read_input_token_cost_priority": 5e-07, - "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, - "input_cost_per_token": 2.5e-06, - "input_cost_per_token_above_272k_tokens": 5e-06, - "input_cost_per_token_priority": 5e-06, - "input_cost_per_token_above_272k_tokens_priority": 1e-05, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "cache_read_input_token_cost_priority": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_272k_tokens": 4e-06, + "input_cost_per_token_priority": 4e-06, + "input_cost_per_token_above_272k_tokens_priority": 8e-06, "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.5e-05, - "output_cost_per_token_above_272k_tokens": 2.25e-05, - "output_cost_per_token_priority": 3e-05, - "output_cost_per_token_above_272k_tokens_priority": 4.5e-05, + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_above_272k_tokens": 1.8e-05, + "output_cost_per_token_priority": 2.4e-05, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6470,23 +6470,23 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.6-luna": { - "cache_read_input_token_cost": 1e-07, - "cache_read_input_token_cost_above_272k_tokens": 2e-07, - "cache_read_input_token_cost_priority": 2e-07, - "cache_read_input_token_cost_above_272k_tokens_priority": 4e-07, - "input_cost_per_token": 1e-06, - "input_cost_per_token_above_272k_tokens": 2e-06, - "input_cost_per_token_priority": 2e-06, - "input_cost_per_token_above_272k_tokens_priority": 4e-06, + "cache_read_input_token_cost": 2e-08, + "cache_read_input_token_cost_above_272k_tokens": 4e-08, + "cache_read_input_token_cost_priority": 4e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, + "input_cost_per_token": 2e-07, + "input_cost_per_token_above_272k_tokens": 4e-07, + "input_cost_per_token_priority": 4e-07, + "input_cost_per_token_above_272k_tokens_priority": 8e-07, "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 6e-06, - "output_cost_per_token_above_272k_tokens": 9e-06, - "output_cost_per_token_priority": 1.2e-05, - "output_cost_per_token_above_272k_tokens_priority": 1.8e-05, + "output_cost_per_token": 1.2e-06, + "output_cost_per_token_above_272k_tokens": 1.8e-06, + "output_cost_per_token_priority": 2.4e-06, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6599,20 +6599,20 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6-terra": { - "cache_read_input_token_cost": 2.75e-07, - "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, - "cache_read_input_token_cost_priority": 6.875e-07, - "input_cost_per_token": 2.75e-06, - "input_cost_per_token_above_272k_tokens": 5.5e-06, - "input_cost_per_token_priority": 6.875e-06, + "cache_read_input_token_cost": 2.2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4.4e-07, + "cache_read_input_token_cost_priority": 5.5e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_above_272k_tokens": 4.4e-06, + "input_cost_per_token_priority": 5.5e-06, "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.65e-05, - "output_cost_per_token_above_272k_tokens": 2.475e-05, - "output_cost_per_token_priority": 4.125e-05, + "output_cost_per_token": 1.32e-05, + "output_cost_per_token_above_272k_tokens": 1.98e-05, + "output_cost_per_token_priority": 3.3e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6641,20 +6641,20 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6-luna": { - "cache_read_input_token_cost": 1.1e-07, - "cache_read_input_token_cost_above_272k_tokens": 2.2e-07, - "cache_read_input_token_cost_priority": 2.75e-07, - "input_cost_per_token": 1.1e-06, - "input_cost_per_token_above_272k_tokens": 2.2e-06, - "input_cost_per_token_priority": 2.75e-06, + "cache_read_input_token_cost": 2.2e-08, + "cache_read_input_token_cost_above_272k_tokens": 4.4e-08, + "cache_read_input_token_cost_priority": 5.5e-08, + "input_cost_per_token": 2.2e-07, + "input_cost_per_token_above_272k_tokens": 4.4e-07, + "input_cost_per_token_priority": 5.5e-07, "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 6.6e-06, - "output_cost_per_token_above_272k_tokens": 9.9e-06, - "output_cost_per_token_priority": 1.65e-05, + "output_cost_per_token": 1.32e-06, + "output_cost_per_token_above_272k_tokens": 1.98e-06, + "output_cost_per_token_priority": 3.3e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6767,20 +6767,20 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6-terra": { - "cache_read_input_token_cost": 2.75e-07, - "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, - "cache_read_input_token_cost_priority": 6.875e-07, - "input_cost_per_token": 2.75e-06, - "input_cost_per_token_above_272k_tokens": 5.5e-06, - "input_cost_per_token_priority": 6.875e-06, + "cache_read_input_token_cost": 2.2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4.4e-07, + "cache_read_input_token_cost_priority": 5.5e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_above_272k_tokens": 4.4e-06, + "input_cost_per_token_priority": 5.5e-06, "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.65e-05, - "output_cost_per_token_above_272k_tokens": 2.475e-05, - "output_cost_per_token_priority": 4.125e-05, + "output_cost_per_token": 1.32e-05, + "output_cost_per_token_above_272k_tokens": 1.98e-05, + "output_cost_per_token_priority": 3.3e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6809,20 +6809,20 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6-luna": { - "cache_read_input_token_cost": 1.1e-07, - "cache_read_input_token_cost_above_272k_tokens": 2.2e-07, - "cache_read_input_token_cost_priority": 2.75e-07, - "input_cost_per_token": 1.1e-06, - "input_cost_per_token_above_272k_tokens": 2.2e-06, - "input_cost_per_token_priority": 2.75e-06, + "cache_read_input_token_cost": 2.2e-08, + "cache_read_input_token_cost_above_272k_tokens": 4.4e-08, + "cache_read_input_token_cost_priority": 5.5e-08, + "input_cost_per_token": 2.2e-07, + "input_cost_per_token_above_272k_tokens": 4.4e-07, + "input_cost_per_token_priority": 5.5e-07, "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 6.6e-06, - "output_cost_per_token_above_272k_tokens": 9.9e-06, - "output_cost_per_token_priority": 1.65e-05, + "output_cost_per_token": 1.32e-06, + "output_cost_per_token_above_272k_tokens": 1.98e-06, + "output_cost_per_token_priority": 3.3e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index fbdf9b64bc0..9145e5dc76d 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -757,11 +757,11 @@ def test_generic_cost_per_token_gpt56_terra_cache_costs_by_tier_and_context( [ ("azure/gpt-5.6", 5e-6, 3e-5, 5e-7), ("azure/gpt-5.6-sol", 5e-6, 3e-5, 5e-7), - ("azure/gpt-5.6-terra", 2.5e-6, 1.5e-5, 2.5e-7), - ("azure/gpt-5.6-luna", 1e-6, 6e-6, 1e-7), + ("azure/gpt-5.6-terra", 2e-6, 1.2e-5, 2e-7), + ("azure/gpt-5.6-luna", 2e-7, 1.2e-6, 2e-8), ("azure/us/gpt-5.6", 5.5e-6, 3.3e-5, 5.5e-7), - ("azure/eu/gpt-5.6-terra", 2.75e-6, 1.65e-5, 2.75e-7), - ("azure/eu/gpt-5.6-luna", 1.1e-6, 6.6e-6, 1.1e-7), + ("azure/eu/gpt-5.6-terra", 2.2e-6, 1.32e-5, 2.2e-7), + ("azure/eu/gpt-5.6-luna", 2.2e-7, 1.32e-6, 2.2e-8), ], ) def test_generic_cost_per_token_azure_gpt56( From 0de901abf79da2481e343a87c2e1e4a0579120a3 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Sat, 1 Aug 2026 12:43:41 -0700 Subject: [PATCH 030/124] chore: drop fork-only GHCR publish workflow from this branch That workflow is fork-local for Concourse and does not belong in the Azure pricing PR against BerriAI staging --- .github/workflows/publish-ghcr.yml | 129 ----------------------------- 1 file changed, 129 deletions(-) delete mode 100644 .github/workflows/publish-ghcr.yml diff --git a/.github/workflows/publish-ghcr.yml b/.github/workflows/publish-ghcr.yml deleted file mode 100644 index 7530e85116a..00000000000 --- a/.github/workflows/publish-ghcr.yml +++ /dev/null @@ -1,129 +0,0 @@ -# Build and push LiteLLM images to THIS fork's GHCR. -name: Publish GHCR (fork) - -on: - workflow_dispatch: - inputs: - image_tag: - description: Primary image tag (e.g. dev, rc, short sha) - required: true - type: string - default: dev - git_ref: - description: Git ref to build. Empty uses the branch the workflow runs on. - required: false - type: string - default: "" - variants: - description: "Comma-separated: litellm,database,non_root" - required: false - type: string - default: litellm - dry_run: - description: Build only; skip push - required: false - type: boolean - default: false - -permissions: - contents: read - packages: write - -concurrency: - group: publish-ghcr-${{ github.event.inputs.image_tag }} - cancel-in-progress: false - -jobs: - publish: - name: Build and push ${{ matrix.name }} - runs-on: ubuntu-latest - timeout-minutes: 180 - strategy: - fail-fast: false - matrix: - include: - - name: litellm - dockerfile: Dockerfile - image_suffix: litellm - - name: database - dockerfile: docker/Dockerfile.database - image_suffix: litellm-database - - name: non_root - dockerfile: docker/Dockerfile.non_root - image_suffix: litellm-non_root - steps: - - name: Select variant - id: pick - shell: bash - run: | - set -euo pipefail - wanted="${{ github.event.inputs.variants }}" - name="${{ matrix.name }}" - if [[ ",${wanted}," == *",${name},"* ]] || [[ "${wanted}" == "${name}" ]]; then - echo "run=true" >> "$GITHUB_OUTPUT" - else - echo "run=false" >> "$GITHUB_OUTPUT" - fi - - - name: Checkout - if: steps.pick.outputs.run == 'true' - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.git_ref != '' && github.event.inputs.git_ref || github.ref }} - fetch-depth: 1 - - - name: Set up Docker Buildx - if: steps.pick.outputs.run == 'true' - uses: docker/setup-buildx-action@v3 - - - name: Log in to GHCR - if: steps.pick.outputs.run == 'true' && github.event.inputs.dry_run != 'true' - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Image metadata - if: steps.pick.outputs.run == 'true' - id: meta - shell: bash - run: | - set -euo pipefail - owner="${GITHUB_REPOSITORY_OWNER,,}" - tag="${{ github.event.inputs.image_tag }}" - sha="$(git rev-parse --short HEAD)" - image="ghcr.io/${owner}/${{ matrix.image_suffix }}" - { - echo "image=${image}" - echo "tags=${image}:${tag},${image}:${sha}" - echo "sha=${sha}" - } >> "$GITHUB_OUTPUT" - echo "Will publish: ${image}:${tag} and ${image}:${sha}" - - - name: Build and push - if: steps.pick.outputs.run == 'true' - uses: docker/build-push-action@v6 - with: - context: . - file: ${{ matrix.dockerfile }} - push: ${{ github.event.inputs.dry_run != 'true' }} - tags: ${{ steps.meta.outputs.tags }} - platforms: linux/amd64 - provenance: false - sbom: false - cache-from: type=gha,scope=${{ matrix.name }} - cache-to: type=gha,mode=max,scope=${{ matrix.name }} - - - name: Summary - if: steps.pick.outputs.run == 'true' - shell: bash - run: | - { - echo "### ${{ matrix.name }}" - echo "" - echo "- image: \`${{ steps.meta.outputs.image }}\`" - echo "- tags: \`${{ steps.meta.outputs.tags }}\`" - echo "- dry_run: \`${{ github.event.inputs.dry_run }}\`" - echo "- sha: \`${{ steps.meta.outputs.sha }}\`" - } >> "$GITHUB_STEP_SUMMARY" From fd0809861cea366faf1830c529e86511ff469d0c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:48:11 -0700 Subject: [PATCH 031/124] fix(lint): restore group-header comments the import sort displaced in _lazy_imports.py --- litellm/_lazy_imports.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index 8f9cd74f171..4eee525f6a4 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -23,6 +23,7 @@ from typing import Any, cast # Import all the data structures that define what can be lazy-loaded # These are just lists of names and maps of where to find them from ._lazy_imports_registry import ( + # Import maps _BEDROCK_TYPES_IMPORT_MAP, _CACHING_IMPORT_MAP, _COST_CALCULATOR_IMPORT_MAP, @@ -33,12 +34,11 @@ from ._lazy_imports_registry import ( _TOKEN_COUNTER_IMPORT_MAP, _TYPES_IMPORT_MAP, _TYPES_UTILS_IMPORT_MAP, - # Import maps _UTILS_IMPORT_MAP, _UTILS_MODULE_IMPORT_MAP, + # Name tuples BEDROCK_TYPES_NAMES, CACHING_NAMES, - # Name tuples COST_CALCULATOR_NAMES, DOTPROMPT_NAMES, HTTP_HANDLER_NAMES, From 075babd00fbc4ccba28506f0e700323b22440814 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:22:05 -0700 Subject: [PATCH 032/124] chore(lint): clear the new LIT001/LIT002 violations and ratchet the lint budgets The type-discipline gate flagged 17 new mutable-collection annotations and 31 new mutable-collection constructions added by this branch. Replace raw dict literals with the OpenAI SDK's TypedDict call forms, annotate read-only params as Mapping/Sequence, precompute the custom tool call id set as a frozenset, and accumulate streamed arguments as tuples. The few places where a plain list/dict is a hard contract (pydantic response fields, fastapi route tags, parsed request bodies, in-place tool call patching) carry reasoned mutable-ok suppressions instead. Ratchet the ruff, type-discipline, and basedpyright budgets down by the violations this branch now fixes on net --- basedpyright-code-budget.json | 6 +- .../transformation.py | 123 +++++++++++------- litellm/integrations/helicone.py | 31 ++--- litellm/integrations/lunary.py | 19 +-- .../convert_dict_to_response.py | 7 +- .../prompt_templates/common_utils.py | 37 ++++-- .../streaming_chunk_builder_utils.py | 25 ++-- .../llms/openai/chat/gpt_transformation.py | 6 +- litellm/main.py | 2 +- .../proxy/response_api_endpoints/endpoints.py | 69 ++++++---- .../transformation.py | 14 +- litellm/types/utils.py | 25 ++-- ruff-strict-budget.json | 10 +- type-discipline-budget.json | 6 +- 14 files changed, 227 insertions(+), 153 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index f6dd90077b1..73b9d0c8192 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -42,7 +42,7 @@ "limit": 18 }, "reportIndexIssue": { - "limit": 37 + "limit": 36 }, "reportInvalidTypeForm": { "limit": 35 @@ -114,10 +114,10 @@ "limit": 31978 }, "reportUnnecessaryCast": { - "limit": 177 + "limit": 175 }, "reportUnnecessaryComparison": { - "limit": 1021 + "limit": 1019 }, "reportUnnecessaryContains": { "limit": 7 diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 768bf6c3e66..d999c9f4e60 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -4,6 +4,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req import json import os +from collections.abc import Mapping from typing import ( TYPE_CHECKING, Any, @@ -21,6 +22,13 @@ from typing import ( ) from openai.types.responses.custom_tool_param import CustomToolParam +from openai.types.responses.response_input_param import ( + FunctionCallOutput, + ResponseCustomToolCallOutputParam, + ResponseCustomToolCallParam, +) +from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam +from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam from openai.types.responses.tool_param import FunctionToolParam from pydantic import BaseModel @@ -40,6 +48,8 @@ from litellm.responses.utils import normalize_responses_api_stream_options from litellm.types.llms.openai import ( ChatCompletionAnnotation, ChatCompletionReasoningItem, + ChatCompletionToolCallChunk, + ChatCompletionToolCallFunctionChunk, ChatCompletionToolParamFunctionChunk, Reasoning, ResponsesAPIOptionalRequestParams, @@ -101,7 +111,11 @@ def _build_reasoning_item( } -def _tool_call_dict_from_output_item(item: dict[str, Any]) -> dict[str, Any]: +class _ChatToolCallDict(ChatCompletionToolCallChunk, total=False): + provider_specific_fields: Mapping[str, Any] + + +def _tool_call_dict_from_output_item(item: Mapping[str, Any], index: int) -> _ChatToolCallDict: """Convert a ``function_call`` or ``custom_tool_call`` output item dict to a chat completions tool_call dict. Custom (grammar/freeform) tool calls carry their raw string payload in ``input`` rather than ``arguments``; both map to @@ -115,22 +129,32 @@ def _tool_call_dict_from_output_item(item: dict[str, Any]) -> dict[str, Any]: is_custom = item.get("type") == "custom_tool_call" arguments = (item.get("input") if is_custom else item.get("arguments")) or "" name = item.get("name") or ("custom_tool" if is_custom else "") - tool_call_dict: dict[str, Any] = { - "id": LiteLLMCompletionResponsesConfig._tool_call_id_from_responses_item(item.get("id"), item.get("call_id")), - "function": {"name": name, "arguments": arguments}, - "type": "function", - } - provider_specific_fields = item.get("provider_specific_fields") - if provider_specific_fields and not isinstance(provider_specific_fields, dict): - provider_specific_fields = ( - dict(provider_specific_fields) if hasattr(provider_specific_fields, "__dict__") else None - ) + function_chunk = ChatCompletionToolCallFunctionChunk(name=name, arguments=arguments) + tool_call_dict = _ChatToolCallDict( + id=LiteLLMCompletionResponsesConfig._tool_call_id_from_responses_item(item.get("id"), item.get("call_id")), + type="function", + function=function_chunk, + index=index, + ) + raw_provider_fields = item.get("provider_specific_fields") + if isinstance(raw_provider_fields, dict): + provider_specific_fields = raw_provider_fields + elif raw_provider_fields and hasattr(raw_provider_fields, "__dict__"): + provider_specific_fields = vars(raw_provider_fields) + else: + provider_specific_fields = None if provider_specific_fields: tool_call_dict["provider_specific_fields"] = provider_specific_fields - tool_call_dict["function"]["provider_specific_fields"] = provider_specific_fields + function_chunk["provider_specific_fields"] = provider_specific_fields return tool_call_dict +def _flat_responses_tool_choice(choice_type: str, name: str) -> Union[ToolChoiceFunctionParam, ToolChoiceCustomParam]: + if choice_type == "custom": + return ToolChoiceCustomParam(type="custom", name=name) + return ToolChoiceFunctionParam(type="function", name=name) + + def _reasoning_item_to_response_input( r_item: Union[ChatCompletionReasoningItem, Dict[str, Any]], ) -> Dict[str, Any]: @@ -163,12 +187,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): return tool_choice if isinstance(tool_choice.get("name"), str) and tool_choice.get("name"): # Return only Responses shape so stray chat ``function``/``custom`` keys are not sent upstream. - return {"type": choice_type, "name": tool_choice["name"]} + return _flat_responses_tool_choice(choice_type, tool_choice["name"]) nested = tool_choice.get(choice_type) if isinstance(nested, dict): nested_name = nested.get("name") if isinstance(nested_name, str) and nested_name: - return {"type": choice_type, "name": nested_name} + return _flat_responses_tool_choice(choice_type, nested_name) return tool_choice def _handle_raw_dict_response_item(self, item: Dict[str, Any], index: int) -> Tuple[Optional[Any], int]: @@ -221,7 +245,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) -> Tuple[List[Any], Optional[str]]: input_items: List[Any] = [] instructions: Optional[str] = None - custom_tool_call_ids: set = set() + custom_tool_call_ids = frozenset( + tool_call["id"] + for msg in messages + if msg.get("role") == "assistant" and isinstance(msg.get("tool_calls"), list) + for tool_call in msg.get("tool_calls") or () + if isinstance(tool_call, dict) + and not tool_call.get("function") + and isinstance(tool_call.get("custom"), dict) + ) for msg in messages: role = msg.get("role") @@ -269,19 +301,19 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): tool_output = [{"type": "input_text", "text": str(content)}] if tool_call_id in custom_tool_call_ids: input_items.append( - { - "type": "custom_tool_call_output", - "call_id": tool_call_id, - "output": content if isinstance(content, str) else tool_output, - } + ResponseCustomToolCallOutputParam( + type="custom_tool_call_output", + call_id=tool_call_id, + output=content if isinstance(content, str) else tool_output, + ) ) else: input_items.append( - { - "type": "function_call_output", - "call_id": tool_call_id, - "output": tool_output, - } + FunctionCallOutput( + type="function_call_output", + call_id=tool_call_id, + output=tool_output, + ) ) elif role == "assistant" and tool_calls and isinstance(tool_calls, list): for r_item in _get_reasoning_items(msg): @@ -300,14 +332,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): input_tool_call["arguments"] = function["arguments"] input_items.append(input_tool_call) elif isinstance(custom, dict): - custom_tool_call_ids.add(tool_call["id"]) input_items.append( - { - "type": "custom_tool_call", - "call_id": tool_call["id"], - "name": custom.get("name", ""), - "input": custom.get("input", ""), - } + ResponseCustomToolCallParam( + type="custom_tool_call", + call_id=tool_call["id"], + name=custom.get("name", ""), + input=custom.get("input", ""), + ) ) else: raise ValueError(f"tool call not supported: {tool_call}") @@ -598,7 +629,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): # Tool calls accumulate into the single trailing tool_calls choice # like the typed branches above; a choice per call would hide every # call after choices[0] from chat clients - accumulated_tool_calls.append(_tool_call_dict_from_output_item(raw_item)) + accumulated_tool_calls.append(_tool_call_dict_from_output_item(raw_item, tool_call_index)) tool_call_index += 1 elif handle_raw_dict_callback is not None: choice, index = handle_raw_dict_callback(item=raw_item, index=index) @@ -925,10 +956,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) custom_payload = tool["custom"] - flat_custom: CustomToolParam = { - "type": "custom", - "name": custom_payload.get("name", ""), - } + flat_custom = CustomToolParam(type="custom", name=custom_payload.get("name", "")) if custom_payload.get("description") is not None: flat_custom["description"] = custom_payload["description"] if isinstance(custom_payload.get("format"), dict): @@ -1130,7 +1158,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): def __init__(self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False): super().__init__(streaming_response, sync_stream, json_mode) self._chat_completion_id: str | None = None - self._tool_call_index_map: dict[int, int] = {} + self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state def _handle_string_chunk( self, str_line: Union[str, "BaseModel"] @@ -1151,7 +1179,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): @staticmethod def _sequential_tool_call_index( - tool_call_index_map: dict[int, int] | None, + tool_call_index_map: dict[int, int] | None, # mutable-ok: per-stream state, remapped in place output_index: int, ) -> int: """Chat-completions tool_call indices must be 0-based and sequential, but @@ -1170,7 +1198,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): @staticmethod def translate_responses_chunk_to_openai_stream( parsed_chunk: Union[dict, BaseModel], - tool_call_index_map: dict[int, int] | None = None, + tool_call_index_map: dict[int, int] | None = None, # mutable-ok: per-stream state, remapped in place ) -> "ModelResponseStream": """ Translate a Responses API streaming chunk to OpenAI chat completion streaming format. @@ -1229,7 +1257,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): # New output item added output_item = parsed_chunk.get("item", {}) if output_item.get("type") in ("function_call", "custom_tool_call"): - converted = _tool_call_dict_from_output_item(output_item) + converted = _tool_call_dict_from_output_item(output_item, parsed_chunk.get("output_index", 0)) provider_specific_fields = converted.get("provider_specific_fields") function_chunk = ChatCompletionToolCallFunctionChunk( @@ -1299,16 +1327,15 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): # tool call; per-stream callers already received it via # output_item.added and the argument delta events return ModelResponseStream( - choices=[ + choices=[ # mutable-ok: ModelResponseStream coerces only list choices StreamingChoices( index=0, delta=Delta( - tool_calls=[ - { - **_tool_call_dict_from_output_item(dict(output_item)), - "index": parsed_chunk.get("output_index", 0), - } - ] + tool_calls=( + _tool_call_dict_from_output_item( + output_item, parsed_chunk.get("output_index", 0) + ), + ) ), finish_reason=None, ) diff --git a/litellm/integrations/helicone.py b/litellm/integrations/helicone.py index c9346f7e6cf..5d072ad873c 100644 --- a/litellm/integrations/helicone.py +++ b/litellm/integrations/helicone.py @@ -61,24 +61,19 @@ class HeliconeLogger: for tool_call in message["tool_calls"]: function = tool_call.get("function") custom = tool_call.get("custom") - if function: - content.append( - { - "type": "tool_use", - "id": tool_call["id"], - "name": function["name"], - "input": function["arguments"], - } - ) - elif custom: - content.append( - { - "type": "tool_use", - "id": tool_call["id"], - "name": custom["name"], - "input": custom["input"], - } - ) + if not function and not custom: + continue + name, tool_input = ( + (function["name"], function["arguments"]) if function else (custom["name"], custom["input"]) + ) + content.append( + { + "type": "tool_use", + "id": tool_call["id"], + "name": name, + "input": tool_input, + } + ) elif "content" in message and message["content"]: content = [{"type": "text", "text": message["content"]}] diff --git a/litellm/integrations/lunary.py b/litellm/integrations/lunary.py index 94cb5bab8fe..02a035bc445 100644 --- a/litellm/integrations/lunary.py +++ b/litellm/integrations/lunary.py @@ -22,25 +22,18 @@ def parse_tool_calls(tool_calls): def clean_tool_call(tool_call): custom = getattr(tool_call, "custom", None) if custom is not None: - return { - "type": tool_call.type, - "id": tool_call.id, - "function": { - "name": custom.name, - "arguments": custom.input, - }, - } - serialized = { + name, arguments = custom.name, custom.input + else: + name, arguments = tool_call.function.name, tool_call.function.arguments + return { "type": tool_call.type, "id": tool_call.id, "function": { - "name": tool_call.function.name, - "arguments": tool_call.function.arguments, + "name": name, + "arguments": arguments, }, } - return serialized - return [ clean_tool_call(tool_call) for tool_call in tool_calls diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 1b23db87264..cf3937072c2 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -3,6 +3,7 @@ import json import re import time import traceback +from collections.abc import Sequence from typing import Dict, Iterable, List, Literal, Optional, Tuple, Union, cast import litellm @@ -371,7 +372,9 @@ from collections import defaultdict def _handle_invalid_parallel_tool_calls( - tool_calls: List[Union[ChatCompletionMessageToolCall, ChatCompletionMessageCustomToolCall]], + tool_calls: List[ + Union[ChatCompletionMessageToolCall, ChatCompletionMessageCustomToolCall] + ], # mutable-ok: patched in place via slice assignment ): """ Handle hallucinated parallel tool call from openai - https://community.openai.com/t/model-tries-to-call-unknown-function-multi-tool-use-parallel/490653 @@ -532,7 +535,7 @@ class LiteLLMResponseObjectHandler: def _should_convert_tool_call_to_json_mode( tool_calls: ( - list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall] | list[DatabricksTool] | None + Sequence[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall] | Sequence[DatabricksTool] | None ) = None, convert_tool_call_to_json_mode: Optional[bool] = None, ) -> bool: diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 3a7a710c6a9..52974d42b96 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -21,6 +21,14 @@ from typing import ( cast, ) +from openai.types.chat.chat_completion_custom_tool_param import ( + CustomFormatGrammar, + CustomFormatGrammarGrammar, +) +from openai.types.shared_params.custom_tool_input_format import ( + Grammar as ResponsesGrammarFormat, +) + import litellm from litellm import verbose_logger from litellm.router_utils.batch_utils import InMemoryFile @@ -1252,29 +1260,36 @@ def is_function_call(optional_params: dict) -> bool: return False -def convert_custom_tool_format_to_chat_shape(format_obj: dict) -> dict: +def convert_custom_tool_format_to_chat_shape(format_obj: Mapping[str, Any]) -> Mapping[str, Any]: """ Responses API grammar formats are flat ({"type": "grammar", "definition", "syntax"}); Chat Completions wraps the same fields in a "grammar" object. Text formats are identical on both surfaces and pass through, as does anything unrecognized. """ - if format_obj.get("type") == "grammar" and "grammar" not in format_obj: - return { - "type": "grammar", - "grammar": {k: format_obj[k] for k in ("definition", "syntax") if k in format_obj}, - } - return format_obj + if format_obj.get("type") != "grammar" or "grammar" in format_obj: + return format_obj + grammar = CustomFormatGrammarGrammar() + if "definition" in format_obj: + grammar["definition"] = format_obj["definition"] + if "syntax" in format_obj: + grammar["syntax"] = format_obj["syntax"] + return CustomFormatGrammar(type="grammar", grammar=grammar) -def convert_custom_tool_format_to_responses_shape(format_obj: dict) -> dict: +def convert_custom_tool_format_to_responses_shape(format_obj: Mapping[str, Any]) -> Mapping[str, Any]: """ Inverse of convert_custom_tool_format_to_chat_shape: unwrap the Chat Completions "grammar" object into the flat Responses API grammar shape. """ grammar = format_obj.get("grammar") - if format_obj.get("type") == "grammar" and isinstance(grammar, dict): - return {"type": "grammar", **{k: grammar[k] for k in ("definition", "syntax") if k in grammar}} - return format_obj + if format_obj.get("type") != "grammar" or not isinstance(grammar, dict): + return format_obj + flat = ResponsesGrammarFormat(type="grammar") + if "definition" in grammar: + flat["definition"] = grammar["definition"] + if "syntax" in grammar: + flat["syntax"] = grammar["syntax"] + return flat def get_file_ids_from_messages(messages: List[AllMessageValues]) -> List[str]: diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index f4f1b6fca0d..6d013718668 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -1,5 +1,6 @@ import base64 import time +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast from litellm.types.llms.openai import ( @@ -205,9 +206,13 @@ class ChunkProcessor: return response def get_combined_tool_content( - self, tool_call_chunks: List[Dict[str, Any]] - ) -> List[Union[ChatCompletionMessageToolCall, ChatCompletionMessageCustomToolCall]]: - tool_calls_list: List[Union[ChatCompletionMessageToolCall, ChatCompletionMessageCustomToolCall]] = [] + self, tool_call_chunks: Sequence[Mapping[str, Any]] + ) -> List[ + Union[ChatCompletionMessageToolCall, ChatCompletionMessageCustomToolCall] + ]: # mutable-ok: assigned verbatim to Message.tool_calls, a List field + tool_calls_list: List[ + Union[ChatCompletionMessageToolCall, ChatCompletionMessageCustomToolCall] + ] = [] # mutable-ok: see return type tool_call_map: Dict[int, Dict[str, Any]] = {} # Map to store tool calls by index for chunk in tool_call_chunks: @@ -245,9 +250,9 @@ class ChunkProcessor: "id": None, "name": None, "type": None, - "arguments": [], + "arguments": (), "custom_name": None, - "custom_input": [], + "custom_input": (), "provider_specific_fields": None, } @@ -263,20 +268,20 @@ class ChunkProcessor: if function.get("name"): tool_call_map[index]["name"] = function["name"] if function.get("arguments"): - tool_call_map[index]["arguments"].append(function["arguments"]) + tool_call_map[index]["arguments"] += (function["arguments"],) else: # function is an object if hasattr(function, "name") and function.name: tool_call_map[index]["name"] = function.name if hasattr(function, "arguments") and function.arguments: - tool_call_map[index]["arguments"].append(function.arguments) + tool_call_map[index]["arguments"] += (function.arguments,) custom = tool_call.get("custom") if isinstance(custom, dict): if custom.get("name"): tool_call_map[index]["custom_name"] = custom["name"] if custom.get("input"): - tool_call_map[index]["custom_input"].append(custom["input"]) + tool_call_map[index]["custom_input"] += (custom["input"],) else: # tool_call is an object if hasattr(tool_call, "id") and tool_call.id: @@ -287,14 +292,14 @@ class ChunkProcessor: if hasattr(tool_call.function, "name") and tool_call.function.name: tool_call_map[index]["name"] = tool_call.function.name if hasattr(tool_call.function, "arguments") and tool_call.function.arguments: - tool_call_map[index]["arguments"].append(tool_call.function.arguments) + tool_call_map[index]["arguments"] += (tool_call.function.arguments,) custom = getattr(tool_call, "custom", None) if custom is not None: if getattr(custom, "name", None): tool_call_map[index]["custom_name"] = custom.name if getattr(custom, "input", None): - tool_call_map[index]["custom_input"].append(custom.input) + tool_call_map[index]["custom_input"] += (custom.input,) # Preserve provider_specific_fields from streaming chunks provider_fields = None diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index e4492a8aba6..b6c8b7f2a07 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -533,12 +533,14 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): for choice in choices: ## HANDLE JSON MODE - anthropic returns single function call] tool_calls = choice["message"].get("tool_calls", None) - new_tool_calls: list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall] | None = None + new_tool_calls: list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall] | None = ( + None # mutable-ok: holds _handle_invalid_parallel_tool_calls' list; Message.__init__ expects list + ) message_content = choice["message"].get("content", None) if tool_calls is not None: _openai_tool_calls = [] for _tc in tool_calls: - _openai_tc = chat_completion_tool_call_from_dict(dict(_tc)) + _openai_tc = chat_completion_tool_call_from_dict(_tc) _openai_tool_calls.append(_openai_tc) fixed_tool_calls = _handle_invalid_parallel_tool_calls(_openai_tool_calls) diff --git a/litellm/main.py b/litellm/main.py index fbb43dd41fa..cadf65c3e50 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1058,7 +1058,7 @@ def responses_api_bridge_check( # summary alias is present with ``reasoning_effort`` (tools alone stay on chat). has_function_tool = any( (tool.get("type") == "function" if isinstance(tool, dict) else getattr(tool, "type", None) == "function") - for tool in (tools or []) + for tool in (tools or ()) ) if isinstance(reasoning_effort, dict): reasoning_active = reasoning_effort.get("effort") != "none" or reasoning_effort.get("summary") is not None diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 3a2c57aa2ba..2068de17785 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -1,6 +1,8 @@ import asyncio import json import time +from collections.abc import Mapping +from types import MappingProxyType from typing import Any, AsyncIterator, Dict, Optional, cast from uuid import uuid4 @@ -23,19 +25,30 @@ from litellm.types.responses.main import DeleteResponseResult router = APIRouter() _user_api_key_auth_dep = Depends(user_api_key_auth) +_RESPONSES_TAGS = ["responses"] # mutable-ok: fastapi's route signature requires List[str] tags -_TOOL_PAYLOAD_KEYS = { - "custom": ("name", "description", "format"), - "function": ("name", "description", "parameters", "strict"), -} +_TOOL_PAYLOAD_KEYS: Mapping[str, tuple[str, ...]] = MappingProxyType( + { + "custom": ("name", "description", "format"), + "function": ("name", "description", "parameters", "strict"), + } +) +_EMPTY_TOOL_PAYLOAD: Mapping[str, Any] = MappingProxyType({}) -def _convert_tool_envelope(obj: object, *, to_chat: bool) -> object: +def _convert_tool_payload_value(key: str, value: object, *, to_chat: bool) -> object: + if key != "format" or not isinstance(value, dict): + return value from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_custom_tool_format_to_chat_shape, convert_custom_tool_format_to_responses_shape, ) + convert = convert_custom_tool_format_to_chat_shape if to_chat else convert_custom_tool_format_to_responses_shape + return convert(value) + + +def _convert_tool_envelope(obj: object, *, to_chat: bool) -> object: if not isinstance(obj, dict): return obj tool_type = obj.get("type") @@ -43,35 +56,37 @@ def _convert_tool_envelope(obj: object, *, to_chat: bool) -> object: if payload_keys is None: return obj nested = obj.get(tool_type) - nested_source = nested if isinstance(nested, dict) else {} - payload = { - key: nested_source[key] if key in nested_source else obj[key] + nested_source = nested if isinstance(nested, dict) else _EMPTY_TOOL_PAYLOAD + payload = { # mutable-ok: tool entries are embedded verbatim in the JSON request body + key: _convert_tool_payload_value(key, nested_source[key] if key in nested_source else obj[key], to_chat=to_chat) for key in payload_keys if key in nested_source or key in obj } if "name" not in payload: return obj - if isinstance(payload.get("format"), dict): - convert = convert_custom_tool_format_to_chat_shape if to_chat else convert_custom_tool_format_to_responses_shape - payload = {**payload, "format": convert(payload["format"])} - return {"type": tool_type, tool_type: payload} if to_chat else {"type": tool_type, **payload} + return {"type": tool_type, tool_type: payload} if to_chat else {"type": tool_type, **payload} # mutable-ok: same -def _normalize_tool_dialect(data: dict, *, to_chat: bool) -> dict: - converted: dict = {} +def _normalize_tool_dialect( + data: dict, *, to_chat: bool +) -> dict: # mutable-ok: the parsed request body contract is a plain dict tools = data.get("tools") - if isinstance(tools, list): - normalized_tools = [_convert_tool_envelope(tool, to_chat=to_chat) for tool in tools] - if normalized_tools != tools: - converted["tools"] = normalized_tools tool_choice = data.get("tool_choice") + normalized_tools = ( + [ + _convert_tool_envelope(tool, to_chat=to_chat) for tool in tools + ] # mutable-ok: body's tools stays a plain JSON list + if isinstance(tools, list) + else tools + ) normalized_choice = _convert_tool_envelope(tool_choice, to_chat=to_chat) - if normalized_choice != tool_choice: - converted["tool_choice"] = normalized_choice - return {**data, **converted} if converted else data + if normalized_tools == tools and normalized_choice == tool_choice: + return data + replaceable = (("tools", normalized_tools), ("tool_choice", normalized_choice)) + return {**data, **{key: value for key, value in replaceable if key in data}} # mutable-ok: plain body dict -def _is_chat_completions_body(data: dict) -> bool: +def _is_chat_completions_body(data: Mapping[str, Any]) -> bool: messages = data.get("messages") if isinstance(messages, list) and len(messages) > 0: return True @@ -340,13 +355,13 @@ async def responses_api( @router.get( "/cursor/models", - dependencies=[Depends(user_api_key_auth)], - tags=["responses"], + dependencies=(_user_api_key_auth_dep,), + tags=_RESPONSES_TAGS, ) @router.get( "/cursor/v1/models", - dependencies=[Depends(user_api_key_auth)], - tags=["responses"], + dependencies=(_user_api_key_auth_dep,), + tags=_RESPONSES_TAGS, ) async def cursor_model_list( user_api_key_dict: UserAPIKeyAuth = _user_api_key_auth_dep, @@ -447,7 +462,7 @@ async def cursor_chat_completions( # Rebuild rather than pop: _read_request_body can return the request-scope # cached parsed-body dict itself, and removing keys from it corrupts the # cache's key snapshot so later readers get an empty body - data = {key: value for key, value in data.items() if key != "stream_options"} + data = {key: value for key, value in data.items() if key != "stream_options"} # mutable-ok: plain body dict data = _normalize_tool_dialect(data, to_chat=False) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 176274d236f..090723edb85 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -7,6 +7,12 @@ import re from collections.abc import Sequence from typing import Any, Literal, cast +from openai.types.chat.chat_completion_named_tool_choice_param import ( + ChatCompletionNamedToolChoiceParam, +) +from openai.types.chat.chat_completion_named_tool_choice_param import ( + Function as NamedToolChoiceFunction, +) from openai.types.responses import ResponseFunctionToolCall from openai.types.responses.response_create_params import ResponseInputParam from openai.types.responses.tool_param import FunctionToolParam @@ -160,13 +166,17 @@ class LiteLLMCompletionResponsesConfig: elif tool_choice_type == "function": function_name = tool_choice.get("name") if function_name: - return {"type": "function", "function": {"name": function_name}} + return ChatCompletionNamedToolChoiceParam( + type="function", function=NamedToolChoiceFunction(name=function_name) + ) return "required" elif tool_choice_type == "custom": custom = tool_choice.get("custom") custom_name = tool_choice.get("name") or (custom.get("name") if isinstance(custom, dict) else None) if custom_name: - return {"type": "function", "function": {"name": custom_name}} + return ChatCompletionNamedToolChoiceParam( + type="function", function=NamedToolChoiceFunction(name=custom_name) + ) return "required" # Return as-is for unknown formats diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 404725ec61b..6d051b70432 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1,6 +1,7 @@ import json import time from enum import Enum +from types import MappingProxyType from typing import ( TYPE_CHECKING, Any, @@ -1162,16 +1163,16 @@ class ChatCompletionMessageToolCall(OpenAIObject): setattr(self, key, value) -def is_custom_tool_call_dict(tool_call: dict) -> bool: +def is_custom_tool_call_dict(tool_call: Mapping[str, Any]) -> bool: return tool_call.get("type") == "custom" or tool_call.get("custom") is not None def chat_completion_tool_call_from_dict( - tool_call: dict, + tool_call: Mapping[str, Any], ) -> "ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall": if is_custom_tool_call_dict(tool_call): return ChatCompletionMessageCustomToolCall( - **{k: v for k, v in tool_call.items() if not (k in ("function", "type") and v is None)} + **MappingProxyType({k: v for k, v in tool_call.items() if not (k in ("function", "type") and v is None)}) ) return ChatCompletionMessageToolCall(**tool_call) @@ -1228,7 +1229,9 @@ def add_provider_specific_fields(object: BaseModel, provider_specific_fields: Op class Message(SafeAttributeModel, OpenAIObject): content: Optional[str] role: Literal["assistant", "user", "system", "tool", "function"] - tool_calls: Optional[List[Union[ChatCompletionMessageToolCall, ChatCompletionMessageCustomToolCall]]] + tool_calls: Optional[ + List[Union[ChatCompletionMessageToolCall, ChatCompletionMessageCustomToolCall]] + ] # mutable-ok: public pydantic response field; only the union member is new function_call: Optional[FunctionCall] audio: Optional[ChatCompletionAudioResponse] = None images: Optional[List[ImageURLListItem]] = None @@ -1352,7 +1355,9 @@ class Delta(SafeAttributeModel, OpenAIObject): content: Optional[str] role: Optional[str] function_call: Optional[FunctionCall] - tool_calls: Optional[List[Union[ChatCompletionDeltaToolCall, ChatCompletionDeltaCustomToolCall]]] + tool_calls: Optional[ + List[Union[ChatCompletionDeltaToolCall, ChatCompletionDeltaCustomToolCall]] + ] # mutable-ok: public pydantic response field; only the union member is new audio: Optional[ChatCompletionAudioResponse] images: Optional[List[ImageURLListItem]] annotations: Optional[List[ChatCompletionAnnotation]] @@ -1389,8 +1394,10 @@ class Delta(SafeAttributeModel, OpenAIObject): if function_call is not None and isinstance(function_call, dict): function_call = FunctionCall(**function_call) - if tool_calls is not None and isinstance(tool_calls, list): - coerced_tool_calls: List[Union[ChatCompletionDeltaToolCall, ChatCompletionDeltaCustomToolCall]] = [] + if tool_calls is not None and isinstance(tool_calls, (list, tuple)): + coerced_tool_calls: List[ + Union[ChatCompletionDeltaToolCall, ChatCompletionDeltaCustomToolCall] + ] = [] # mutable-ok: public Delta.tool_calls contract is a list current_index = 0 for tool_call in tool_calls: if isinstance(tool_call, dict): @@ -1400,7 +1407,9 @@ class Delta(SafeAttributeModel, OpenAIObject): if is_custom_tool_call_dict(tool_call): coerced_tool_calls.append( ChatCompletionDeltaCustomToolCall( - **{k: v for k, v in tool_call.items() if not (k == "function" and v is None)} + **MappingProxyType( + {k: v for k, v in tool_call.items() if not (k == "function" and v is None)} + ) ) ) else: diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index b8650eea7aa..4c8132ff859 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -135,7 +135,7 @@ "limit": 30 }, "PERF401": { - "limit": 142 + "limit": 141 }, "PERF402": { "limit": 9 @@ -222,7 +222,7 @@ "limit": 38 }, "RET504": { - "limit": 702 + "limit": 701 }, "RUF010": { "limit": 874 @@ -267,7 +267,7 @@ "limit": 324 }, "SIM103": { - "limit": 129 + "limit": 128 }, "SIM113": { "limit": 6 @@ -324,7 +324,7 @@ "limit": 879 }, "UP006": { - "limit": 12050 + "limit": 12045 }, "UP007": { "limit": 2526 @@ -363,6 +363,6 @@ "limit": 104 }, "UP045": { - "limit": 17793 + "limit": 17791 } } diff --git a/type-discipline-budget.json b/type-discipline-budget.json index ff037a2872e..bc28630a4e5 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 23191 + "limit": 23180 }, "LIT002": { - "limit": 27276 + "limit": 27259 }, "LIT003": { "limit": 292 @@ -24,6 +24,6 @@ "limit": 1004 }, "LIT009": { - "limit": 2467 + "limit": 2465 } } From 7b2d3440cba3160277470f7a0180098ae9b87864 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:59:05 -0700 Subject: [PATCH 033/124] refactor(logging): drop redundant !s conversion flags from f-strings --- litellm/batch_completion/main.py | 2 +- litellm/batches/main.py | 4 +- litellm/caching/caching.py | 10 +- litellm/caching/dual_cache.py | 6 +- litellm/caching/qdrant_semantic_cache.py | 2 +- litellm/caching/redis_cache.py | 22 +-- litellm/caching/redis_cluster_cache.py | 4 +- litellm/caching/redis_semantic_cache.py | 16 +- litellm/caching/valkey_semantic_cache.py | 10 +- litellm/cost_calculator.py | 8 +- litellm/exceptions.py | 2 +- litellm/experimental_mcp_client/client.py | 16 +- litellm/google_genai/adapters/handler.py | 4 +- .../SlackAlerting/batching_handler.py | 2 +- .../SlackAlerting/slack_alerting.py | 6 +- litellm/integrations/arize/arize.py | 2 +- .../azure_sentinel/azure_sentinel.py | 8 +- .../azure_storage/azure_storage.py | 20 +- litellm/integrations/cloudzero/cloudzero.py | 6 +- litellm/integrations/cloudzero/database.py | 2 +- litellm/integrations/custom_logger.py | 2 +- litellm/integrations/datadog/datadog.py | 14 +- .../datadog/datadog_cost_management.py | 4 +- .../integrations/datadog/datadog_llm_obs.py | 12 +- .../integrations/datadog/datadog_metrics.py | 6 +- litellm/integrations/dynamodb.py | 2 +- litellm/integrations/galileo.py | 2 +- litellm/integrations/gcs_bucket/gcs_bucket.py | 10 +- litellm/integrations/gcs_pubsub/pub_sub.py | 4 +- .../generic_api/generic_api_callback.py | 10 +- litellm/integrations/langfuse/langfuse.py | 2 +- .../langfuse/langfuse_prompt_management.py | 4 +- litellm/integrations/logfire_logger.py | 4 +- litellm/integrations/opik/opik.py | 10 +- litellm/integrations/posthog.py | 14 +- litellm/integrations/prometheus.py | 28 ++- litellm/integrations/prometheus_services.py | 2 +- litellm/integrations/s3.py | 6 +- litellm/integrations/s3_v2.py | 12 +- litellm/integrations/sqs.py | 8 +- .../vector_store_pre_call_hook.py | 6 +- .../websearch_interception/handler.py | 16 +- .../exception_mapping_utils.py | 6 +- litellm/litellm_core_utils/fallback_utils.py | 2 +- .../get_llm_provider_logic.py | 4 +- .../litellm_core_utils/get_model_cost_map.py | 2 +- litellm/litellm_core_utils/litellm_logging.py | 32 ++-- .../litellm_core_utils/llm_cost_calc/utils.py | 4 +- .../llm_response_utils/get_api_base.py | 2 +- litellm/litellm_core_utils/logging_utils.py | 8 +- .../prompt_templates/common_utils.py | 2 +- .../prompt_templates/factory.py | 10 +- .../litellm_core_utils/streaming_handler.py | 14 +- litellm/litellm_core_utils/token_counter.py | 2 +- litellm/llms/a2a/chat/transformation.py | 2 +- litellm/llms/anthropic/chat/transformation.py | 4 +- .../llms/anthropic/count_tokens/handler.py | 6 +- litellm/llms/azure/azure.py | 2 +- litellm/llms/azure/common_utils.py | 10 +- litellm/llms/azure_ai/agents/handler.py | 2 +- .../anthropic/count_tokens/handler.py | 6 +- .../azure_ai/vector_stores/transformation.py | 2 +- .../files/azure_blob_storage_backend.py | 4 +- .../bedrock/chat/agentcore/transformation.py | 8 +- .../bedrock/chat/converse_transformation.py | 2 +- .../chat/invoke_agent/transformation.py | 4 +- litellm/llms/bedrock/chat/invoke_handler.py | 2 +- ...mazon_twelvelabs_pegasus_transformation.py | 4 +- .../base_invoke_transformation.py | 4 +- litellm/llms/bedrock/count_tokens/handler.py | 6 +- litellm/llms/bedrock/files/handler.py | 2 +- litellm/llms/bedrock/files/transformation.py | 2 +- litellm/llms/bedrock/realtime/handler.py | 2 +- .../black_forest_labs/image_edit/handler.py | 4 +- .../image_generation/handler.py | 4 +- litellm/llms/clarifai/chat/transformation.py | 2 +- litellm/llms/codestral/completion/handler.py | 2 +- litellm/llms/custom_httpx/llm_http_handler.py | 6 +- .../llms/dashscope/embed/transformation.py | 2 +- .../llms/databricks/chat/transformation.py | 2 +- litellm/llms/databricks/common_utils.py | 2 +- .../audio_transcription/transformation.py | 2 +- .../audio_transcription/transformation.py | 2 +- .../llms/fireworks_ai/chat/transformation.py | 2 +- .../fireworks_ai/rerank/transformation.py | 2 +- litellm/llms/gdc/chat/transformation.py | 2 +- litellm/llms/gemini/count_tokens/handler.py | 4 +- litellm/llms/gemini/files/transformation.py | 12 +- .../gemini/vector_stores/transformation.py | 4 +- litellm/llms/gigachat/authenticator.py | 4 +- litellm/llms/github_copilot/authenticator.py | 40 ++-- litellm/llms/huggingface/common_utils.py | 2 +- litellm/llms/langgraph/chat/sse_iterator.py | 4 +- litellm/llms/langgraph/chat/transformation.py | 6 +- .../litellm_proxy/skills/code_execution.py | 2 +- litellm/llms/manus/files/transformation.py | 4 +- .../milvus/vector_stores/transformation.py | 2 +- .../minimax/text_to_speech/transformation.py | 6 +- litellm/llms/mistral/chat/transformation.py | 2 +- litellm/llms/oci/chat/cohere.py | 4 +- litellm/llms/oci/chat/generic.py | 4 +- litellm/llms/oci/chat/transformation.py | 2 +- litellm/llms/oci/common_utils.py | 2 +- litellm/llms/ollama/chat/transformation.py | 2 +- .../llms/ollama/completion/transformation.py | 2 +- .../llms/openai/chat/gpt_transformation.py | 2 +- litellm/llms/openai/openai.py | 8 +- litellm/llms/openai/realtime/handler.py | 2 +- .../openai/responses/count_tokens/handler.py | 6 +- .../openrouter/image_edit/transformation.py | 4 +- .../image_generation/transformation.py | 4 +- litellm/llms/predibase/chat/handler.py | 2 +- litellm/llms/sagemaker/completion/handler.py | 2 +- .../sagemaker/embedding/transformation.py | 2 +- .../vertex_ai/agent_engine/transformation.py | 6 +- litellm/llms/vertex_ai/common_utils.py | 2 +- litellm/llms/vertex_ai/cost_calculator.py | 4 +- .../llms/vertex_ai/files/transformation.py | 2 +- .../llms/vertex_ai/gemini/transformation.py | 6 +- .../vertex_and_google_ai_studio_gemini.py | 4 +- .../llama3/transformation.py | 2 +- litellm/llms/vertex_ai/vertex_llm_base.py | 6 +- .../volcengine/embedding/transformation.py | 2 +- .../audio_transcription/transformation.py | 2 +- litellm/llms/watsonx/rerank/transformation.py | 2 +- litellm/main.py | 4 +- .../mcp_server/auth/user_api_key_auth_mcp.py | 52 +++--- litellm/proxy/_experimental/mcp_server/db.py | 2 +- .../mcp_server/elicitation_handler.py | 2 +- .../mcp_server/mcp_server_manager.py | 50 ++--- .../mcp_server/rest_endpoints.py | 16 +- .../mcp_server/sampling_handler.py | 2 +- .../proxy/_experimental/mcp_server/server.py | 38 ++-- .../_experimental/mcp_server/toolset_db.py | 2 +- litellm/proxy/a2a/discovery.py | 6 +- litellm/proxy/a2a/endpoints.py | 2 +- .../proxy/agent_endpoints/a2a_endpoints.py | 6 +- .../proxy/agent_endpoints/agent_registry.py | 14 +- .../auth/agent_permission_handler.py | 12 +- litellm/proxy/agent_endpoints/endpoints.py | 4 +- .../claude_code_marketplace.py | 4 +- .../proxy/anthropic_endpoints/endpoints.py | 10 +- litellm/proxy/auth/auth_checks.py | 8 +- litellm/proxy/auth/auth_exception_handler.py | 4 +- litellm/proxy/auth/auth_utils.py | 6 +- litellm/proxy/auth/handle_jwt.py | 8 +- litellm/proxy/auth/litellm_license.py | 6 +- litellm/proxy/auth/user_api_key_auth.py | 6 +- litellm/proxy/batches_endpoints/endpoints.py | 8 +- litellm/proxy/caching_routes.py | 10 +- litellm/proxy/client/cli/commands/chat.py | 2 +- .../proxy/client/cli/commands/credentials.py | 2 +- litellm/proxy/client/cli/commands/keys.py | 6 +- litellm/proxy/client/cli/commands/teams.py | 6 +- litellm/proxy/common_request_processing.py | 8 +- .../proxy/common_utils/custom_openapi_spec.py | 8 +- litellm/proxy/common_utils/debug_utils.py | 4 +- .../common_utils/encrypt_decrypt_utils.py | 2 +- .../proxy/common_utils/http_parsing_utils.py | 12 +- .../proxy/common_utils/load_config_utils.py | 12 +- .../db_transaction_queue/spend_log_cleanup.py | 2 +- .../proxy/fine_tuning_endpoints/endpoints.py | 14 +- .../proxy/guardrails/guardrail_endpoints.py | 8 +- .../guardrail_hooks/bedrock_guardrails.py | 2 +- .../guardrail_hooks/custom_code/primitives.py | 4 +- .../guardrail_hooks/deepkeep/deepkeep.py | 2 +- .../generic_guardrail_api.py | 2 +- .../guardrails/guardrail_hooks/lasso/lasso.py | 8 +- .../litellm_content_filter/content_filter.py | 2 +- .../litellm_content_filter/patterns.py | 2 +- .../guardrails/guardrail_hooks/noma/noma.py | 24 +-- .../guardrails/guardrail_hooks/onyx/onyx.py | 4 +- .../guardrail_hooks/ovalix/ovalix.py | 2 +- .../panw_prisma_airs/panw_prisma_airs.py | 14 +- .../guardrail_hooks/pillar/pillar.py | 2 +- .../guardrails/guardrail_hooks/presidio.py | 8 +- .../prompt_security/prompt_security.py | 10 +- .../zscaler_ai_guard/zscaler_ai_guard.py | 2 +- .../proxy/guardrails/guardrail_registry.py | 12 +- litellm/proxy/guardrails/init_guardrails.py | 2 +- .../health_endpoints/_health_endpoints.py | 20 +- litellm/proxy/hooks/azure_content_safety.py | 2 +- litellm/proxy/hooks/batch_rate_limiter.py | 8 +- litellm/proxy/hooks/batch_redis_get.py | 2 +- litellm/proxy/hooks/cache_control_check.py | 2 +- litellm/proxy/hooks/dynamic_rate_limiter.py | 6 +- .../proxy/hooks/dynamic_rate_limiter_v3.py | 8 +- litellm/proxy/hooks/litellm_skills/main.py | 4 +- litellm/proxy/hooks/max_budget_limiter.py | 2 +- .../proxy/hooks/parallel_request_limiter.py | 2 +- .../hooks/parallel_request_limiter_v3.py | 22 +-- .../proxy/hooks/prompt_injection_detection.py | 2 +- .../proxy/hooks/proxy_track_cost_callback.py | 2 +- .../hooks/user_management_event_hooks.py | 2 +- litellm/proxy/image_endpoints/endpoints.py | 4 +- .../cache_settings_endpoints.py | 14 +- .../common_daily_activity.py | 8 +- .../cost_tracking_settings.py | 14 +- .../customer_endpoints.py | 12 +- .../fallback_management_endpoints.py | 12 +- .../internal_user_endpoints.py | 30 +-- .../key_management_endpoints.py | 30 ++- .../management_v1/budgets.py | 2 +- .../management_v1/spend_logs.py | 2 +- .../mcp_management_endpoints.py | 24 +-- ...model_access_group_management_endpoints.py | 24 +-- .../model_management_endpoints.py | 30 +-- .../organization_endpoints.py | 2 +- .../router_settings_endpoints.py | 4 +- .../tag_management_endpoints.py | 8 +- .../team_callback_endpoints.py | 8 +- .../management_endpoints/team_endpoints.py | 10 +- litellm/proxy/management_endpoints/ui_sso.py | 6 +- .../user_agent_analytics_endpoints.py | 14 +- litellm/proxy/ocr_endpoints/endpoints.py | 2 +- .../openai_files_endpoints/files_endpoints.py | 22 +-- .../llm_passthrough_endpoints.py | 6 +- .../anthropic_passthrough_logging_handler.py | 2 +- .../assembly_passthrough_logging_handler.py | 4 +- .../openai_passthrough_logging_handler.py | 10 +- .../vertex_passthrough_logging_handler.py | 2 +- .../pass_through_endpoints.py | 8 +- .../streaming_handler.py | 6 +- .../policy_engine/attachment_registry.py | 14 +- litellm/proxy/policy_engine/init_policies.py | 4 +- .../proxy/policy_engine/policy_registry.py | 28 +-- .../proxy/policy_engine/policy_validator.py | 10 +- litellm/proxy/prompts/prompt_endpoints.py | 2 +- litellm/proxy/proxy_server.py | 172 +++++++++--------- .../public_endpoints/public_endpoints.py | 2 +- litellm/proxy/rerank_endpoints/endpoints.py | 4 +- .../proxy/response_api_endpoints/endpoints.py | 4 +- .../response_polling/background_streaming.py | 2 +- litellm/proxy/search_endpoints/endpoints.py | 2 +- .../search_endpoints/search_tool_registry.py | 24 +-- .../spend_tracking/cloudzero_endpoints.py | 28 +-- .../spend_management_endpoints.py | 16 +- .../proxy/spend_tracking/vantage_endpoints.py | 28 +-- litellm/proxy/types_utils/utils.py | 4 +- .../proxy_setting_endpoints.py | 2 +- litellm/proxy/utils.py | 18 +- .../management_endpoints.py | 14 +- .../vertex_ai_endpoints/langfuse_endpoints.py | 4 +- litellm/rerank_api/main.py | 2 +- .../streaming_iterator.py | 12 +- .../mcp/litellm_proxy_mcp_handler.py | 12 +- litellm/router.py | 54 +++--- .../router_strategy/base_routing_strategy.py | 6 +- litellm/router_strategy/budget_limiter.py | 6 +- litellm/router_strategy/lowest_cost.py | 4 +- litellm/router_strategy/lowest_latency.py | 6 +- litellm/router_strategy/lowest_tpm_rpm.py | 4 +- litellm/router_strategy/lowest_tpm_rpm_v2.py | 4 +- litellm/router_utils/cooldown_cache.py | 4 +- .../router_utils/fallback_event_handlers.py | 4 +- .../router_utils/pattern_match_deployments.py | 2 +- .../pre_call_checks/model_rate_limit_check.py | 8 +- litellm/router_utils/search_api_router.py | 4 +- litellm/secret_managers/main.py | 4 +- .../secret_managers/secret_manager_handler.py | 6 +- litellm/utils.py | 22 +-- .../vector_stores/vector_store_registry.py | 6 +- 262 files changed, 1048 insertions(+), 1082 deletions(-) diff --git a/litellm/batch_completion/main.py b/litellm/batch_completion/main.py index 792be3ff7ad..fb892789b15 100644 --- a/litellm/batch_completion/main.py +++ b/litellm/batch_completion/main.py @@ -249,7 +249,7 @@ def batch_completion_models_all_responses(*args, **kwargs): if result is not None: responses.append(result) except Exception as e: - print_verbose(f"batch_completion_models_all_responses: model request failed: {e!s}") + print_verbose(f"batch_completion_models_all_responses: model request failed: {e}") continue return responses diff --git a/litellm/batches/main.py b/litellm/batches/main.py index b27939be8bf..3a057d41744 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -182,7 +182,7 @@ def create_batch( ) except Exception as e: verbose_logger.exception( - f"litellm.batches.main.py::create_batch() - Error inferring custom_llm_provider - {e!s}" + f"litellm.batches.main.py::create_batch() - Error inferring custom_llm_provider - {e}" ) _is_async = kwargs.pop("acreate_batch", False) is True @@ -890,7 +890,7 @@ def cancel_batch( ) except Exception as e: verbose_logger.exception( - f"litellm.batches.main.py::cancel_batch() - Error inferring custom_llm_provider - {e!s}" + f"litellm.batches.main.py::cancel_batch() - Error inferring custom_llm_provider - {e}" ) optional_params = GenericLiteLLMParams(**kwargs) litellm_params = get_litellm_params( diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index f69c2fa3b58..9542be0999a 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -353,13 +353,13 @@ class Cache: if param in combined_kwargs: param_value: str | None = self._get_param_value(param, kwargs) if param_value is not None: - cache_key += f"{param!s}: {param_value!s}" + cache_key += f"{param}: {param_value}" elif param not in litellm_param_kwargs: # check if user passed in optional param - e.g. top_k if litellm.enable_caching_on_provider_specific_optional_params is True: # feature flagged for now if kwargs[param] is None: continue # ignore None params param_value = kwargs[param] - cache_key += f"{param!s}: {param_value!s}" + cache_key += f"{param}: {param_value}" if is_semantic_cache: cache_key += self._get_semantic_cache_tenant_scope(kwargs) @@ -676,7 +676,7 @@ class Cache: cache_key, cached_data, kwargs = self._add_cache_logic(result=result, **kwargs) self.cache.set_cache(cache_key, cached_data, **kwargs) except Exception as e: - verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {e!s}") + verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {e}") async def async_add_cache(self, result, dynamic_cache_object: BaseCache | None = None, **kwargs): """ @@ -695,7 +695,7 @@ class Cache: else: await self.cache.async_set_cache(cache_key, cached_data, **kwargs) except Exception as e: - verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {e!s}") + verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {e}") def _convert_to_cached_embedding( self, @@ -874,7 +874,7 @@ class Cache: else: await self.cache.async_set_cache_pipeline(cache_list=cache_list, **kwargs) except Exception as e: - verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {e!s}") + verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {e}") def should_use_cache(self, **kwargs): """ diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index 5b56789e8db..b641c600a0e 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -147,7 +147,7 @@ class DualCache(BaseCache): return result except Exception as e: - verbose_logger.error(f"LiteLLM Cache: Excepton async add_cache: {e!s}") + verbose_logger.error(f"LiteLLM Cache: Excepton async add_cache: {e}") raise e def get_cache( @@ -347,7 +347,7 @@ class DualCache(BaseCache): if self.redis_cache is not None and local_only is False: await self.redis_cache.async_set_cache(key, value, **kwargs) except Exception as e: - verbose_logger.exception(f"LiteLLM Cache: Excepton async add_cache: {e!s}") + verbose_logger.exception(f"LiteLLM Cache: Excepton async add_cache: {e}") # async_batch_set_cache async def async_set_cache_pipeline(self, cache_list: list, local_only: bool = False, **kwargs): @@ -366,7 +366,7 @@ class DualCache(BaseCache): cache_list=cache_list, ttl=kwargs.pop("ttl", None), **kwargs ) except Exception as e: - verbose_logger.exception(f"LiteLLM Cache: Excepton async add_cache: {e!s}") + verbose_logger.exception(f"LiteLLM Cache: Excepton async add_cache: {e}") async def async_increment_cache( self, diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index 6e36dfbc096..98fd9cfd1d2 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -178,7 +178,7 @@ class QdrantSemanticCache(BaseCache): if response.status_code not in (200, 201): print_verbose(f"Qdrant semantic-cache could not create cache-key payload index: {response.text}") except Exception as exc: - print_verbose(f"Qdrant semantic-cache could not create cache-key payload index: {exc!s}") + print_verbose(f"Qdrant semantic-cache could not create cache-key payload index: {exc}") def _payload_matches_cache_key(self, payload: dict, key: str) -> bool: # Pre-isolation points stored only prompt + response with no cache-key diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 1b0aa778f4e..e3c0e3616f0 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -346,7 +346,7 @@ class RedisCache(BaseCache): verbose_logger.debug("Ignoring async redis ping. No running event loop.") else: verbose_logger.error( - f"Error connecting to Async Redis client - {e!s}", + f"Error connecting to Async Redis client - {e}", extra={"error": str(e)}, ) self._handle_async_ping_error(e) @@ -483,7 +483,7 @@ class RedisCache(BaseCache): ) except Exception as e: # NON blocking - notify users Redis is throwing an exception - print_verbose(f"litellm.caching.caching: set() - Got exception from REDIS : {e!s}") + print_verbose(f"litellm.caching.caching: set() - Got exception from REDIS : {e}") def increment_cache(self, key, value: int, ttl: float | None = None, **kwargs) -> int: _redis_client = self.redis_client @@ -1139,7 +1139,7 @@ class RedisCache(BaseCache): return decoded_results except Exception as e: - verbose_logger.error(f"Error occurred in batch get cache - {e!s}") + verbose_logger.error(f"Error occurred in batch get cache - {e}") return key_value_dict @_redis_circuit_breaker_guard @@ -1185,7 +1185,7 @@ class RedisCache(BaseCache): event_metadata={"key": key}, ) ) - print_verbose(f"litellm.caching.caching: async get() - Got exception from REDIS: {e!s}") + print_verbose(f"litellm.caching.caching: async get() - Got exception from REDIS: {e}") _record_swallowed_redis_failure(self._circuit_breaker, e) @_redis_circuit_breaker_guard @@ -1257,7 +1257,7 @@ class RedisCache(BaseCache): parent_otel_span=parent_otel_span, ) ) - verbose_logger.error(f"Error occurred in async batch get cache - {e!s}") + verbose_logger.error(f"Error occurred in async batch get cache - {e}") _record_swallowed_redis_failure(self._circuit_breaker, e) return key_value_dict @@ -1292,7 +1292,7 @@ class RedisCache(BaseCache): error=e, call_type=f"sync_ping <- {_get_call_stack_info()}", ) - verbose_logger.error(f"LiteLLM Redis Cache PING: - Got exception from REDIS : {e!s}") + verbose_logger.error(f"LiteLLM Redis Cache PING: - Got exception from REDIS : {e}") raise e async def ping(self) -> bool: @@ -1326,7 +1326,7 @@ class RedisCache(BaseCache): call_type=f"async_ping <- {_get_call_stack_info()}", ) ) - verbose_logger.error(f"LiteLLM Redis Cache PING: - Got exception from REDIS : {e!s}") + verbose_logger.error(f"LiteLLM Redis Cache PING: - Got exception from REDIS : {e}") raise e @_redis_circuit_breaker_guard @@ -1388,10 +1388,10 @@ class RedisCache(BaseCache): else: return {"status": "failed", "message": "Redis ping returned False"} except Exception as e: - verbose_logger.error(f"Redis connection test failed: {e!s}") + verbose_logger.error(f"Redis connection test failed: {e}") return { "status": "failed", - "message": f"Redis connection failed: {e!s}", + "message": f"Redis connection failed: {e}", "error": str(e), } @@ -1565,7 +1565,7 @@ class RedisCache(BaseCache): call_type=f"async_rpush <- {_get_call_stack_info()}", ) ) - verbose_logger.error(f"LiteLLM Redis Cache RPUSH: - Got exception from REDIS : {e!s}") + verbose_logger.error(f"LiteLLM Redis Cache RPUSH: - Got exception from REDIS : {e}") raise e async def _pipeline_rpush_helper( @@ -1711,7 +1711,7 @@ class RedisCache(BaseCache): call_type=f"async_lpop <- {_get_call_stack_info()}", ) ) - verbose_logger.error(f"LiteLLM Redis Cache LPOP: - Got exception from REDIS : {e!s}") + verbose_logger.error(f"LiteLLM Redis Cache LPOP: - Got exception from REDIS : {e}") raise e async def _pipeline_lpop_helper( diff --git a/litellm/caching/redis_cluster_cache.py b/litellm/caching/redis_cluster_cache.py index 1e4c4684f48..127a5c3bd29 100644 --- a/litellm/caching/redis_cluster_cache.py +++ b/litellm/caching/redis_cluster_cache.py @@ -100,9 +100,9 @@ class RedisClusterCache(RedisCache): except Exception as e: from litellm._logging import verbose_logger - verbose_logger.error(f"Redis Cluster connection test failed: {e!s}") + verbose_logger.error(f"Redis Cluster connection test failed: {e}") return { "status": "failed", - "message": f"Redis Cluster connection failed: {e!s}", + "message": f"Redis Cluster connection failed: {e}", "error": str(e), } diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index b2d8efa1dba..f55274d446d 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -364,7 +364,7 @@ class RedisSemanticCache(BaseCache): try: cached_response = ast.literal_eval(cached_response) except (ValueError, SyntaxError) as e: - print_verbose(f"Error parsing cached response: {e!s}") + print_verbose(f"Error parsing cached response: {e}") return None return cached_response @@ -403,7 +403,7 @@ class RedisSemanticCache(BaseCache): store_kwargs["ttl"] = int(ttl) self.llmcache.store(prompt, value_str, **store_kwargs) except Exception as e: - print_verbose(f"Error setting {value_str or value} in the Redis semantic cache: {e!s}") + print_verbose(f"Error setting {value_str or value} in the Redis semantic cache: {e}") def get_cache(self, key: str, **kwargs) -> Any: """ @@ -468,7 +468,7 @@ class RedisSemanticCache(BaseCache): return self._get_cache_logic(cached_response=cached_response) except Exception as e: - print_verbose(f"Error retrieving from Redis semantic cache: {e!s}") + print_verbose(f"Error retrieving from Redis semantic cache: {e}") kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 async def _get_async_embedding(self, prompt: str, metadata: dict[str, Any] | None = None) -> list[float]: @@ -505,8 +505,8 @@ class RedisSemanticCache(BaseCache): ) return embedding_response["data"][0]["embedding"] except Exception as e: - print_verbose(f"Error generating async embedding: {e!s}") - raise ValueError(f"Failed to generate embedding: {e!s}") from e + print_verbose(f"Error generating async embedding: {e}") + raise ValueError(f"Failed to generate embedding: {e}") from e async def async_set_cache(self, key: str, value: Any, **kwargs) -> None: """ @@ -546,7 +546,7 @@ class RedisSemanticCache(BaseCache): **store_kwargs, ) except Exception as e: - print_verbose(f"Error in async_set_cache: {e!s}") + print_verbose(f"Error in async_set_cache: {e}") async def async_get_cache(self, key: str, **kwargs) -> Any: """ @@ -612,7 +612,7 @@ class RedisSemanticCache(BaseCache): return self._get_cache_logic(cached_response=cached_response) except Exception as e: - print_verbose(f"Error in async_get_cache: {e!s}") + print_verbose(f"Error in async_get_cache: {e}") kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 async def _index_info(self) -> dict[str, Any]: @@ -639,4 +639,4 @@ class RedisSemanticCache(BaseCache): tasks.append(self.async_set_cache(val[0], val[1], **kwargs)) await asyncio.gather(*tasks) except Exception as e: - print_verbose(f"Error in async_set_cache_pipeline: {e!s}") + print_verbose(f"Error in async_set_cache_pipeline: {e}") diff --git a/litellm/caching/valkey_semantic_cache.py b/litellm/caching/valkey_semantic_cache.py index 86e687c0009..e01bb430987 100644 --- a/litellm/caching/valkey_semantic_cache.py +++ b/litellm/caching/valkey_semantic_cache.py @@ -249,7 +249,7 @@ class ValkeySemanticCache(RedisSemanticCache): if ttl is not None: self.sync_client.expire(doc_key, ttl) except Exception as e: - print_verbose(f"Error in Valkey semantic-cache set_cache: {e!s}") + print_verbose(f"Error in Valkey semantic-cache set_cache: {e}") def get_cache(self, key: str, **kwargs: Any) -> Any: print_verbose(f"Valkey semantic-cache get_cache, kwargs: {kwargs}") @@ -268,7 +268,7 @@ class ValkeySemanticCache(RedisSemanticCache): ) return self._resolve_hit(self._first_hit(search_result), key, **kwargs) except Exception as e: - print_verbose(f"Error in Valkey semantic-cache get_cache: {e!s}") + print_verbose(f"Error in Valkey semantic-cache get_cache: {e}") kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 async def async_set_cache(self, key: str, value: Any, **kwargs: Any) -> None: @@ -288,7 +288,7 @@ class ValkeySemanticCache(RedisSemanticCache): if ttl is not None: await self.async_client.expire(doc_key, ttl) except Exception as e: - print_verbose(f"Error in async Valkey semantic-cache set_cache: {e!s}") + print_verbose(f"Error in async Valkey semantic-cache set_cache: {e}") async def async_get_cache(self, key: str, **kwargs: Any) -> Any: print_verbose(f"Async Valkey semantic-cache get_cache, kwargs: {kwargs}") @@ -307,14 +307,14 @@ class ValkeySemanticCache(RedisSemanticCache): ) return self._resolve_hit(self._first_hit(search_result), key, **kwargs) except Exception as e: - print_verbose(f"Error in async Valkey semantic-cache get_cache: {e!s}") + print_verbose(f"Error in async Valkey semantic-cache get_cache: {e}") kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 async def async_set_cache_pipeline(self, cache_list: list[tuple[str, Any]], **kwargs: Any) -> None: try: await asyncio.gather(*[self.async_set_cache(key, value, **kwargs) for key, value in cache_list]) except Exception as e: - print_verbose(f"Error in Valkey semantic-cache async_set_cache_pipeline: {e!s}") + print_verbose(f"Error in Valkey semantic-cache async_set_cache_pipeline: {e}") async def _index_info(self) -> dict: return await self.async_client.ft(self.index_name).info() diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index f10a9e327d6..f04a9d61d4a 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -715,7 +715,7 @@ def _get_provider_for_cost_calc( _, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model) except Exception as e: verbose_logger.debug( - f"litellm.cost_calculator.py::_get_provider_for_cost_calc() - Error inferring custom_llm_provider - {e!s}" + f"litellm.cost_calculator.py::_get_provider_for_cost_calc() - Error inferring custom_llm_provider - {e}" ) return None @@ -1092,7 +1092,7 @@ def _store_cost_breakdown_in_logging_obj( ) except Exception as breakdown_error: - verbose_logger.debug(f"Error storing cost breakdown: {breakdown_error!s}") + verbose_logger.debug(f"Error storing cost breakdown: {breakdown_error}") # Don't fail the main cost calculation if breakdown storage fails @@ -1315,7 +1315,7 @@ def completion_cost( ) # strip the llm provider from the model name -> for image gen cost calculation except Exception as e: verbose_logger.debug( - f"litellm.cost_calculator.py::completion_cost() - Error inferring custom_llm_provider - {e!s}" + f"litellm.cost_calculator.py::completion_cost() - Error inferring custom_llm_provider - {e}" ) if CostCalculatorUtils._call_type_has_image_response(call_type) and isinstance( completion_response, ImageResponse @@ -1662,7 +1662,7 @@ def completion_cost( return _final_cost except Exception as e: verbose_logger.debug( - f"litellm.cost_calculator.py::completion_cost() - Error calculating cost for model={model} - {e!s}" + f"litellm.cost_calculator.py::completion_cost() - Error calculating cost for model={model} - {e}" ) if idx == len(potential_model_names) - 1: raise e diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 0d85c795c7b..c4a64e0ad9b 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -1140,7 +1140,7 @@ class MidStreamFallbackError(ServiceUnavailableError): # type: ignore if self.max_retries: _message += f", LiteLLM Max Retries: {self.max_retries}" if self.original_exception: - _message += f" Original exception: {type(self.original_exception).__name__}: {self.original_exception!s}" + _message += f" Original exception: {type(self.original_exception).__name__}: {self.original_exception}" return _message def __repr__(self): diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index 72248c4448d..8815c38192b 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -515,7 +515,7 @@ class MCPClient: _log( f"MCP client list_tools failed - " f"Error Type: {error_type}, " - f"Error: {e!s}, " + f"Error: {e}, " f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" ) @@ -536,7 +536,7 @@ class MCPClient: def error_tool_result(exc: Exception) -> MCPCallToolResult: """The error result ``call_tool`` returns when it swallows a failure (no re-execution).""" return MCPCallToolResult( - content=[TextContent(type="text", text=f"{type(exc).__name__}: {exc!s}")], + content=[TextContent(type="text", text=f"{type(exc).__name__}: {exc}")], isError=True, ) @@ -601,7 +601,7 @@ class MCPClient: _log( f"MCP client call_tool failed - " f"Error Type: {error_type}, " - f"Error: {e!s}, " + f"Error: {e}, " f"Tool: {call_tool_request_params.name}, " f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" @@ -640,7 +640,7 @@ class MCPClient: verbose_logger.error( f"MCP client list_prompts failed - " f"Error Type: {error_type}, " - f"Error: {e!s}, " + f"Error: {e}, " f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" ) @@ -681,7 +681,7 @@ class MCPClient: verbose_logger.error( f"MCP client get_prompt failed - " f"Error Type: {error_type}, " - f"Error: {e!s}, " + f"Error: {e}, " f"Prompt: {get_prompt_request_params.name}, " f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" @@ -717,7 +717,7 @@ class MCPClient: verbose_logger.error( f"MCP client list_resources failed - " f"Error Type: {error_type}, " - f"Error: {e!s}, " + f"Error: {e}, " f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" ) @@ -753,7 +753,7 @@ class MCPClient: verbose_logger.error( f"MCP client list_resource_templates failed - " f"Error Type: {error_type}, " - f"Error: {e!s}, " + f"Error: {e}, " f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" ) @@ -791,7 +791,7 @@ class MCPClient: verbose_logger.error( f"MCP client read_resource failed - " f"Error Type: {error_type}, " - f"Error: {e!s}, " + f"Error: {e}, " f"Url: {url}, " f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" diff --git a/litellm/google_genai/adapters/handler.py b/litellm/google_genai/adapters/handler.py index 573f0633af5..5236e207cc5 100644 --- a/litellm/google_genai/adapters/handler.py +++ b/litellm/google_genai/adapters/handler.py @@ -98,7 +98,7 @@ class GenerateContentToCompletionHandler: return generate_content_response except Exception as e: - raise ValueError(f"Error calling litellm.acompletion for generate_content: {e!s}") + raise ValueError(f"Error calling litellm.acompletion for generate_content: {e}") @staticmethod def generate_content_handler( @@ -159,4 +159,4 @@ class GenerateContentToCompletionHandler: return generate_content_response except Exception as e: - raise ValueError(f"Error calling litellm.completion for generate_content: {e!s}") + raise ValueError(f"Error calling litellm.completion for generate_content: {e}") diff --git a/litellm/integrations/SlackAlerting/batching_handler.py b/litellm/integrations/SlackAlerting/batching_handler.py index e5a60640ee2..da905b606a5 100644 --- a/litellm/integrations/SlackAlerting/batching_handler.py +++ b/litellm/integrations/SlackAlerting/batching_handler.py @@ -70,6 +70,6 @@ async def send_to_webhook(slackAlertingInstance: SlackAlertingType, item, count) if response.status_code != 200: verbose_proxy_logger.debug(f"Error sending slack alert to url={item['url']}. Error={response.text}") except Exception as e: - verbose_proxy_logger.debug(f"Error sending slack alert: {e!s}") + verbose_proxy_logger.debug(f"Error sending slack alert: {e}") finally: _print_alerting_payload_warning(payload, slackAlertingInstance=slackAlertingInstance) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 4378b2f754e..114924e7359 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -1467,7 +1467,7 @@ Model Info: try: await self._flush_digest_buckets() except Exception as e: - verbose_proxy_logger.debug(f"Error flushing digest buckets: {e!s}") + verbose_proxy_logger.debug(f"Error flushing digest buckets: {e}") await self.flush_queue() async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -1502,7 +1502,7 @@ Model Info: ) except Exception as e: verbose_proxy_logger.error( - f"[Non-Blocking Error] Slack Alerting: Got error in logging LLM deployment latency: {e!s}" + f"[Non-Blocking Error] Slack Alerting: Got error in logging LLM deployment latency: {e}" ) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): @@ -1522,7 +1522,7 @@ Model Info: ) ) except Exception as e: - verbose_logger.debug(f"Exception raises -{e!s}") + verbose_logger.debug(f"Exception raises -{e}") if isinstance(kwargs.get("exception", ""), APIError): if "outage_alerts" in self.alert_types: diff --git a/litellm/integrations/arize/arize.py b/litellm/integrations/arize/arize.py index 9d743659135..86e861afb8a 100644 --- a/litellm/integrations/arize/arize.py +++ b/litellm/integrations/arize/arize.py @@ -169,7 +169,7 @@ class ArizeLogger(OpenTelemetry): except Exception as e: return { "status": "unhealthy", - "error_message": f"Arize health check failed: {e!s}", + "error_message": f"Arize health check failed: {e}", } def construct_dynamic_otel_headers( diff --git a/litellm/integrations/azure_sentinel/azure_sentinel.py b/litellm/integrations/azure_sentinel/azure_sentinel.py index f0200b75c43..e0ed0cd7cf3 100644 --- a/litellm/integrations/azure_sentinel/azure_sentinel.py +++ b/litellm/integrations/azure_sentinel/azure_sentinel.py @@ -203,7 +203,7 @@ class AzureSentinelLogger(CustomBatchLogger): await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"Azure Sentinel Layer Error - {e!s}\n{traceback.format_exc()}") + verbose_logger.exception(f"Azure Sentinel Layer Error - {e}\n{traceback.format_exc()}") async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): """ @@ -233,7 +233,7 @@ class AzureSentinelLogger(CustomBatchLogger): await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"Azure Sentinel Layer Error - {e!s}\n{traceback.format_exc()}") + verbose_logger.exception(f"Azure Sentinel Layer Error - {e}\n{traceback.format_exc()}") async def async_log_audit_log_event(self, audit_log: StandardAuditLogPayload) -> None: """ @@ -256,7 +256,7 @@ class AzureSentinelLogger(CustomBatchLogger): await self.async_send_audit_batch() except Exception as e: - verbose_logger.exception(f"Azure Sentinel Audit Log Layer Error - {e!s}\n{traceback.format_exc()}") + verbose_logger.exception(f"Azure Sentinel Audit Log Layer Error - {e}\n{traceback.format_exc()}") async def async_send_batch(self): """ @@ -323,7 +323,7 @@ class AzureSentinelLogger(CustomBatchLogger): ) except Exception as e: - verbose_logger.exception(f"Azure Sentinel Error sending batch API - {e!s}\n{traceback.format_exc()}") + verbose_logger.exception(f"Azure Sentinel Error sending batch API - {e}\n{traceback.format_exc()}") finally: log_queue.clear() diff --git a/litellm/integrations/azure_storage/azure_storage.py b/litellm/integrations/azure_storage/azure_storage.py index bbd6e9698bb..d2dd3d37dc7 100644 --- a/litellm/integrations/azure_storage/azure_storage.py +++ b/litellm/integrations/azure_storage/azure_storage.py @@ -53,9 +53,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): self.log_queue: list[StandardLoggingPayload] = [] super().__init__(**kwargs, flush_lock=self.flush_lock) except Exception as e: - verbose_logger.exception( - f"AzureBlobStorageLogger: Got exception on init AzureBlobStorageLogger client {e!s}" - ) + verbose_logger.exception(f"AzureBlobStorageLogger: Got exception on init AzureBlobStorageLogger client {e}") raise e async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -79,7 +77,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): self.log_queue.append(standard_logging_payload) except Exception as e: - verbose_logger.exception(f"AzureBlobStorageLogger Layer Error - {e!s}") + verbose_logger.exception(f"AzureBlobStorageLogger Layer Error - {e}") async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): """ @@ -101,7 +99,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): self.log_queue.append(standard_logging_payload) except Exception as e: - verbose_logger.exception(f"AzureBlobStorageLogger Layer Error - {e!s}") + verbose_logger.exception(f"AzureBlobStorageLogger Layer Error - {e}") async def async_send_batch(self): """ @@ -124,7 +122,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): await self.async_upload_payload_to_azure_blob_storage(payload=payload) except Exception as e: - verbose_logger.exception(f"AzureBlobStorageLogger Error sending batch API - {e!s}") + verbose_logger.exception(f"AzureBlobStorageLogger Error sending batch API - {e}") async def async_upload_payload_to_azure_blob_storage(self, payload: StandardLoggingPayload): """ @@ -153,7 +151,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): verbose_logger.debug(f"Successfully uploaded log to Azure Blob Storage: {filename}") except Exception as e: - verbose_logger.exception(f"Error uploading to Azure Blob Storage: {e!s}") + verbose_logger.exception(f"Error uploading to Azure Blob Storage: {e}") raise e async def _create_file(self, client: AsyncHTTPHandler, base_url: str): @@ -169,7 +167,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): response.raise_for_status() verbose_logger.debug("Successfully created file resource") except Exception as e: - verbose_logger.exception(f"Error creating file resource: {e!s}") + verbose_logger.exception(f"Error creating file resource: {e}") raise async def _append_data(self, client: AsyncHTTPHandler, base_url: str, json_payload: str): @@ -189,7 +187,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): response.raise_for_status() verbose_logger.debug("Successfully appended data") except Exception as e: - verbose_logger.exception(f"Error appending data: {e!s}") + verbose_logger.exception(f"Error appending data: {e}") raise async def _flush_data(self, client: AsyncHTTPHandler, base_url: str, position: int): @@ -205,7 +203,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): response.raise_for_status() verbose_logger.debug("Successfully flushed data") except Exception as e: - verbose_logger.exception(f"Error flushing data: {e!s}") + verbose_logger.exception(f"Error flushing data: {e}") raise ####### Helper methods to managing Authentication to Azure Storage ####### @@ -345,4 +343,4 @@ class AzureBlobStorageLogger(CustomBatchLogger): verbose_logger.debug(f"Successfully uploaded and wrote to {today}/{file_name}") except Exception as e: - verbose_logger.exception(f"Error occurred: {e!s}") + verbose_logger.exception(f"Error occurred: {e}") diff --git a/litellm/integrations/cloudzero/cloudzero.py b/litellm/integrations/cloudzero/cloudzero.py index e6faf4a6a62..52b41f74fce 100644 --- a/litellm/integrations/cloudzero/cloudzero.py +++ b/litellm/integrations/cloudzero/cloudzero.py @@ -153,7 +153,7 @@ class CloudZeroLogger(CustomLogger): verbose_logger.debug(f"CloudZero Logger: Successfully exported {len(cbf_data)} records to CloudZero") except Exception as e: - verbose_logger.error(f"CloudZero Logger: Error exporting usage data: {e!s}") + verbose_logger.error(f"CloudZero Logger: Error exporting usage data: {e}") raise async def dry_run_export_usage_data(self, limit: int | None = 10000): @@ -244,8 +244,8 @@ class CloudZeroLogger(CustomLogger): } except Exception as e: - verbose_logger.error(f"CloudZero Logger: Error in dry run export: {e!s}") - verbose_logger.error(f"CloudZero Dry Run Error: {e!s}") + verbose_logger.error(f"CloudZero Logger: Error in dry run export: {e}") + verbose_logger.error(f"CloudZero Dry Run Error: {e}") raise def _display_cbf_data_on_screen(self, cbf_data): diff --git a/litellm/integrations/cloudzero/database.py b/litellm/integrations/cloudzero/database.py index 16fb99517ae..2d0f81af98b 100644 --- a/litellm/integrations/cloudzero/database.py +++ b/litellm/integrations/cloudzero/database.py @@ -98,4 +98,4 @@ class LiteLLMDatabase: # This prevents schema mismatch errors when data types vary across rows return pl.DataFrame(db_response, infer_schema_length=None) except Exception as e: - raise Exception(f"Error retrieving usage data: {e!s}") + raise Exception(f"Error retrieving usage data: {e}") diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 971d53ffec4..9915224ba09 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -927,7 +927,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac except Exception as e: from litellm._logging import verbose_logger - verbose_logger.debug(f"Error in handle_callback_failure for {callback_name}: {e!s}") + verbose_logger.debug(f"Error in handle_callback_failure for {callback_name}: {e}") async def _strip_base64_from_messages( self, diff --git a/litellm/integrations/datadog/datadog.py b/litellm/integrations/datadog/datadog.py index 047d69c9c9c..fa14e1fa459 100644 --- a/litellm/integrations/datadog/datadog.py +++ b/litellm/integrations/datadog/datadog.py @@ -171,7 +171,7 @@ class DataDogLogger( batch_size=_resolve_dd_batch_size(), ) except Exception as e: - verbose_logger.exception(f"Datadog: Got exception on init Datadog client {e!s}") + verbose_logger.exception(f"Datadog: Got exception on init Datadog client {e}") raise e def _get_datadog_params(self) -> dict: @@ -257,7 +257,7 @@ class DataDogLogger( await self._log_async_event(kwargs, response_obj, start_time, end_time) except Exception as e: - verbose_logger.exception(f"Datadog Layer Error - {e!s}\n{traceback.format_exc()}") + verbose_logger.exception(f"Datadog Layer Error - {e}\n{traceback.format_exc()}") async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): try: @@ -265,7 +265,7 @@ class DataDogLogger( await self._log_async_event(kwargs, response_obj, start_time, end_time) except Exception as e: - verbose_logger.exception(f"Datadog Layer Error - {e!s}\n{traceback.format_exc()}") + verbose_logger.exception(f"Datadog Layer Error - {e}\n{traceback.format_exc()}") async def async_post_call_failure_hook( self, @@ -340,7 +340,7 @@ class DataDogLogger( if len(self.log_queue) >= self.batch_size: await self.flush_queue() except Exception as e: - verbose_logger.exception(f"Datadog: async_post_call_failure_hook - {e!s}\n{traceback.format_exc()}") + verbose_logger.exception(f"Datadog: async_post_call_failure_hook - {e}\n{traceback.format_exc()}") return None async def async_send_batch(self): @@ -380,7 +380,7 @@ class DataDogLogger( except Exception as e: self.log_queue = batch_to_send + self.log_queue - verbose_logger.exception(f"Datadog Error sending batch API - {e!s}\n{traceback.format_exc()}") + verbose_logger.exception(f"Datadog Error sending batch API - {e}\n{traceback.format_exc()}") async def _send_with_413_split(self, batch: list) -> list: """ @@ -411,7 +411,7 @@ class DataDogLogger( if isinstance(e, MaskedHTTPStatusError) and e.status_code == 413: response = e.response else: - verbose_logger.exception(f"Datadog Error sending batch API - {e!s}") + verbose_logger.exception(f"Datadog Error sending batch API - {e}") return self._undelivered(chunk, pending) if response.status_code == 413: @@ -515,7 +515,7 @@ class DataDogLogger( ) except Exception as e: - verbose_logger.exception(f"Datadog Layer Error - {e!s}\n{traceback.format_exc()}") + verbose_logger.exception(f"Datadog Layer Error - {e}\n{traceback.format_exc()}") async def _log_async_event(self, kwargs, response_obj, start_time, end_time): dd_payload = self.create_datadog_logging_payload( diff --git a/litellm/integrations/datadog/datadog_cost_management.py b/litellm/integrations/datadog/datadog_cost_management.py index 7b22f4658f2..da45f94f02b 100644 --- a/litellm/integrations/datadog/datadog_cost_management.py +++ b/litellm/integrations/datadog/datadog_cost_management.py @@ -84,7 +84,7 @@ class DatadogCostManagementLogger(CustomBatchLogger): await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"Datadog Cost Management: Error in async_log_success_event: {e!s}") + verbose_logger.exception(f"Datadog Cost Management: Error in async_log_success_event: {e}") async def async_send_batch(self): if not self.log_queue: @@ -104,7 +104,7 @@ class DatadogCostManagementLogger(CustomBatchLogger): await self._upload_to_datadog(aggregated_entries) except Exception as e: self.log_queue = batch_to_send + self.log_queue - verbose_logger.exception(f"Datadog Cost Management: Error in async_send_batch: {e!s}") + verbose_logger.exception(f"Datadog Cost Management: Error in async_send_batch: {e}") def _aggregate_costs(self, logs: list[StandardLoggingPayload]) -> list[DatadogFOCUSCostEntry]: """ diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index e10071cb083..02e1affd361 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -89,7 +89,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): kwargs.update(dict_datadog_llm_obs_params) CustomBatchLogger.__init__(self, **kwargs, flush_lock=self.flush_lock) except Exception as e: - verbose_logger.exception(f"DataDogLLMObs: Error initializing - {e!s}") + verbose_logger.exception(f"DataDogLLMObs: Error initializing - {e}") raise e def _configure_dd_agent(self, dd_agent_host: str): @@ -145,7 +145,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): if len(self.log_queue) >= self.batch_size: await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"DataDogLLMObs: Error logging success event - {e!s}") + verbose_logger.exception(f"DataDogLLMObs: Error logging success event - {e}") async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): try: @@ -157,7 +157,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): if len(self.log_queue) >= self.batch_size: await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"DataDogLLMObs: Error logging failure event - {e!s}") + verbose_logger.exception(f"DataDogLLMObs: Error logging failure event - {e}") async def async_send_batch(self): try: @@ -214,7 +214,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): except httpx.HTTPStatusError as e: verbose_logger.exception(f"DataDogLLMObs: Error sending batch - {e.response.text}") except Exception as e: - verbose_logger.exception(f"DataDogLLMObs: Error sending batch - {e!s}") + verbose_logger.exception(f"DataDogLLMObs: Error sending batch - {e}") def create_llm_obs_payload(self, kwargs: dict, start_time: datetime, end_time: datetime) -> LLMObsPayload: standard_logging_payload: StandardLoggingPayload | None = kwargs.get("standard_logging_object") @@ -707,7 +707,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): kv_pairs[f"tool_calls.{idx}.function.arguments"] = json.dumps(function_arguments) except (KeyError, TypeError, ValueError) as e: - verbose_logger.debug(f"DataDogLLMObs: Error processing tool call {idx}: {e!s}") + verbose_logger.debug(f"DataDogLLMObs: Error processing tool call {idx}: {e}") continue return kv_pairs @@ -747,6 +747,6 @@ class DataDogLLMObsLogger(CustomBatchLogger): tool_call_metadata[f"output_{key}"] = value except Exception as e: - verbose_logger.debug(f"DataDogLLMObs: Error extracting tool call metadata: {e!s}") + verbose_logger.debug(f"DataDogLLMObs: Error extracting tool call metadata: {e}") return tool_call_metadata diff --git a/litellm/integrations/datadog/datadog_metrics.py b/litellm/integrations/datadog/datadog_metrics.py index 3fbd0f917dc..9fb86bfb125 100644 --- a/litellm/integrations/datadog/datadog_metrics.py +++ b/litellm/integrations/datadog/datadog_metrics.py @@ -180,7 +180,7 @@ class DatadogMetricsLogger(CustomBatchLogger): await self.flush_queue() except Exception as e: - verbose_logger.exception(f"Datadog Metrics: Error in async_log_success_event: {e!s}") + verbose_logger.exception(f"Datadog Metrics: Error in async_log_success_event: {e}") async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): try: @@ -202,7 +202,7 @@ class DatadogMetricsLogger(CustomBatchLogger): await self.flush_queue() except Exception as e: - verbose_logger.exception(f"Datadog Metrics: Error in async_log_failure_event: {e!s}") + verbose_logger.exception(f"Datadog Metrics: Error in async_log_failure_event: {e}") async def async_send_batch(self): if not self.log_queue: @@ -214,7 +214,7 @@ class DatadogMetricsLogger(CustomBatchLogger): try: await self._upload_to_datadog(payload_data) except Exception as e: - verbose_logger.exception(f"Datadog Metrics: Error in async_send_batch: {e!s}") + verbose_logger.exception(f"Datadog Metrics: Error in async_send_batch: {e}") raise async def _upload_to_datadog(self, payload: DatadogMetricsPayload): diff --git a/litellm/integrations/dynamodb.py b/litellm/integrations/dynamodb.py index 5826a06b0ec..a41130cbab1 100644 --- a/litellm/integrations/dynamodb.py +++ b/litellm/integrations/dynamodb.py @@ -70,7 +70,7 @@ class DyanmoDBLogger: # Assuming log_data is a dictionary with log information response = table.put_item(Item=payload) - print_verbose(f"Response from DynamoDB:{response!s}") + print_verbose(f"Response from DynamoDB:{response}") print_verbose(f"DynamoDB Layer Logging - final response object: {response_obj}") return response diff --git a/litellm/integrations/galileo.py b/litellm/integrations/galileo.py index 0180af51992..f7870a6c0f8 100644 --- a/litellm/integrations/galileo.py +++ b/litellm/integrations/galileo.py @@ -128,7 +128,7 @@ class GalileoObserve(CustomLogger): except Exception as e: return IntegrationHealthCheckStatus( status="unhealthy", - error_message=f"Galileo health check failed: {e!s}", + error_message=f"Galileo health check failed: {e}", ) async def async_set_galileo_headers(self) -> None: diff --git a/litellm/integrations/gcs_bucket/gcs_bucket.py b/litellm/integrations/gcs_bucket/gcs_bucket.py index 552e078cb60..b5b3d4e81a3 100644 --- a/litellm/integrations/gcs_bucket/gcs_bucket.py +++ b/litellm/integrations/gcs_bucket/gcs_bucket.py @@ -76,7 +76,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): await self.log_queue.put(GCSLogQueueItem(payload=logging_payload, kwargs=kwargs, response_obj=response_obj)) except Exception as e: - verbose_logger.exception(f"GCS Bucket logging error: {e!s}") + verbose_logger.exception(f"GCS Bucket logging error: {e}") async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): try: @@ -95,7 +95,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): await self.log_queue.put(GCSLogQueueItem(payload=logging_payload, kwargs=kwargs, response_obj=response_obj)) except Exception as e: - verbose_logger.exception(f"GCS Bucket logging error: {e!s}") + verbose_logger.exception(f"GCS Bucket logging error: {e}") def _drain_queue_batch(self) -> list[GCSLogQueueItem]: """ @@ -218,7 +218,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): except Exception as e: success_count = 0 error_count = len(items) - verbose_logger.exception(f"GCS Bucket error logging batch payload to GCS bucket: {e!s}") + verbose_logger.exception(f"GCS Bucket error logging batch payload to GCS bucket: {e}") return (success_count, error_count) async def _send_individual_logs(self, items: list[GCSLogQueueItem]) -> None: @@ -255,7 +255,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): logging_payload=item["payload"], ) except Exception as e: - verbose_logger.exception(f"GCS Bucket error logging individual payload to GCS bucket: {e!s}") + verbose_logger.exception(f"GCS Bucket error logging individual payload to GCS bucket: {e}") async def async_send_batch(self): """ @@ -336,7 +336,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): loaded_response = json.loads(response) return loaded_response except Exception as e: - verbose_logger.debug(f"Failed to fetch payload for date {date_str}: {e!s}") + verbose_logger.debug(f"Failed to fetch payload for date {date_str}: {e}") continue return None diff --git a/litellm/integrations/gcs_pubsub/pub_sub.py b/litellm/integrations/gcs_pubsub/pub_sub.py index 6ade70ab6d6..b43e7626b77 100644 --- a/litellm/integrations/gcs_pubsub/pub_sub.py +++ b/litellm/integrations/gcs_pubsub/pub_sub.py @@ -132,7 +132,7 @@ class GcsPubSubLogger(CustomBatchLogger): await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"PubSub Layer Error - {e!s}\n{traceback.format_exc()}") + verbose_logger.exception(f"PubSub Layer Error - {e}\n{traceback.format_exc()}") async def async_send_batch(self): """ @@ -148,7 +148,7 @@ class GcsPubSubLogger(CustomBatchLogger): await self.publish_message(message) except Exception as e: - verbose_logger.exception(f"PubSub Error sending batch - {e!s}\n{traceback.format_exc()}") + verbose_logger.exception(f"PubSub Error sending batch - {e}\n{traceback.format_exc()}") finally: self.log_queue.clear() diff --git a/litellm/integrations/generic_api/generic_api_callback.py b/litellm/integrations/generic_api/generic_api_callback.py index a524755540e..c7f2661a5ad 100644 --- a/litellm/integrations/generic_api/generic_api_callback.py +++ b/litellm/integrations/generic_api/generic_api_callback.py @@ -42,7 +42,7 @@ def load_compatible_callbacks() -> dict: with open(json_path, "r") as f: return json.load(f) except Exception as e: - verbose_logger.warning(f"Error loading generic_api_compatible_callbacks.json: {e!s}") + verbose_logger.warning(f"Error loading generic_api_compatible_callbacks.json: {e}") return {} @@ -214,7 +214,7 @@ class GenericAPILogger(CustomBatchLogger): key, value = item.split("=", 1) headers_dict[key.strip()] = value.strip() except Exception as e: - verbose_logger.warning(f"Error parsing headers from environment variables: {e!s}") + verbose_logger.warning(f"Error parsing headers from environment variables: {e}") # 2. Update with litellm generic headers if available if litellm.generic_logger_headers: @@ -308,7 +308,7 @@ class GenericAPILogger(CustomBatchLogger): await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"Generic API Logger Error - {e!s}\n{traceback.format_exc()}") + verbose_logger.exception(f"Generic API Logger Error - {e}\n{traceback.format_exc()}") async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): """ @@ -339,7 +339,7 @@ class GenericAPILogger(CustomBatchLogger): await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"Generic API Logger Error - {e!s}\n{traceback.format_exc()}") + verbose_logger.exception(f"Generic API Logger Error - {e}\n{traceback.format_exc()}") async def async_send_batch(self): """ @@ -395,7 +395,7 @@ class GenericAPILogger(CustomBatchLogger): ) except Exception as e: - verbose_logger.exception(f"Generic API Logger Error sending batch - {e!s}\n{traceback.format_exc()}") + verbose_logger.exception(f"Generic API Logger Error sending batch - {e}\n{traceback.format_exc()}") finally: self.log_queue.clear() diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 3fb50e07b01..2dab1874c01 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -330,7 +330,7 @@ class LangFuseLogger: return {"trace_id": trace_id, "generation_id": generation_id} except Exception as e: - verbose_logger.exception(f"Langfuse Layer Error(): Exception occured - {e!s}") + verbose_logger.exception(f"Langfuse Layer Error(): Exception occured - {e}") return {"trace_id": None, "generation_id": None} def _get_langfuse_input_output_content( diff --git a/litellm/integrations/langfuse/langfuse_prompt_management.py b/litellm/integrations/langfuse/langfuse_prompt_management.py index 9a5ee49bd0d..56383b45a8c 100644 --- a/litellm/integrations/langfuse/langfuse_prompt_management.py +++ b/litellm/integrations/langfuse/langfuse_prompt_management.py @@ -317,7 +317,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge except Exception as e: from litellm._logging import verbose_logger - verbose_logger.exception(f"Langfuse Layer Error - Exception occurred while logging success event: {e!s}") + verbose_logger.exception(f"Langfuse Layer Error - Exception occurred while logging success event: {e}") self.handle_callback_failure(callback_name="langfuse") async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): @@ -347,5 +347,5 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge except Exception as e: from litellm._logging import verbose_logger - verbose_logger.exception(f"Langfuse Layer Error - Exception occurred while logging failure event: {e!s}") + verbose_logger.exception(f"Langfuse Layer Error - Exception occurred while logging failure event: {e}") self.handle_callback_failure(callback_name="langfuse") diff --git a/litellm/integrations/logfire_logger.py b/litellm/integrations/logfire_logger.py index 78735c47e5b..c94fb832ccc 100644 --- a/litellm/integrations/logfire_logger.py +++ b/litellm/integrations/logfire_logger.py @@ -35,7 +35,7 @@ class LogfireLogger: if logfire.DEFAULT_LOGFIRE_INSTANCE.config.send_to_logfire: logfire.configure(token=os.getenv("LOGFIRE_TOKEN")) except Exception as e: - print_verbose(f"Got exception on init logfire client {e!s}") + print_verbose(f"Got exception on init logfire client {e}") raise e def _get_span_config(self, payload) -> SpanConfig: @@ -159,4 +159,4 @@ class LogfireLogger: print_verbose(f"Logfire Layer Logging - final response object: {response_obj}") except Exception as e: - verbose_logger.debug(f"Logfire Layer Error - {e!s}\n{traceback.format_exc()}") + verbose_logger.debug(f"Logfire Layer Error - {e}\n{traceback.format_exc()}") diff --git a/litellm/integrations/opik/opik.py b/litellm/integrations/opik/opik.py index deb325286e9..e4d40a1af8f 100644 --- a/litellm/integrations/opik/opik.py +++ b/litellm/integrations/opik/opik.py @@ -81,7 +81,7 @@ class OpikLogger(CustomBatchLogger): self.flush_lock: asyncio.Lock | None = asyncio.Lock() except Exception as e: verbose_logger.exception( - f"OpikLogger - Asynchronous processing not initialized as we are not running in an async context {e!s}" + f"OpikLogger - Asynchronous processing not initialized as we are not running in an async context {e}" ) self.flush_lock = None @@ -161,7 +161,7 @@ class OpikLogger(CustomBatchLogger): verbose_logger.debug("OpikLogger - Flushing batch") await self.flush_queue() except Exception as e: - verbose_logger.exception(f"OpikLogger failed to log success event - {e!s}\n{traceback.format_exc()}") + verbose_logger.exception(f"OpikLogger failed to log success event - {e}\n{traceback.format_exc()}") def _sync_send(self, url: str, headers: dict[str, str], batch: dict[str, Any]) -> None: try: @@ -174,7 +174,7 @@ class OpikLogger(CustomBatchLogger): if response.status_code != 204: raise Exception(f"Response from opik API status_code: {response.status_code}, text: {response.text}") except Exception as e: - verbose_logger.exception(f"OpikLogger failed to send batch - {e!s}\n{traceback.format_exc()}") + verbose_logger.exception(f"OpikLogger failed to send batch - {e}\n{traceback.format_exc()}") def log_success_event( self, @@ -245,7 +245,7 @@ class OpikLogger(CustomBatchLogger): batch={"spans": [span_payload.__dict__]}, ) except Exception as e: - verbose_logger.exception(f"OpikLogger failed to log success event - {e!s}\n{traceback.format_exc()}") + verbose_logger.exception(f"OpikLogger failed to log success event - {e}\n{traceback.format_exc()}") async def _submit_batch(self, url: str, headers: dict[str, str], batch: dict[str, Any]) -> None: try: @@ -261,7 +261,7 @@ class OpikLogger(CustomBatchLogger): else: verbose_logger.info(f"OpikLogger - {len(self.log_queue)} Opik events submitted") except Exception as e: - verbose_logger.exception(f"OpikLogger failed to send batch - {e!s}") + verbose_logger.exception(f"OpikLogger failed to send batch - {e}") def _create_opik_headers(self) -> dict[str, str]: headers: dict[str, str] = {} diff --git a/litellm/integrations/posthog.py b/litellm/integrations/posthog.py index b61eeb8198f..216edc44d3f 100644 --- a/litellm/integrations/posthog.py +++ b/litellm/integrations/posthog.py @@ -72,7 +72,7 @@ class PostHogLogger(CustomBatchLogger): super().__init__(**kwargs, flush_lock=None, batch_size=POSTHOG_MAX_BATCH_SIZE) except Exception as e: - verbose_logger.exception(f"PostHog: Got exception on init PostHog client {e!s}") + verbose_logger.exception(f"PostHog: Got exception on init PostHog client {e}") raise e def log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -107,7 +107,7 @@ class PostHogLogger(CustomBatchLogger): verbose_logger.debug("PostHog: Sync event successfully sent") except Exception as e: - verbose_logger.exception(f"PostHog Sync Layer Error - {e!s}") + verbose_logger.exception(f"PostHog Sync Layer Error - {e}") async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): try: @@ -115,7 +115,7 @@ class PostHogLogger(CustomBatchLogger): self._ensure_async_setup() # Lazy initialization await self._log_async_event(kwargs, response_obj, start_time, end_time) except Exception as e: - verbose_logger.exception(f"PostHog Layer Error - {e!s}") + verbose_logger.exception(f"PostHog Layer Error - {e}") async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): try: @@ -123,7 +123,7 @@ class PostHogLogger(CustomBatchLogger): self._ensure_async_setup() # Lazy initialization await self._log_async_event(kwargs, response_obj, start_time, end_time) except Exception as e: - verbose_logger.exception(f"PostHog Layer Error - {e!s}") + verbose_logger.exception(f"PostHog Layer Error - {e}") async def _log_async_event(self, kwargs, response_obj=None, start_time=0.0, end_time=0.0): # Note: response_obj, start_time, end_time not used - all data comes from kwargs @@ -367,7 +367,7 @@ class PostHogLogger(CustomBatchLogger): else: verbose_logger.debug(f"PostHog: Batch of {len(self.log_queue)} events successfully sent") except Exception as e: - verbose_logger.exception(f"PostHog Error sending batch API - {e!s}") + verbose_logger.exception(f"PostHog Error sending batch API - {e}") def _ensure_async_setup(self): if not self._async_initialized: @@ -377,7 +377,7 @@ class PostHogLogger(CustomBatchLogger): self._async_initialized = True verbose_logger.debug("PostHog: Async components initialized") except Exception as e: - verbose_logger.error(f"PostHog: Failed to initialize async components: {e!s}") + verbose_logger.error(f"PostHog: Failed to initialize async components: {e}") raise def _extract_metadata(self, kwargs: dict[str, Any]) -> dict[str, Any]: @@ -445,4 +445,4 @@ class PostHogLogger(CustomBatchLogger): self.log_queue.clear() except Exception as e: - verbose_logger.error(f"PostHog: Error flushing events on exit: {e!s}") + verbose_logger.error(f"PostHog: Error flushing events on exit: {e}") diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index c84a6c34f1f..b7705a40e0c 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -683,7 +683,7 @@ class PrometheusLogger(CustomLogger): ) except Exception as e: - print_verbose(f"Got exception on init prometheus client {e!s}") + print_verbose(f"Got exception on init prometheus client {e}") raise e def _parse_prometheus_config(self) -> dict[str, list[str]]: @@ -2132,7 +2132,7 @@ class PrometheusLogger(CustomLogger): response_cost=0, ) except Exception as e: - verbose_logger.exception(f"prometheus Layer Error(): Exception occured - {e!s}") + verbose_logger.exception(f"prometheus Layer Error(): Exception occured - {e}") def _extract_status_code( self, @@ -2383,7 +2383,7 @@ class PrometheusLogger(CustomLogger): ) except Exception as e: - verbose_logger.exception(f"prometheus Layer Error(): Exception occured - {e!s}") + verbose_logger.exception(f"prometheus Layer Error(): Exception occured - {e}") async def async_post_call_success_hook(self, data: dict, user_api_key_dict: UserAPIKeyAuth, response): """ @@ -2608,7 +2608,7 @@ class PrometheusLogger(CustomLogger): ) except Exception as e: - verbose_logger.debug(f"Prometheus Error: set_llm_deployment_failure_metrics. Exception occured - {e!s}") + verbose_logger.debug(f"Prometheus Error: set_llm_deployment_failure_metrics. Exception occured - {e}") def _set_deployment_tpm_rpm_limit_metrics( self, @@ -2722,9 +2722,7 @@ class PrometheusLogger(CustomLogger): ) self.litellm_remaining_requests_metric.labels(**_labels).set(remaining_requests) except Exception as e: - verbose_logger.exception( - f"Prometheus Error: _async_set_router_remaining_metrics. Exception occured - {e!s}" - ) + verbose_logger.exception(f"Prometheus Error: _async_set_router_remaining_metrics. Exception occured - {e}") def set_llm_deployment_success_metrics( self, @@ -2867,7 +2865,7 @@ class PrometheusLogger(CustomLogger): self.litellm_deployment_latency_per_output_token.labels(**_labels).observe(latency_per_token) except Exception as e: - verbose_logger.exception(f"Prometheus Error: set_llm_deployment_success_metrics. Exception occured - {e!s}") + verbose_logger.exception(f"Prometheus Error: set_llm_deployment_success_metrics. Exception occured - {e}") return def _record_guardrail_metrics( @@ -2912,7 +2910,7 @@ class PrometheusLogger(CustomLogger): hook_type=hook_type, ).inc() except Exception as e: - verbose_logger.debug(f"Error recording guardrail metrics: {e!s}") + verbose_logger.debug(f"Error recording guardrail metrics: {e}") ######################################## # Managed Batch Metric Recording Methods @@ -3315,7 +3313,7 @@ class PrometheusLogger(CustomLogger): await set_metrics_function(data) except Exception as e: - verbose_logger.exception(f"Error initializing {data_type} budget metrics: {e!s}") + verbose_logger.exception(f"Error initializing {data_type} budget metrics: {e}") async def _initialize_team_budget_metrics(self): """ @@ -3506,7 +3504,7 @@ class PrometheusLogger(CustomLogger): self.litellm_teams_count_metric.set(total_teams) verbose_logger.debug(f"Prometheus: set litellm_teams_count to {total_teams}") except Exception as e: - verbose_logger.exception(f"Error initializing user/team count metrics: {e!s}") + verbose_logger.exception(f"Error initializing user/team count metrics: {e}") async def _set_key_list_budget_metrics(self, keys: list[str | UserAPIKeyAuth]): """Helper function to set budget metrics for a list of keys""" @@ -3597,7 +3595,7 @@ class PrometheusLogger(CustomLogger): user_api_key_cache=user_api_key_cache, ) except Exception as e: - verbose_logger.debug(f"[Non-Blocking] Prometheus: Error getting team info: {e!s}") + verbose_logger.debug(f"[Non-Blocking] Prometheus: Error getting team info: {e}") return team_object if team_info: @@ -3695,7 +3693,7 @@ class PrometheusLogger(CustomLogger): include_budget_table=True, ) except Exception as e: - verbose_logger.debug(f"[Non-Blocking] Prometheus: Error getting org info: {e!s}") + verbose_logger.debug(f"[Non-Blocking] Prometheus: Error getting org info: {e}") return if org_info is None: @@ -3852,7 +3850,7 @@ class PrometheusLogger(CustomLogger): if key_object: user_api_key_dict.budget_reset_at = key_object.budget_reset_at except Exception as e: - verbose_logger.debug(f"[Non-Blocking] Prometheus: Error getting key info: {e!s}") + verbose_logger.debug(f"[Non-Blocking] Prometheus: Error getting key info: {e}") return user_api_key_dict @@ -3917,7 +3915,7 @@ class PrometheusLogger(CustomLogger): check_db_only=False, ) except Exception as e: - verbose_logger.debug(f"[Non-Blocking] Prometheus: Error getting user info: {e!s}") + verbose_logger.debug(f"[Non-Blocking] Prometheus: Error getting user info: {e}") return user_object if user_info: diff --git a/litellm/integrations/prometheus_services.py b/litellm/integrations/prometheus_services.py index f07606a3192..002d61265a4 100644 --- a/litellm/integrations/prometheus_services.py +++ b/litellm/integrations/prometheus_services.py @@ -82,7 +82,7 @@ class PrometheusServicesLogger: self.mock_testing_failure_calls = 0 except Exception as e: - print_verbose(f"Got exception on init prometheus client {e!s}") + print_verbose(f"Got exception on init prometheus client {e}") raise e def _get_service_metrics_initialize(self, service: ServiceTypes) -> list[ServiceMetrics]: diff --git a/litellm/integrations/s3.py b/litellm/integrations/s3.py index 51de43e302c..c35cc88107f 100644 --- a/litellm/integrations/s3.py +++ b/litellm/integrations/s3.py @@ -78,7 +78,7 @@ class S3Logger: **kwargs, ) except Exception as e: - print_verbose(f"Got exception on init s3 client {e!s}") + print_verbose(f"Got exception on init s3 client {e}") raise e async def _async_log_event(self, kwargs, response_obj, start_time, end_time, print_verbose): @@ -163,12 +163,12 @@ class S3Logger: **sse_params, ) - print_verbose(f"Response from s3:{response!s}") + print_verbose(f"Response from s3:{response}") print_verbose(f"s3 Layer Logging - final response object: {response_obj}") return response except Exception as e: - verbose_logger.exception(f"s3 Layer Error - {e!s}") + verbose_logger.exception(f"s3 Layer Error - {e}") def _validated_sse_value(name: str, value: str | None) -> str | None: diff --git a/litellm/integrations/s3_v2.py b/litellm/integrations/s3_v2.py index 8c6cadd5356..44c6e42f9f0 100644 --- a/litellm/integrations/s3_v2.py +++ b/litellm/integrations/s3_v2.py @@ -125,7 +125,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): BaseAWSLLM.__init__(self) except Exception as e: - print_verbose(f"Got exception on init s3 client {e!s}") + print_verbose(f"Got exception on init s3 client {e}") raise e def _init_s3_params( @@ -284,7 +284,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): self.batch_size, ) except Exception as e: - verbose_logger.exception(f"s3 Layer Error - {e!s}") + verbose_logger.exception(f"s3 Layer Error - {e}") self.handle_callback_failure(callback_name="S3Logger") async def async_upload_data_to_s3(self, batch_logging_element: s3BatchLoggingElement): @@ -383,7 +383,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): response.raise_for_status() break except Exception as e: - verbose_logger.exception(f"Error uploading to s3: {e!s}") + verbose_logger.exception(f"Error uploading to s3: {e}") self.handle_callback_failure(callback_name="S3Logger") async def async_send_batch(self): @@ -557,7 +557,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): response.raise_for_status() break except Exception as e: - verbose_logger.exception(f"Error uploading to s3: {e!s}") + verbose_logger.exception(f"Error uploading to s3: {e}") self.handle_callback_failure(callback_name="S3Logger") async def _download_object_from_s3(self, s3_object_key: str) -> dict | None: @@ -642,7 +642,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): return response.json() except Exception as e: - verbose_logger.exception(f"Error downloading from S3: {e!s}") + verbose_logger.exception(f"Error downloading from S3: {e}") return None async def get_proxy_server_request_from_cold_storage_with_object_key( @@ -666,5 +666,5 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): downloaded_object = await self._download_object_from_s3(object_key) return downloaded_object except Exception as e: - verbose_logger.exception(f"Error retrieving object {object_key} from cold storage: {e!s}") + verbose_logger.exception(f"Error retrieving object {object_key} from cold storage: {e}") return None diff --git a/litellm/integrations/sqs.py b/litellm/integrations/sqs.py index 18717790207..56618b62368 100644 --- a/litellm/integrations/sqs.py +++ b/litellm/integrations/sqs.py @@ -113,7 +113,7 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): BaseAWSLLM.__init__(self) except Exception as e: - print_verbose(f"Got exception on init sqs client {e!s}") + print_verbose(f"Got exception on init sqs client {e}") raise e def _init_sqs_params( @@ -215,7 +215,7 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): self.batch_size, ) except Exception as e: - verbose_logger.exception(f"sqs Layer Error - {e!s}") + verbose_logger.exception(f"sqs Layer Error - {e}") async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): try: @@ -233,7 +233,7 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): ) except Exception as e: - verbose_logger.exception(f"Datadog Layer Error - {e!s}\n{traceback.format_exc()}") + verbose_logger.exception(f"Datadog Layer Error - {e}\n{traceback.format_exc()}") async def async_send_batch(self) -> None: verbose_logger.debug(f"sqs logger - sending batch of {len(self.log_queue)}") @@ -305,7 +305,7 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): ) response.raise_for_status() except Exception as e: - verbose_logger.exception(f"Error sending to SQS: {e!s}") + verbose_logger.exception(f"Error sending to SQS: {e}") async def async_health_check(self) -> IntegrationHealthCheckStatus: """ diff --git a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py index 73c48f72d34..6eac7a27e73 100644 --- a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py +++ b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py @@ -146,7 +146,7 @@ class VectorStorePreCallHook(CustomLogger): return model, modified_messages, non_default_params except Exception as e: - verbose_logger.exception(f"Error in VectorStorePreCallHook: {e!s}") + verbose_logger.exception(f"Error in VectorStorePreCallHook: {e}") # Return original parameters on error return model, messages, non_default_params @@ -275,7 +275,7 @@ class VectorStorePreCallHook(CustomLogger): return response except Exception as e: - verbose_logger.exception(f"Error adding search results to response: {e!s}") + verbose_logger.exception(f"Error adding search results to response: {e}") # Don't fail the request if search results fail to be added return None @@ -322,6 +322,6 @@ class VectorStorePreCallHook(CustomLogger): return response_chunk except Exception as e: - verbose_logger.exception(f"Error adding search results to streaming chunk: {e!s}") + verbose_logger.exception(f"Error adding search results to streaming chunk: {e}") # Don't fail the request if search results fail to be added return response_chunk diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 54278afafc4..718f7b8fcd7 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -224,7 +224,7 @@ class WebSearchInterceptionLogger(CustomLogger): content.append({"type": "text", "text": search_result_text}) response: dict[str, object] = { - "id": f"msg_{uuid.uuid4()!s}", + "id": f"msg_{uuid.uuid4()}", "type": "message", "role": "assistant", "model": model, @@ -1038,8 +1038,8 @@ class WebSearchInterceptionLogger(CustomLogger): @staticmethod def _extract_search_text(result: object) -> str: if isinstance(result, Exception): - verbose_logger.error(f"WebSearchInterception: Responses search failed with error: {result!s}") - return f"Search failed: {result!s}" + verbose_logger.error(f"WebSearchInterception: Responses search failed with error: {result}") + return f"Search failed: {result}" if isinstance(result, tuple) and len(result) == 2: text_value, _ = result return text_value if isinstance(text_value, str) else str(text_value) @@ -1194,8 +1194,8 @@ class WebSearchInterceptionLogger(CustomLogger): structured_results: list[SearchResponse | None] = [] for i, result in enumerate(search_results): if isinstance(result, Exception): - verbose_logger.error(f"WebSearchInterception: Search {i} failed with error: {result!s}") - final_search_results.append(f"Search failed: {result!s}") + verbose_logger.error(f"WebSearchInterception: Search {i} failed with error: {result}") + final_search_results.append(f"Search failed: {result}") structured_results.append(None) elif isinstance(result, tuple) and len(result) == 2: text_value, structured_value = result @@ -1308,7 +1308,7 @@ class WebSearchInterceptionLogger(CustomLogger): ) return search_result_text, result except Exception as e: - verbose_logger.error(f"WebSearchInterception: Search failed for '{query}': {e!s}") + verbose_logger.error(f"WebSearchInterception: Search failed for '{query}': {e}") raise async def _authorize_search_tool( @@ -1486,8 +1486,8 @@ class WebSearchInterceptionLogger(CustomLogger): final_search_results: list[str] = [] for i, result in enumerate(search_results): if isinstance(result, Exception): - verbose_logger.error(f"WebSearchInterception: Search {i} failed with error: {result!s}") - final_search_results.append(f"Search failed: {result!s}") + verbose_logger.error(f"WebSearchInterception: Search {i} failed with error: {result}") + final_search_results.append(f"Search failed: {result}") elif isinstance(result, tuple) and len(result) == 2: text_value, _ = result final_search_results.append(cast(str, text_value) if isinstance(text_value, str) else str(text_value)) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 47b7aa6d568..101cbae23f9 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -679,7 +679,7 @@ def _map_replicate_exception( ) raise APIError( status_code=500, - message=f"ReplicateException - {original_exception!s}", + message=f"ReplicateException - {original_exception}", llm_provider="replicate", model=model, request=httpx.Request( @@ -2459,7 +2459,7 @@ def exception_type( # type: ignore ): # deal with edge-case invalid request error bug in openai-python sdk exception_mapping_worked = True raise BadRequestError( - message=f"{exception_provider} BadRequestError : This can happen due to missing AZURE_API_VERSION: {original_exception!s}", + message=f"{exception_provider} BadRequestError : This can happen due to missing AZURE_API_VERSION: {original_exception}", model=model, llm_provider=custom_llm_provider, response=getattr(original_exception, "response", None), @@ -2478,7 +2478,7 @@ def exception_type( # type: ignore ) else: raise APIConnectionError( - message=f"{original_exception!s}\n{_redact_string(traceback.format_exc())}", + message=f"{original_exception}\n{_redact_string(traceback.format_exc())}", llm_provider=custom_llm_provider, model=model, request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), # stub the request diff --git a/litellm/litellm_core_utils/fallback_utils.py b/litellm/litellm_core_utils/fallback_utils.py index ff4a4c9c74c..4e7ce828a58 100644 --- a/litellm/litellm_core_utils/fallback_utils.py +++ b/litellm/litellm_core_utils/fallback_utils.py @@ -70,7 +70,7 @@ async def async_completion_with_fallbacks(**kwargs): ) except Exception as e: - verbose_logger.exception(f"Fallback attempt failed for model {model}: {e!s}") + verbose_logger.exception(f"Fallback attempt failed for model {model}: {e}") most_recent_exception_str = str(e) continue diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index f869909e751..32e517883b7 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -501,9 +501,9 @@ def get_llm_provider( if isinstance(e, litellm.exceptions.BadRequestError): raise e else: - error_str = f"GetLLMProvider Exception - {e!s}\n\noriginal model: {model}" + error_str = f"GetLLMProvider Exception - {e}\n\noriginal model: {model}" raise litellm.exceptions.BadRequestError( # type: ignore - message=f"GetLLMProvider Exception - {e!s}\n\noriginal model: {model}", + message=f"GetLLMProvider Exception - {e}\n\noriginal model: {model}", model=model, response=None, llm_provider="", diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 0addc7586fe..e87e3d8aca2 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -292,7 +292,7 @@ def get_model_cost_map(url: str) -> dict: str(e), ) _cost_map_source_info.source = "local" - _cost_map_source_info.fallback_reason = f"Remote fetch failed: {e!s}" + _cost_map_source_info.fallback_reason = f"Remote fetch failed: {e}" return _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()) # Validate using cached count (cheap int comparison, no file I/O) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index db10e18e324..b00130653c5 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -199,7 +199,7 @@ try: EnterpriseStandardLoggingPayloadSetup ) except Exception as e: - verbose_logger.debug(f"[Non-Blocking] Unable to import GenericAPILogger - LiteLLM Enterprise Feature - {e!s}") + verbose_logger.debug(f"[Non-Blocking] Unable to import GenericAPILogger - LiteLLM Enterprise Feature - {e}") GenericAPILogger = CustomLogger # type: ignore ResendEmailLogger = CustomLogger # type: ignore SendGridEmailLogger = CustomLogger # type: ignore @@ -968,7 +968,7 @@ class Logging(LiteLLMLoggingBaseClass): error=str(e), ) _metadata["raw_request"] = f"Unable to Log \ - raw request: {e!s}" + raw request: {e}" if getattr(self, "logger_fn", None) and callable(self.logger_fn): try: self.logger_fn( @@ -976,7 +976,7 @@ class Logging(LiteLLMLoggingBaseClass): ) # Expectation: any logger function passed in by the user should accept a dict object except Exception as e: verbose_logger.exception( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {e!s}" + f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {e}" ) self.model_call_details["api_call_start_time"] = datetime.datetime.now() @@ -1036,14 +1036,14 @@ class Logging(LiteLLMLoggingBaseClass): callback_func=callback, ) except Exception as e: - verbose_logger.exception(f"litellm.Logging.pre_call(): Exception occured - {e!s}") + verbose_logger.exception(f"litellm.Logging.pre_call(): Exception occured - {e}") verbose_logger.debug( f"LiteLLM.Logging: is sentry capture exception initialized {capture_exception}" ) if capture_exception: # log this error to sentry for debugging capture_exception(e) except Exception as e: - verbose_logger.exception(f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {e!s}") + verbose_logger.exception(f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {e}") verbose_logger.error(f"LiteLLM.Logging: is sentry capture exception initialized {capture_exception}") if capture_exception: # log this error to sentry for debugging capture_exception(e) @@ -1159,7 +1159,7 @@ class Logging(LiteLLMLoggingBaseClass): ) # Expectation: any logger function passed in by the user should accept a dict object except Exception as e: verbose_logger.exception( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {e!s}" + f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {e}" ) original_response = redact_message_input_output_from_logging( model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), @@ -1196,7 +1196,7 @@ class Logging(LiteLLMLoggingBaseClass): ) except Exception as e: verbose_logger.exception( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while post-call logging with integrations {e!s}" + f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while post-call logging with integrations {e}" ) verbose_logger.debug( f"LiteLLM.Logging: is sentry capture exception initialized {capture_exception}" @@ -1204,7 +1204,7 @@ class Logging(LiteLLMLoggingBaseClass): if capture_exception: # log this error to sentry for debugging capture_exception(e) except Exception as e: - verbose_logger.exception(f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {e!s}") + verbose_logger.exception(f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {e}") async def async_post_mcp_tool_call_hook( self, @@ -1244,7 +1244,7 @@ class Logging(LiteLLMLoggingBaseClass): if response is not None: response_obj = self._parse_post_mcp_call_hook_response(response=response) except Exception as e: - verbose_logger.exception(f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {e!s}") + verbose_logger.exception(f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {e}") return response_obj def _parse_post_mcp_call_hook_response(self, response: MCPPostCallResponseObject | None) -> Any: @@ -1889,7 +1889,7 @@ class Logging(LiteLLMLoggingBaseClass): return start_time, end_time, result except Exception as e: - raise Exception(f"[Non-Blocking] LiteLLM.Success_Call Error: {e!s}") + raise Exception(f"[Non-Blocking] LiteLLM.Success_Call Error: {e}") def _is_recognized_call_type_for_logging( self, @@ -2378,7 +2378,7 @@ class Logging(LiteLLMLoggingBaseClass): pass except Exception as e: verbose_logger.exception( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging {e!s}", + f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging {e}", ) async def async_success_handler(self, result=None, start_time=None, end_time=None, cache_hit=None, **kwargs): @@ -2694,7 +2694,7 @@ class Logging(LiteLLMLoggingBaseClass): break # Only increment once except Exception as e: - verbose_logger.debug(f"Error in _handle_callback_failure: {e!s}") + verbose_logger.debug(f"Error in _handle_callback_failure: {e}") def _failure_handler_helper_fn(self, exception, traceback_exception, start_time=None, end_time=None): if start_time is None: @@ -2931,14 +2931,14 @@ class Logging(LiteLLMLoggingBaseClass): except Exception as e: print_verbose( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure logging with integrations {e!s}" + f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure logging with integrations {e}" ) print_verbose(f"LiteLLM.Logging: is sentry capture exception initialized {capture_exception}") if capture_exception: # log this error to sentry for debugging capture_exception(e) except Exception as e: verbose_logger.exception( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure logging {e!s}" + f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure logging {e}" ) async def async_failure_handler(self, exception, traceback_exception, start_time=None, end_time=None): @@ -2995,7 +2995,7 @@ class Logging(LiteLLMLoggingBaseClass): except Exception as e: verbose_logger.exception( f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure \ - logging {e!s}\nCallback={callback}" + logging {e}\nCallback={callback}" ) # Track callback logging failures in Prometheus self._handle_callback_failure(callback=callback) @@ -5426,7 +5426,7 @@ def get_standard_logging_object_payload( return payload except Exception as e: - verbose_logger.exception(f"Error creating standard logging object - {e!s}") + verbose_logger.exception(f"Error creating standard logging object - {e}") return None diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 5bc6107dbec..face1d1b49f 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -150,7 +150,7 @@ def _generic_cost_per_character( prompt_cost = prompt_characters * custom_prompt_cost except Exception as e: verbose_logger.exception( - f"litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - {e!s}\nDefaulting to None" + f"litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - {e}\nDefaulting to None" ) prompt_cost = None @@ -165,7 +165,7 @@ def _generic_cost_per_character( completion_cost = completion_characters * custom_completion_cost except Exception as e: verbose_logger.exception( - f"litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - {e!s}\nDefaulting to None" + f"litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - {e}\nDefaulting to None" ) completion_cost = None diff --git a/litellm/litellm_core_utils/llm_response_utils/get_api_base.py b/litellm/litellm_core_utils/llm_response_utils/get_api_base.py index 5e332f4c8d6..1982e40448d 100644 --- a/litellm/litellm_core_utils/llm_response_utils/get_api_base.py +++ b/litellm/litellm_core_utils/llm_response_utils/get_api_base.py @@ -53,7 +53,7 @@ def get_api_base(model: str, optional_params: dict | LiteLLM_Params) -> str | No api_key=_optional_params.api_key, ) except Exception as e: - verbose_logger.debug(f"Error occurred in getting api base - {e!s}") + verbose_logger.debug(f"Error occurred in getting api base - {e}") custom_llm_provider = None dynamic_api_base = None diff --git a/litellm/litellm_core_utils/logging_utils.py b/litellm/litellm_core_utils/logging_utils.py index 32e2abc53b0..9340554b6d9 100644 --- a/litellm/litellm_core_utils/logging_utils.py +++ b/litellm/litellm_core_utils/logging_utils.py @@ -178,7 +178,7 @@ def _get_parent_otel_span_from_logging_obj( return _get_parent_otel_span_from_kwargs(logging_obj.model_call_details) except Exception as e: - verbose_logger.exception(f"Error in _get_parent_otel_span_from_logging_obj: {e!s}") + verbose_logger.exception(f"Error in _get_parent_otel_span_from_logging_obj: {e}") return None @@ -265,7 +265,7 @@ def _set_duration_in_model_call_details( else: verbose_logger.debug("`logging_obj` not found - unable to track `llm_api_duration_ms") except Exception as e: - verbose_logger.warning(f"Error setting `llm_api_duration_ms`: {e!s}") + verbose_logger.warning(f"Error setting `llm_api_duration_ms`: {e}") def track_llm_api_timing(): @@ -321,7 +321,7 @@ def track_llm_api_timing(): ) ) except Exception as e: - verbose_logger.debug(f"Error in service logging: {e!s}") + verbose_logger.debug(f"Error in service logging: {e}") @functools.wraps(func) def sync_wrapper(*args, **kwargs): @@ -366,7 +366,7 @@ def track_llm_api_timing(): parent_otel_span=parent_otel_span, ) except Exception as e: - verbose_logger.debug(f"Error in service logging: {e!s}") + verbose_logger.debug(f"Error in service logging: {e}") # Check if the function is async or sync if inspect.iscoroutinefunction(func): diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 639c93dfb80..90c9fb05e4c 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1683,7 +1683,7 @@ def parse_tool_call_arguments( if context: error_parts.append(f"({context})") - error_message = " ".join(error_parts) + f". Error: {original_error!s}. Arguments: {arguments}" + error_message = " ".join(error_parts) + f". Error: {original_error}. Arguments: {arguments}" raise ValueError(error_message) from original_error diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 147280af1b1..8aa4f60b7c5 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -438,9 +438,7 @@ def _render_chat_template(env, chat_template: str, bos_token: str, eos_token: st return rendered_text except Exception as e: - raise Exception( - f"Error rendering template - {e!s}" - ) # don't use verbose_logger.exception, if exception is raised + raise Exception(f"Error rendering template - {e}") # don't use verbose_logger.exception, if exception is raised async def _afetch_and_extract_template( @@ -858,7 +856,7 @@ def convert_to_anthropic_image_obj(openai_image_url: str, format: str | None) -> raise except Exception as e: raise Exception( - f"""Image url not in expected format. Example Expected input - "image_url": "data:image/jpeg;base64,{{base64_image}}". Supported formats - ['image/jpeg', 'image/png', 'image/gif', 'image/webp']. Error: {e!s}""" + f"""Image url not in expected format. Example Expected input - "image_url": "data:image/jpeg;base64,{{base64_image}}". Supported formats - ['image/jpeg', 'image/png', 'image/gif', 'image/webp']. Error: {e}""" ) @@ -1361,7 +1359,7 @@ def convert_to_gemini_tool_call_invoke( ) return _parts_list except Exception as e: - raise Exception(f"Unable to convert openai tool calls={message} to gemini tool calls. Received error={e!s}") + raise Exception(f"Unable to convert openai tool calls={message} to gemini tool calls. Received error={e}") def convert_to_gemini_tool_call_result( @@ -3713,7 +3711,7 @@ def _convert_to_bedrock_tool_call_invoke( _parts_list.append(cache_point_block) return _parts_list except Exception as e: - raise Exception(f"Unable to convert openai tool calls={tool_calls} to bedrock tool calls. Received error={e!s}") + raise Exception(f"Unable to convert openai tool calls={tool_calls} to bedrock tool calls. Received error={e}") def _append_bedrock_tool_result_media_block( diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index fb7d06bee93..25155068baa 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -618,7 +618,7 @@ class CustomStreamWrapper: else: return "" except Exception as e: - verbose_logger.exception(f"litellm.CustomStreamWrapper.handle_baseten_chunk(): Exception occured - {e!s}") + verbose_logger.exception(f"litellm.CustomStreamWrapper.handle_baseten_chunk(): Exception occured - {e}") return "" def handle_triton_stream(self, chunk): @@ -1179,7 +1179,7 @@ class CustomStreamWrapper: content=None, tool_calls=[ { - "id": f"call_{uuid.uuid4()!s}", + "id": f"call_{uuid.uuid4()}", "function": { "arguments": args_str, "name": function_call.name, @@ -1204,7 +1204,7 @@ class CustomStreamWrapper: ) except Exception: if chunk.candidates[0].finish_reason.name == "SAFETY": # type: ignore - raise Exception(f"The response was blocked by VertexAI. {chunk!s}") + raise Exception(f"The response was blocked by VertexAI. {chunk}") else: completion_obj["content"] = str(chunk) elif self.custom_llm_provider == "petals": @@ -1430,7 +1430,7 @@ class CustomStreamWrapper: model_response.choices[0].delta = Delta(**_json_delta) except Exception as e: verbose_logger.exception( - f"litellm.CustomStreamWrapper.chunk_creator(): Exception occured - {e!s}" + f"litellm.CustomStreamWrapper.chunk_creator(): Exception occured - {e}" ) model_response.choices[0].delta = Delta() elif self._has_any_special_delta_attributes(delta): @@ -1538,7 +1538,7 @@ class CustomStreamWrapper: except Exception as e: from litellm._logging import verbose_logger - verbose_logger.exception(f"Error in post-call streaming deployment hook: {e!s}") + verbose_logger.exception(f"Error in post-call streaming deployment hook: {e}") return chunk def _add_mcp_list_tools_to_first_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream: @@ -1578,7 +1578,7 @@ class CustomStreamWrapper: except Exception as e: from litellm._logging import verbose_logger - verbose_logger.exception(f"Error adding MCP list tools to first chunk: {e!s}") + verbose_logger.exception(f"Error adding MCP list tools to first chunk: {e}") return chunk @@ -1615,7 +1615,7 @@ class CustomStreamWrapper: except Exception as e: from litellm._logging import verbose_logger - verbose_logger.exception(f"Error adding MCP metadata to final chunk: {e!s}") + verbose_logger.exception(f"Error adding MCP metadata to final chunk: {e}") return chunk diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index fbd19b43f3e..ff94965f628 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -104,7 +104,7 @@ def get_modified_max_tokens( return user_max_tokens except Exception as e: verbose_logger.debug( - f"litellm.litellm_core_utils.token_counter.py::get_modified_max_tokens() - Error while checking max token limit: {e!s}\nmodel={model}, base_model={base_model}" + f"litellm.litellm_core_utils.token_counter.py::get_modified_max_tokens() - Error while checking max token limit: {e}\nmodel={model}, base_model={base_model}" ) return user_max_tokens diff --git a/litellm/llms/a2a/chat/transformation.py b/litellm/llms/a2a/chat/transformation.py index 3de584d1d5f..967fcc354a5 100644 --- a/litellm/llms/a2a/chat/transformation.py +++ b/litellm/llms/a2a/chat/transformation.py @@ -279,7 +279,7 @@ class A2AConfig(BaseConfig): except Exception as e: raise A2AError( status_code=raw_response.status_code, - message=f"Failed to parse A2A response: {e!s}", + message=f"Failed to parse A2A response: {e}", headers=dict(raw_response.headers), ) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 40d1dbac187..51b862e79d9 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1875,7 +1875,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): except Exception as e: raise AnthropicError( status_code=400, - message=f"{e!s}\nReceived Messages={messages}", + message=f"{e}\nReceived Messages={messages}", ) # don't use verbose_logger.exception, if exception is raised ## Auto-strip advisor blocks from history if advisor tool is absent. @@ -2454,7 +2454,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): except Exception as e: response_headers = getattr(raw_response, "headers", None) raise AnthropicError( - message=f"Unable to get json response - {e!s}, Original Response: {raw_response.text}", + message=f"Unable to get json response - {e}, Original Response: {raw_response.text}", status_code=raw_response.status_code, headers=response_headers, ) diff --git a/litellm/llms/anthropic/count_tokens/handler.py b/litellm/llms/anthropic/count_tokens/handler.py index b1584b98456..0c3d0e931a2 100644 --- a/litellm/llms/anthropic/count_tokens/handler.py +++ b/litellm/llms/anthropic/count_tokens/handler.py @@ -109,14 +109,14 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): raise except httpx.HTTPStatusError as e: # HTTP errors - preserve the actual status code - verbose_logger.error(f"HTTP error in CountTokens handler: {e!s}") + verbose_logger.error(f"HTTP error in CountTokens handler: {e}") raise AnthropicError( status_code=e.response.status_code, message=e.response.text, ) except Exception as e: - verbose_logger.error(f"Error in CountTokens handler: {e!s}") + verbose_logger.error(f"Error in CountTokens handler: {e}") raise AnthropicError( status_code=500, - message=f"CountTokens processing error: {e!s}", + message=f"CountTokens processing error: {e}", ) diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index 373460c151d..a04bb29d7a5 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -684,7 +684,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): except json.JSONDecodeError as json_error: raise AzureOpenAIError( status_code=raw_response.status_code or 500, - message=f"Failed to parse raw Azure embedding response: {json_error!s}", + message=f"Failed to parse raw Azure embedding response: {json_error}", ) from json_error if isinstance(response, str): raise AzureOpenAIError( diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index dcbd3985dfd..8e0bd363a8a 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -333,7 +333,7 @@ def get_azure_ad_token( verbose_logger.debug("Azure AD Token Provider could not be used.") except Exception as e: verbose_logger.error( - f"Error calling Azure AD token provider: {e!s}. Follow docs - https://docs.litellm.ai/docs/providers/azure/#azure-ad-token-refresh---defaultazurecredential" + f"Error calling Azure AD token provider: {e}. Follow docs - https://docs.litellm.ai/docs/providers/azure/#azure-ad-token-refresh---defaultazurecredential" ) raise e @@ -359,8 +359,8 @@ def get_azure_ad_token( # Re-raise TypeError directly raise except Exception as e: - verbose_logger.error(f"Error calling Azure AD token provider: {e!s}") - raise RuntimeError(f"Failed to get Azure AD token: {e!s}") from e + verbose_logger.error(f"Error calling Azure AD token provider: {e}") + raise RuntimeError(f"Failed to get Azure AD token: {e}") from e return azure_ad_token @@ -393,7 +393,7 @@ class BaseAzureLLM(BaseOpenAILLM): verbose_logger.debug("Successfully obtained Azure AD token provider using DefaultAzureCredential") return azure_ad_token_provider except Exception as e: - verbose_logger.debug(f"DefaultAzureCredential failed: {e!s}") + verbose_logger.debug(f"DefaultAzureCredential failed: {e}") return None def get_azure_openai_client( @@ -580,7 +580,7 @@ class BaseAzureLLM(BaseOpenAILLM): # only show first 5 chars of api_key _api_key = _api_key[:8] + "*" * 15 verbose_logger.debug( - f"Initializing Azure OpenAI Client for {model_name}, Api Base: {api_base!s}, Api Key:{_api_key}" + f"Initializing Azure OpenAI Client for {model_name}, Api Base: {api_base}, Api Key:{_api_key}" ) azure_client_params = { "api_key": api_key, diff --git a/litellm/llms/azure_ai/agents/handler.py b/litellm/llms/azure_ai/agents/handler.py index b12f2203e51..7023dbca0b8 100644 --- a/litellm/llms/azure_ai/agents/handler.py +++ b/litellm/llms/azure_ai/agents/handler.py @@ -193,7 +193,7 @@ class AzureAIAgentsHandler: ), ) except Exception as e: - verbose_logger.warning(f"Failed to calculate token usage: {e!s}") + verbose_logger.warning(f"Failed to calculate token usage: {e}") return model_response diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py index 3ac04729267..65d8c0182ee 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py @@ -114,14 +114,14 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): raise except httpx.HTTPStatusError as e: # HTTP errors - preserve the actual status code - verbose_logger.error(f"HTTP error in CountTokens handler: {e!s}") + verbose_logger.error(f"HTTP error in CountTokens handler: {e}") raise AnthropicError( status_code=e.response.status_code, message=e.response.text, ) except Exception as e: - verbose_logger.error(f"Error in CountTokens handler: {e!s}") + verbose_logger.error(f"Error in CountTokens handler: {e}") raise AnthropicError( status_code=500, - message=f"CountTokens processing error: {e!s}", + message=f"CountTokens processing error: {e}", ) diff --git a/litellm/llms/azure_ai/vector_stores/transformation.py b/litellm/llms/azure_ai/vector_stores/transformation.py index 88a38fc1ec7..943232dc348 100644 --- a/litellm/llms/azure_ai/vector_stores/transformation.py +++ b/litellm/llms/azure_ai/vector_stores/transformation.py @@ -132,7 +132,7 @@ class AzureAIVectorStoreConfig(BaseVectorStoreConfig, BaseAzureLLM): ) query_vector = embedding_response.data[0]["embedding"] except Exception as e: - raise Exception(f"Failed to generate embedding for query: {e!s}") + raise Exception(f"Failed to generate embedding for query: {e}") # Azure AI Search endpoint for search index_name = vector_store_id # vector_store_id is the index name diff --git a/litellm/llms/base_llm/files/azure_blob_storage_backend.py b/litellm/llms/base_llm/files/azure_blob_storage_backend.py index 7c76003de3a..33255657287 100644 --- a/litellm/llms/base_llm/files/azure_blob_storage_backend.py +++ b/litellm/llms/base_llm/files/azure_blob_storage_backend.py @@ -133,7 +133,7 @@ class AzureBlobStorageBackend(BaseFileStorageBackend, AzureBlobStorageLogger): return storage_url except Exception as e: - verbose_logger.exception(f"Error uploading file to Azure Blob Storage: {e!s}") + verbose_logger.exception(f"Error uploading file to Azure Blob Storage: {e}") raise async def _upload_file_with_account_key(self, file_content: bytes, full_path: str) -> str: @@ -247,7 +247,7 @@ class AzureBlobStorageBackend(BaseFileStorageBackend, AzureBlobStorageLogger): return await self._download_file_with_azure_ad(file_path) except Exception as e: - verbose_logger.exception(f"Error downloading file from Azure Blob Storage: {e!s}") + verbose_logger.exception(f"Error downloading file from Azure Blob Storage: {e}") raise async def _download_file_with_account_key(self, file_path: str) -> bytes: diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index 40b12e17e8a..d6626562393 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -186,7 +186,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): return session_id # Generate a session ID with 33+ characters - generated_id = f"litellm-session-{uuid.uuid4()!s}" + generated_id = f"litellm-session-{uuid.uuid4()}" verbose_logger.debug(f"Generated new session ID: {generated_id}") return generated_id @@ -370,7 +370,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): total_tokens=total_tokens, ) except Exception as e: - verbose_logger.warning(f"Failed to calculate token usage: {e!s}") + verbose_logger.warning(f"Failed to calculate token usage: {e}") return None def _parse_json_response(self, response_json: dict) -> AgentCoreParsedResponse: @@ -1023,9 +1023,9 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): return model_response except Exception as e: - verbose_logger.error(f"Error processing Bedrock AgentCore response: {e!s}") + verbose_logger.error(f"Error processing Bedrock AgentCore response: {e}") raise BedrockError( - message=f"Error processing response: {e!s}", + message=f"Error processing response: {e}", status_code=raw_response.status_code, ) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 5bd498a465e..2b34c9f2654 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -2073,7 +2073,7 @@ class AmazonConverseConfig(BaseConfig): completion_response = ConverseResponseBlock(**response.json()) # type: ignore except Exception as e: raise BedrockError( - message=f"Error converting to valid response block={e!s}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues", + message=f"Error converting to valid response block={e}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues", status_code=422, ) diff --git a/litellm/llms/bedrock/chat/invoke_agent/transformation.py b/litellm/llms/bedrock/chat/invoke_agent/transformation.py index d877ca81244..da6224ec487 100644 --- a/litellm/llms/bedrock/chat/invoke_agent/transformation.py +++ b/litellm/llms/bedrock/chat/invoke_agent/transformation.py @@ -464,9 +464,9 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): ) except Exception as e: - verbose_logger.error(f"Error processing Bedrock Invoke Agent response: {e!s}") + verbose_logger.error(f"Error processing Bedrock Invoke Agent response: {e}") raise BedrockError( - message=f"Error processing response: {e!s}", + message=f"Error processing response: {e}", status_code=raw_response.status_code, ) diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index d069929df92..4a429b639d2 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -590,7 +590,7 @@ class AWSEventStreamDecoder: return response except Exception as e: - raise Exception(f"Received streaming error - {e!s}") + raise Exception(f"Received streaming error - {e}") def _chunk_parser(self, chunk_data: dict) -> Union[GChunk, ModelResponseStream, dict]: text = "" diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py index b96756f1e4e..8de9c3de3f2 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py @@ -208,7 +208,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig): completion_response = raw_response.json() except Exception as e: raise BedrockError( - message=f"Error parsing response: {raw_response.text}, error: {e!s}", + message=f"Error parsing response: {raw_response.text}, error: {e}", status_code=raw_response.status_code, ) @@ -237,7 +237,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig): raise Exception("Unable to set message content") except Exception as e: raise BedrockError( - message=f"Error setting response content: {e!s}. Response: {completion_response}", + message=f"Error setting response content: {e}. Response: {completion_response}", status_code=raw_response.status_code, ) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 0c6436030af..a54bf8d6b2b 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -356,7 +356,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): outputText = completion_response.get("results")[0].get("outputText") except Exception as e: raise BedrockError( - message=f"Error processing={raw_response.text}, Received error={e!s}", + message=f"Error processing={raw_response.text}, Received error={e}", status_code=422, ) @@ -379,7 +379,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): raise Exception() except Exception as e: raise BedrockError( - message=f"Error parsing received text={outputText}.\nError-{e!s}", + message=f"Error parsing received text={outputText}.\nError-{e}", status_code=raw_response.status_code, ) diff --git a/litellm/llms/bedrock/count_tokens/handler.py b/litellm/llms/bedrock/count_tokens/handler.py index 8e993c6f8b2..44cc535385d 100644 --- a/litellm/llms/bedrock/count_tokens/handler.py +++ b/litellm/llms/bedrock/count_tokens/handler.py @@ -120,14 +120,14 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): raise except httpx.HTTPStatusError as e: # HTTP errors - preserve the actual status code - verbose_logger.error(f"HTTP error in CountTokens handler: {e!s}") + verbose_logger.error(f"HTTP error in CountTokens handler: {e}") raise BedrockError( status_code=e.response.status_code, message=e.response.text, ) except Exception as e: - verbose_logger.error(f"Error in CountTokens handler: {e!s}") + verbose_logger.error(f"Error in CountTokens handler: {e}") raise BedrockError( status_code=500, - message=f"CountTokens processing error: {e!s}", + message=f"CountTokens processing error: {e}", ) diff --git a/litellm/llms/bedrock/files/handler.py b/litellm/llms/bedrock/files/handler.py index 12ebc52dff3..8f590bd917c 100644 --- a/litellm/llms/bedrock/files/handler.py +++ b/litellm/llms/bedrock/files/handler.py @@ -130,7 +130,7 @@ class BedrockFilesHandler(BaseAWSLLM): response = s3_client.get_object(Bucket=bucket_name, Key=object_key) file_content = response["Body"].read() except Exception as e: - raise ValueError(f"Failed to download file from S3: {s3_uri}. Error: {e!s}") + raise ValueError(f"Failed to download file from S3: {s3_uri}. Error: {e}") # Create mock HTTP response mock_response = httpx.Response( diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 3656088cb9d..d3e61829681 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -652,7 +652,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): ) except Exception as e: verbose_logger.exception( - f"litellm.llms.bedrock.files.transformation.py::_transform_openai_jsonl_content_to_bedrock_jsonl_content() - Error inferring custom_llm_provider - {e!s}" + f"litellm.llms.bedrock.files.transformation.py::_transform_openai_jsonl_content_to_bedrock_jsonl_content() - Error inferring custom_llm_provider - {e}" ) # Determine provider from model name diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index 17007f48fb0..a8969894dda 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -175,7 +175,7 @@ class BedrockRealtime(BaseAWSLLM): except Exception as e: verbose_proxy_logger.exception(f"Error in BedrockRealtime.async_realtime: {e}") try: - await websocket.close(code=1011, reason=_redact_string(f"Internal error: {e!s}")) + await websocket.close(code=1011, reason=_redact_string(f"Internal error: {e}")) except Exception: pass raise diff --git a/litellm/llms/black_forest_labs/image_edit/handler.py b/litellm/llms/black_forest_labs/image_edit/handler.py index 62aaa6da77a..cf0cc31283b 100644 --- a/litellm/llms/black_forest_labs/image_edit/handler.py +++ b/litellm/llms/black_forest_labs/image_edit/handler.py @@ -159,7 +159,7 @@ class BlackForestLabsImageEdit: except Exception as e: raise BlackForestLabsError( status_code=500, - message=f"Request failed: {e!s}", + message=f"Request failed: {e}", ) # Poll for result @@ -262,7 +262,7 @@ class BlackForestLabsImageEdit: except Exception as e: raise BlackForestLabsError( status_code=500, - message=f"Request failed: {e!s}", + message=f"Request failed: {e}", ) # Poll for result diff --git a/litellm/llms/black_forest_labs/image_generation/handler.py b/litellm/llms/black_forest_labs/image_generation/handler.py index af321fad580..054d28003f1 100644 --- a/litellm/llms/black_forest_labs/image_generation/handler.py +++ b/litellm/llms/black_forest_labs/image_generation/handler.py @@ -156,7 +156,7 @@ class BlackForestLabsImageGeneration: except Exception as e: raise BlackForestLabsError( status_code=500, - message=f"Request failed: {e!s}", + message=f"Request failed: {e}", ) # Poll for result @@ -262,7 +262,7 @@ class BlackForestLabsImageGeneration: except Exception as e: raise BlackForestLabsError( status_code=500, - message=f"Request failed: {e!s}", + message=f"Request failed: {e}", ) # Poll for result diff --git a/litellm/llms/clarifai/chat/transformation.py b/litellm/llms/clarifai/chat/transformation.py index 147c7986f2a..bf922893f13 100644 --- a/litellm/llms/clarifai/chat/transformation.py +++ b/litellm/llms/clarifai/chat/transformation.py @@ -106,7 +106,7 @@ class ClarifaiConfig(OpenAIGPTConfig): except Exception as e: raise OpenAIError( status_code=raw_response.status_code, - message=f"Failed to parse Clarifai response: {e!s}", + message=f"Failed to parse Clarifai response: {e}", headers=raw_response.headers, ) from e diff --git a/litellm/llms/codestral/completion/handler.py b/litellm/llms/codestral/completion/handler.py index 1261604e6a7..eb4b8acd71f 100644 --- a/litellm/llms/codestral/completion/handler.py +++ b/litellm/llms/codestral/completion/handler.py @@ -356,7 +356,7 @@ class CodestralTextCompletion: ) except Exception as e: raise TextCompletionCodestralError( - status_code=500, message=f"{e!s}" + status_code=500, message=f"{e}" ) # don't use verbose_logger.exception, if exception is raised return self.process_text_completion_response( model=model, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index a203f0d6c8c..f7bf174f9ac 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -5659,7 +5659,7 @@ class BaseLLMHTTPHandler: fingerprint=fingerprint, ) except Exception as e: - verbose_logger.exception(f"LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: {e!s}") + verbose_logger.exception(f"LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: {e}") # Check if we need to convert response to fake stream for chat completions # This happens when: @@ -5906,7 +5906,7 @@ class BaseLLMHTTPHandler: except Exception as e: verbose_logger.exception(f"Error connecting to backend: {e}") try: - await websocket.close(code=1011, reason=_redact_string(f"Internal server error: {e!s}")) + await websocket.close(code=1011, reason=_redact_string(f"Internal server error: {e}")) except RuntimeError as close_error: if "already completed" in str(close_error) or "websocket.close" in str(close_error): # The WebSocket is already closed or the response is completed, so we can ignore this error @@ -6303,7 +6303,7 @@ class BaseLLMHTTPHandler: except Exception as e: verbose_logger.exception(f"Error in responses WS: {e}") try: - await websocket.close(code=1011, reason=_redact_string(f"Internal server error: {e!s}")) + await websocket.close(code=1011, reason=_redact_string(f"Internal server error: {e}")) except RuntimeError as close_error: if "already completed" in str(close_error) or "websocket.close" in str(close_error): pass diff --git a/litellm/llms/dashscope/embed/transformation.py b/litellm/llms/dashscope/embed/transformation.py index 55722ce35d1..870c96edb65 100644 --- a/litellm/llms/dashscope/embed/transformation.py +++ b/litellm/llms/dashscope/embed/transformation.py @@ -130,7 +130,7 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig): except Exception as e: raise DashScopeError( status_code=raw_response.status_code, - message=f"Failed to parse DashScope response as JSON: {e!s}", + message=f"Failed to parse DashScope response as JSON: {e}", ) logging_obj.post_call( diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 9f6f669a264..b7c9dc23762 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -630,7 +630,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): except Exception as e: response_headers = getattr(raw_response, "headers", None) raise DatabricksException( - message=f"Unable to get json response - {e!s}, Original Response: {raw_response.text}", + message=f"Unable to get json response - {e}, Original Response: {raw_response.text}", status_code=raw_response.status_code, headers=response_headers, ) diff --git a/litellm/llms/databricks/common_utils.py b/litellm/llms/databricks/common_utils.py index 2fb7cacb9bf..62e2245db99 100644 --- a/litellm/llms/databricks/common_utils.py +++ b/litellm/llms/databricks/common_utils.py @@ -245,7 +245,7 @@ class DatabricksBase: except requests.RequestException as e: raise DatabricksException( status_code=500, - message=f"OAuth M2M token request failed: {e!s}", + message=f"OAuth M2M token request failed: {e}", ) if response.status_code != 200: diff --git a/litellm/llms/deepgram/audio_transcription/transformation.py b/litellm/llms/deepgram/audio_transcription/transformation.py index 034c41c79fb..4c21f6eb3c7 100644 --- a/litellm/llms/deepgram/audio_transcription/transformation.py +++ b/litellm/llms/deepgram/audio_transcription/transformation.py @@ -122,7 +122,7 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): return response except Exception as e: - raise ValueError(f"Error transforming Deepgram response: {e!s}\nResponse: {raw_response.text}") + raise ValueError(f"Error transforming Deepgram response: {e}\nResponse: {raw_response.text}") def _reconstruct_diarized_transcript(self, words: list) -> str: """ diff --git a/litellm/llms/elevenlabs/audio_transcription/transformation.py b/litellm/llms/elevenlabs/audio_transcription/transformation.py index a33e221dafd..3672d080b22 100644 --- a/litellm/llms/elevenlabs/audio_transcription/transformation.py +++ b/litellm/llms/elevenlabs/audio_transcription/transformation.py @@ -144,7 +144,7 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig): return response except Exception as e: - raise ValueError(f"Error transforming ElevenLabs response: {e!s}\nResponse: {raw_response.text}") + raise ValueError(f"Error transforming ElevenLabs response: {e}\nResponse: {raw_response.text}") def get_complete_url( self, diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 9fcb81e00e3..64ef731a0e2 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -542,7 +542,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): except Exception as e: response_headers = getattr(raw_response, "headers", None) raise FireworksAIException( - message=f"Unable to get json response - {e!s}, Original Response: {raw_response.text}", + message=f"Unable to get json response - {e}, Original Response: {raw_response.text}", status_code=raw_response.status_code, headers=response_headers, ) diff --git a/litellm/llms/fireworks_ai/rerank/transformation.py b/litellm/llms/fireworks_ai/rerank/transformation.py index 7979eeeba42..c8a79878b56 100644 --- a/litellm/llms/fireworks_ai/rerank/transformation.py +++ b/litellm/llms/fireworks_ai/rerank/transformation.py @@ -178,7 +178,7 @@ class FireworksAIRerankConfig(FireworksAIMixin, BaseRerankConfig): raw_response_json = raw_response.json() except Exception as e: raise self.get_error_class( - error_message=f"Failed to parse response: {e!s}", + error_message=f"Failed to parse response: {e}", status_code=raw_response.status_code, headers=raw_response.headers, ) diff --git a/litellm/llms/gdc/chat/transformation.py b/litellm/llms/gdc/chat/transformation.py index 0416d246ea1..69056075a9d 100644 --- a/litellm/llms/gdc/chat/transformation.py +++ b/litellm/llms/gdc/chat/transformation.py @@ -220,7 +220,7 @@ class GDCGeminiConfig(OpenAILikeChatConfig): AttributeError, ) as e: raise litellm.utils.AuthenticationError( - message=f"Failed to load service account credentials from api_key: {e!s}", + message=f"Failed to load service account credentials from api_key: {e}", llm_provider="gdc", model=model, ) from e diff --git a/litellm/llms/gemini/count_tokens/handler.py b/litellm/llms/gemini/count_tokens/handler.py index ed82a37e47b..25f767a348e 100644 --- a/litellm/llms/gemini/count_tokens/handler.py +++ b/litellm/llms/gemini/count_tokens/handler.py @@ -155,8 +155,8 @@ class GoogleAIStudioTokenCounter: status_code=e.response.status_code, ) from e except httpx.RequestError as e: - error_msg = f"Request to Google Gen AI Studio failed: {e!s}" + error_msg = f"Request to Google Gen AI Studio failed: {e}" raise litellm.APIConnectionError(message=error_msg, llm_provider="gemini", model=model) from e except Exception as e: - error_msg = f"Unexpected error during token counting: {e!s}" + error_msg = f"Unexpected error during token counting: {e}" raise Exception(error_msg) from e diff --git a/litellm/llms/gemini/files/transformation.py b/litellm/llms/gemini/files/transformation.py index f91737ae613..89ac56979bb 100644 --- a/litellm/llms/gemini/files/transformation.py +++ b/litellm/llms/gemini/files/transformation.py @@ -190,8 +190,8 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): status_details=None, ) except Exception as e: - verbose_logger.exception(f"Error parsing file upload response: {e!s}") - raise ValueError(f"Error parsing file upload response: {e!s}") + verbose_logger.exception(f"Error parsing file upload response: {e}") + raise ValueError(f"Error parsing file upload response: {e}") def transform_retrieve_file_request( self, @@ -294,8 +294,8 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): status_details=(str(response_json.get("error", "")) if gemini_state == "FAILED" else None), ) except Exception as e: - verbose_logger.exception(f"Error parsing file retrieve response: {e!s}") - raise ValueError(f"Error parsing file retrieve response: {e!s}") + verbose_logger.exception(f"Error parsing file retrieve response: {e}") + raise ValueError(f"Error parsing file retrieve response: {e}") def transform_delete_file_request( self, @@ -362,8 +362,8 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): else: raise ValueError(f"Failed to delete file: {raw_response.text}") except Exception as e: - verbose_logger.exception(f"Error parsing file delete response: {e!s}") - raise ValueError(f"Error parsing file delete response: {e!s}") + verbose_logger.exception(f"Error parsing file delete response: {e}") + raise ValueError(f"Error parsing file delete response: {e}") def transform_list_files_request( self, diff --git a/litellm/llms/gemini/vector_stores/transformation.py b/litellm/llms/gemini/vector_stores/transformation.py index 051c0c544f5..9a823011289 100644 --- a/litellm/llms/gemini/vector_stores/transformation.py +++ b/litellm/llms/gemini/vector_stores/transformation.py @@ -256,7 +256,7 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig): except Exception as e: raise self.get_error_class( - error_message=f"Failed to parse Gemini response: {e!s}", + error_message=f"Failed to parse Gemini response: {e}", status_code=response.status_code, headers=response.headers, ) @@ -327,7 +327,7 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig): except Exception as e: raise self.get_error_class( - error_message=f"Failed to parse Gemini create response: {e!s}", + error_message=f"Failed to parse Gemini create response: {e}", status_code=response.status_code, headers=response.headers, ) diff --git a/litellm/llms/gigachat/authenticator.py b/litellm/llms/gigachat/authenticator.py index f5bced63869..356d438c6b2 100644 --- a/litellm/llms/gigachat/authenticator.py +++ b/litellm/llms/gigachat/authenticator.py @@ -177,7 +177,7 @@ def _request_token_sync( except httpx.RequestError as e: raise GigaChatAuthError( status_code=500, - message=f"GigaChat authentication request failed: {e!s}", + message=f"GigaChat authentication request failed: {e}", ) @@ -212,7 +212,7 @@ async def _request_token_async( except httpx.RequestError as e: raise GigaChatAuthError( status_code=500, - message=f"GigaChat authentication request failed: {e!s}", + message=f"GigaChat authentication request failed: {e}", ) diff --git a/litellm/llms/github_copilot/authenticator.py b/litellm/llms/github_copilot/authenticator.py index 2cb099edfb4..180c2215212 100644 --- a/litellm/llms/github_copilot/authenticator.py +++ b/litellm/llms/github_copilot/authenticator.py @@ -68,7 +68,7 @@ class Authenticator: verbose_logger.error("Error saving access token to file") return access_token except (GetDeviceCodeError, GetAccessTokenError, RefreshAPIKeyError) as e: - verbose_logger.warning(f"Failed attempt {attempt + 1}: {e!s}") + verbose_logger.warning(f"Failed attempt {attempt + 1}: {e}") continue raise GetAccessTokenError( @@ -100,7 +100,7 @@ class Authenticator: except OSError: verbose_logger.warning("No API key file found or error opening file") except (json.JSONDecodeError, KeyError) as e: - verbose_logger.warning(f"Error reading API key from file: {e!s}") + verbose_logger.warning(f"Error reading API key from file: {e}") except APIKeyExpiredError: pass # Already logged in the try block @@ -117,14 +117,14 @@ class Authenticator: status_code=401, ) except OSError as e: - verbose_logger.error(f"Error saving API key to file: {e!s}") + verbose_logger.error(f"Error saving API key to file: {e}") raise GetAPIKeyError( - message=f"Failed to save API key: {e!s}", + message=f"Failed to save API key: {e}", status_code=500, ) except RefreshAPIKeyError as e: raise GetAPIKeyError( - message=f"Failed to refresh API key: {e!s}", + message=f"Failed to refresh API key: {e}", status_code=401, ) @@ -142,7 +142,7 @@ class Authenticator: api_endpoint = endpoints.get("api") return api_endpoint except (OSError, json.JSONDecodeError, KeyError) as e: - verbose_logger.warning(f"Error reading API endpoint from file: {e!s}") + verbose_logger.warning(f"Error reading API endpoint from file: {e}") return None def _refresh_api_key(self) -> dict[str, Any]: @@ -173,9 +173,9 @@ class Authenticator: else: verbose_logger.warning(f"API key response missing token: {response_json}") except httpx.HTTPStatusError as e: - verbose_logger.error(f"HTTP error refreshing API key (attempt {attempt + 1}/{max_retries}): {e!s}") + verbose_logger.error(f"HTTP error refreshing API key (attempt {attempt + 1}/{max_retries}): {e}") except Exception as e: - verbose_logger.error(f"Unexpected error refreshing API key: {e!s}") + verbose_logger.error(f"Unexpected error refreshing API key: {e}") raise RefreshAPIKeyError( message="Failed to refresh API key after maximum retries", @@ -245,21 +245,21 @@ class Authenticator: return resp_json except httpx.HTTPStatusError as e: - verbose_logger.error(f"HTTP error getting device code: {e!s}") + verbose_logger.error(f"HTTP error getting device code: {e}") raise GetDeviceCodeError( - message=f"Failed to get device code: {e!s}", + message=f"Failed to get device code: {e}", status_code=400, ) except json.JSONDecodeError as e: - verbose_logger.error(f"Error decoding JSON response: {e!s}") + verbose_logger.error(f"Error decoding JSON response: {e}") raise GetDeviceCodeError( - message=f"Failed to decode device code response: {e!s}", + message=f"Failed to decode device code response: {e}", status_code=400, ) except Exception as e: - verbose_logger.error(f"Unexpected error getting device code: {e!s}") + verbose_logger.error(f"Unexpected error getting device code: {e}") raise GetDeviceCodeError( - message=f"Failed to get device code: {e!s}", + message=f"Failed to get device code: {e}", status_code=400, ) @@ -304,21 +304,21 @@ class Authenticator: else: verbose_logger.warning(f"Unexpected response: {resp_json}") except httpx.HTTPStatusError as e: - verbose_logger.error(f"HTTP error polling for access token: {e!s}") + verbose_logger.error(f"HTTP error polling for access token: {e}") raise GetAccessTokenError( - message=f"Failed to get access token: {e!s}", + message=f"Failed to get access token: {e}", status_code=400, ) except json.JSONDecodeError as e: - verbose_logger.error(f"Error decoding JSON response: {e!s}") + verbose_logger.error(f"Error decoding JSON response: {e}") raise GetAccessTokenError( - message=f"Failed to decode access token response: {e!s}", + message=f"Failed to decode access token response: {e}", status_code=400, ) except Exception as e: - verbose_logger.error(f"Unexpected error polling for access token: {e!s}") + verbose_logger.error(f"Unexpected error polling for access token: {e}") raise GetAccessTokenError( - message=f"Failed to get access token: {e!s}", + message=f"Failed to get access token: {e}", status_code=400, ) diff --git a/litellm/llms/huggingface/common_utils.py b/litellm/llms/huggingface/common_utils.py index 9dbdf05d0ec..07b580e68ce 100644 --- a/litellm/llms/huggingface/common_utils.py +++ b/litellm/llms/huggingface/common_utils.py @@ -96,7 +96,7 @@ def _fetch_inference_provider_mapping(model: str) -> dict: status_code = 500 headers = {} raise HuggingFaceError( - message=f"Failed to fetch provider mapping: {e!s}", + message=f"Failed to fetch provider mapping: {e}", status_code=status_code, headers=headers, ) diff --git a/litellm/llms/langgraph/chat/sse_iterator.py b/litellm/llms/langgraph/chat/sse_iterator.py index 895bbdca656..bdaa34871cf 100644 --- a/litellm/llms/langgraph/chat/sse_iterator.py +++ b/litellm/llms/langgraph/chat/sse_iterator.py @@ -196,7 +196,7 @@ class LangGraphSSEStreamIterator: except httpx.StreamClosed: raise StopIteration except Exception as e: - verbose_logger.error(f"Error in LangGraph SSE stream: {e!s}") + verbose_logger.error(f"Error in LangGraph SSE stream: {e}") raise StopIteration async def __anext__(self) -> ModelResponseStream: @@ -224,5 +224,5 @@ class LangGraphSSEStreamIterator: except httpx.StreamClosed: raise StopAsyncIteration except Exception as e: - verbose_logger.error(f"Error in LangGraph SSE stream: {e!s}") + verbose_logger.error(f"Error in LangGraph SSE stream: {e}") raise StopAsyncIteration diff --git a/litellm/llms/langgraph/chat/transformation.py b/litellm/llms/langgraph/chat/transformation.py index a40c08738f9..2aa96ddb978 100644 --- a/litellm/llms/langgraph/chat/transformation.py +++ b/litellm/llms/langgraph/chat/transformation.py @@ -451,14 +451,14 @@ class LangGraphConfig(BaseConfig): ) setattr(model_response, "usage", usage) except Exception as e: - verbose_logger.warning(f"Failed to calculate token usage: {e!s}") + verbose_logger.warning(f"Failed to calculate token usage: {e}") return model_response except Exception as e: - verbose_logger.error(f"Error processing LangGraph response: {e!s}") + verbose_logger.error(f"Error processing LangGraph response: {e}") raise LangGraphError( - message=f"Error processing response: {e!s}", + message=f"Error processing response: {e}", status_code=raw_response.status_code, ) diff --git a/litellm/llms/litellm_proxy/skills/code_execution.py b/litellm/llms/litellm_proxy/skills/code_execution.py index c99698a5c8e..f1142a8e355 100644 --- a/litellm/llms/litellm_proxy/skills/code_execution.py +++ b/litellm/llms/litellm_proxy/skills/code_execution.py @@ -239,7 +239,7 @@ class CodeExecutionHandler: tool_result += f"\n\nError:\n{exec_result['error']}" except Exception as e: - tool_result = f"Code execution failed: {e!s}" + tool_result = f"Code execution failed: {e}" execution_results.append( { "iteration": iteration, diff --git a/litellm/llms/manus/files/transformation.py b/litellm/llms/manus/files/transformation.py index cfa6d1cc722..325f6f36814 100644 --- a/litellm/llms/manus/files/transformation.py +++ b/litellm/llms/manus/files/transformation.py @@ -279,8 +279,8 @@ class ManusFilesConfig(BaseFilesConfig): status_details=response_json.get("status_details"), ) except Exception as e: - verbose_logger.exception(f"Error parsing Manus file response: {e!s}") - raise ValueError(f"Error parsing Manus file response: {e!s}") + verbose_logger.exception(f"Error parsing Manus file response: {e}") + raise ValueError(f"Error parsing Manus file response: {e}") def transform_retrieve_file_request( self, diff --git a/litellm/llms/milvus/vector_stores/transformation.py b/litellm/llms/milvus/vector_stores/transformation.py index 48265d095a8..8646258b3db 100644 --- a/litellm/llms/milvus/vector_stores/transformation.py +++ b/litellm/llms/milvus/vector_stores/transformation.py @@ -158,7 +158,7 @@ class MilvusVectorStoreConfig(BaseVectorStoreConfig): ) query_vector = embedding_response.data[0]["embedding"] except Exception as e: - raise Exception(f"Failed to generate embedding for query: {e!s}") + raise Exception(f"Failed to generate embedding for query: {e}") # Azure AI Search endpoint for search index_name = vector_store_id # vector_store_id is the index name diff --git a/litellm/llms/minimax/text_to_speech/transformation.py b/litellm/llms/minimax/text_to_speech/transformation.py index 93845d10789..af08bb8cb5f 100644 --- a/litellm/llms/minimax/text_to_speech/transformation.py +++ b/litellm/llms/minimax/text_to_speech/transformation.py @@ -353,7 +353,7 @@ class MinimaxTextToSpeechConfig(BaseTextToSpeechConfig): except Exception as e: raise MinimaxException( status_code=500, - message=f"Failed to decode audio data: {e!s}", + message=f"Failed to decode audio data: {e}", headers=dict(raw_response.headers), ) @@ -378,7 +378,7 @@ class MinimaxTextToSpeechConfig(BaseTextToSpeechConfig): except json.JSONDecodeError as e: raise MinimaxException( status_code=500, - message=f"Failed to parse MiniMax response: {e!s}", + message=f"Failed to parse MiniMax response: {e}", headers=dict(raw_response.headers), ) except Exception as e: @@ -386,7 +386,7 @@ class MinimaxTextToSpeechConfig(BaseTextToSpeechConfig): raise raise MinimaxException( status_code=500, - message=f"Error processing MiniMax response: {e!s}", + message=f"Error processing MiniMax response: {e}", headers=dict(raw_response.headers), ) diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index d73435dbcfc..91d12fd78ba 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -330,7 +330,7 @@ class MistralConfig(OpenAIGPTConfig): new_content = [{"type": "text", "text": reasoning_prompt + "\n\n"}] + existing_content else: # Fallback for any other type - convert to string - new_content = f"{reasoning_prompt}\n\n{existing_content!s}" + new_content = f"{reasoning_prompt}\n\n{existing_content}" messages[i] = cast(AllMessageValues, {**msg, "content": new_content}) break diff --git a/litellm/llms/oci/chat/cohere.py b/litellm/llms/oci/chat/cohere.py index d3ffa926c46..5db85d355c8 100644 --- a/litellm/llms/oci/chat/cohere.py +++ b/litellm/llms/oci/chat/cohere.py @@ -201,7 +201,7 @@ def handle_cohere_response( cohere_response = CohereChatResult(**json_response) except (TypeError, ValidationError) as e: raise OCIError( - message=f"Response cannot be casted to CohereChatResult: {e!s}", + message=f"Response cannot be casted to CohereChatResult: {e}", status_code=raw_response.status_code, ) @@ -283,7 +283,7 @@ def handle_cohere_stream_chunk( except (TypeError, ValidationError) as e: raise OCIError( status_code=500, - message=f"Chunk cannot be parsed as CohereStreamChunk: {e!s}", + message=f"Chunk cannot be parsed as CohereStreamChunk: {e}", ) if typed_chunk.index is None: diff --git a/litellm/llms/oci/chat/generic.py b/litellm/llms/oci/chat/generic.py index 354bcbed3ba..7c60b3bea65 100644 --- a/litellm/llms/oci/chat/generic.py +++ b/litellm/llms/oci/chat/generic.py @@ -309,7 +309,7 @@ def handle_generic_response( completion_response = OCICompletionResponse(**json_data) except (TypeError, ValidationError) as e: raise OCIError( - message=f"Response cannot be casted to OCICompletionResponse: {e!s}", + message=f"Response cannot be casted to OCICompletionResponse: {e}", status_code=raw_response.status_code, ) @@ -373,7 +373,7 @@ def handle_generic_stream_chunk(dict_chunk: dict) -> ModelResponseStream: except (TypeError, ValidationError) as e: raise OCIError( status_code=500, - message=f"Chunk cannot be parsed as OCIStreamChunk: {e!s}", + message=f"Chunk cannot be parsed as OCIStreamChunk: {e}", ) if typed_chunk.index is None: diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 2d441cb4515..b0fcf85e840 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -741,7 +741,7 @@ class OCIStreamWrapper(CustomStreamWrapper): except json.JSONDecodeError as e: raise OCIError( status_code=500, - message=f"Chunk cannot be parsed as JSON: {e!s}", + message=f"Chunk cannot be parsed as JSON: {e}", ) if dict_chunk.get("apiFormat") == "COHERE": diff --git a/litellm/llms/oci/common_utils.py b/litellm/llms/oci/common_utils.py index 7277972f64a..d5bacede08c 100644 --- a/litellm/llms/oci/common_utils.py +++ b/litellm/llms/oci/common_utils.py @@ -232,7 +232,7 @@ def sign_with_oci_signer( raise OCIError( status_code=500, message=( - f"Failed to sign request with provided oci_signer: {e!s}. " + f"Failed to sign request with provided oci_signer: {e}. " "The signer must implement the OCI SDK Signer interface with a " "do_request_sign(request, enforce_content_headers=True) method. " "See: https://docs.oracle.com/en-us/iaas/tools/python/latest/api/signing.html" diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index e9e60106d2d..e5afb4b87b6 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -369,7 +369,7 @@ class OllamaChatConfig(BaseConfig): content=None, tool_calls=[ { - "id": f"call_{uuid.uuid4()!s}", + "id": f"call_{uuid.uuid4()}", "function": { "name": function_call.get("name", litellm_params.get("function_name")), "arguments": json.dumps(function_call.get("arguments", function_call)), diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 0add66827f8..5823c2dad75 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -282,7 +282,7 @@ class OllamaConfig(BaseConfig): content=None, tool_calls=[ { - "id": f"call_{uuid.uuid4()!s}", + "id": f"call_{uuid.uuid4()}", "function": { "name": function_call["name"], "arguments": json.dumps(function_call["arguments"]), diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index a37e15c1f86..723b22a57b9 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -621,7 +621,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): except Exception as e: response_headers = getattr(raw_response, "headers", None) raise OpenAIError( - message=f"Unable to get json response - {e!s}, Original Response: {raw_response.text}", + message=f"Unable to get json response - {e}, Original Response: {raw_response.text}", status_code=raw_response.status_code, headers=response_headers, ) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index e4a13f0f526..845ad22589f 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -551,7 +551,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): except Exception as e: verbose_logger.exception( - f"LiteLLM.AgenticHookError: Exception in agentic completion hooks for OpenAI: {e!s}" + f"LiteLLM.AgenticHookError: Exception in agentic completion hooks for OpenAI: {e}" ) return None @@ -774,7 +774,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): # e.message except Exception as e: if print_verbose is not None: - print_verbose(f"openai.py: Received openai error - {e!s}") + print_verbose(f"openai.py: Received openai error - {e}") if ( "Conversation roles must alternate user/assistant" in str(e) or "user and assistant roles should be alternating" in str(e) @@ -1089,7 +1089,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): if response is not None and hasattr(response, "text"): raise OpenAIError( status_code=status_code, - message=f"{e!s}\n\nOriginal Response: {response.text}", # type: ignore + message=f"{e}\n\nOriginal Response: {response.text}", # type: ignore headers=error_headers, body=exception_body, ) @@ -1111,7 +1111,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): else: raise OpenAIError( status_code=500, - message=f"{e!s}", + message=f"{e}", headers=error_headers, body=exception_body, ) diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index 14fa6dc9954..a9a2b476776 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -178,7 +178,7 @@ class OpenAIRealtime(OpenAIChatCompletion): await websocket.close(code=e.status_code, reason=_redact_string(str(e))) except Exception as e: try: - await websocket.close(code=1011, reason=_redact_string(f"Internal server error: {e!s}")) + await websocket.close(code=1011, reason=_redact_string(f"Internal server error: {e}")) except RuntimeError as close_error: if "already completed" in str(close_error) or "websocket.close" in str(close_error): # The WebSocket is already closed or the response is completed, so we can ignore this error diff --git a/litellm/llms/openai/responses/count_tokens/handler.py b/litellm/llms/openai/responses/count_tokens/handler.py index b7cc3b1673a..e59a28c2d09 100644 --- a/litellm/llms/openai/responses/count_tokens/handler.py +++ b/litellm/llms/openai/responses/count_tokens/handler.py @@ -88,14 +88,14 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig): except OpenAIError: raise except httpx.HTTPStatusError as e: - verbose_logger.error(f"HTTP error in CountTokens handler: {e!s}") + verbose_logger.error(f"HTTP error in CountTokens handler: {e}") raise OpenAIError( status_code=e.response.status_code, message=e.response.text, ) except (httpx.RequestError, json.JSONDecodeError, ValueError) as e: - verbose_logger.error(f"Error in CountTokens handler: {e!s}") + verbose_logger.error(f"Error in CountTokens handler: {e}") raise OpenAIError( status_code=500, - message=f"CountTokens processing error: {e!s}", + message=f"CountTokens processing error: {e}", ) diff --git a/litellm/llms/openrouter/image_edit/transformation.py b/litellm/llms/openrouter/image_edit/transformation.py index fad7d53577c..8163b92bb19 100644 --- a/litellm/llms/openrouter/image_edit/transformation.py +++ b/litellm/llms/openrouter/image_edit/transformation.py @@ -203,7 +203,7 @@ class OpenRouterImageEditConfig(BaseImageEditConfig): response_json = raw_response.json() except Exception as e: raise OpenRouterException( - message=f"Error parsing OpenRouter response: {e!s}", + message=f"Error parsing OpenRouter response: {e}", status_code=raw_response.status_code, headers=raw_response.headers, ) @@ -246,7 +246,7 @@ class OpenRouterImageEditConfig(BaseImageEditConfig): except Exception as e: raise OpenRouterException( - message=f"Error transforming OpenRouter image edit response: {e!s}", + message=f"Error transforming OpenRouter image edit response: {e}", status_code=500, headers={}, ) diff --git a/litellm/llms/openrouter/image_generation/transformation.py b/litellm/llms/openrouter/image_generation/transformation.py index 1114bb41275..f56ca6ba89e 100644 --- a/litellm/llms/openrouter/image_generation/transformation.py +++ b/litellm/llms/openrouter/image_generation/transformation.py @@ -345,7 +345,7 @@ class OpenRouterImageGenerationConfig(BaseImageGenerationConfig): response_json = raw_response.json() except Exception as e: raise OpenRouterException( - message=f"Error parsing OpenRouter response: {e!s}", + message=f"Error parsing OpenRouter response: {e}", status_code=raw_response.status_code, headers=raw_response.headers, ) @@ -394,7 +394,7 @@ class OpenRouterImageGenerationConfig(BaseImageGenerationConfig): except Exception as e: raise OpenRouterException( - message=f"Error transforming OpenRouter image generation response: {e!s}", + message=f"Error transforming OpenRouter image generation response: {e}", status_code=500, headers={}, ) diff --git a/litellm/llms/predibase/chat/handler.py b/litellm/llms/predibase/chat/handler.py index 36537562638..2bf39966dd1 100644 --- a/litellm/llms/predibase/chat/handler.py +++ b/litellm/llms/predibase/chat/handler.py @@ -225,7 +225,7 @@ class PredibaseChatCompletion: if isinstance(e, exception): raise e raise PredibaseError( - status_code=500, message=f"{e!s}" + status_code=500, message=f"{e}" ) # don't use verbose_logger.exception, if exception is raised return predibase_config.transform_response( model=model, diff --git a/litellm/llms/sagemaker/completion/handler.py b/litellm/llms/sagemaker/completion/handler.py index 868f4f9696e..406a72ffd99 100644 --- a/litellm/llms/sagemaker/completion/handler.py +++ b/litellm/llms/sagemaker/completion/handler.py @@ -529,7 +529,7 @@ class SagemakerLLM(BaseAWSLLM): ) raise e except Exception as e: - error_message = f"{e!s}" + error_message = f"{e}" if "Inference Component Name header is required" in error_message: error_message += "\n pass in via `litellm.completion(..., model_id={InferenceComponentName})`" raise SagemakerError(status_code=500, message=error_message) diff --git a/litellm/llms/sagemaker/embedding/transformation.py b/litellm/llms/sagemaker/embedding/transformation.py index 7221e030d97..51fad1e1c2e 100644 --- a/litellm/llms/sagemaker/embedding/transformation.py +++ b/litellm/llms/sagemaker/embedding/transformation.py @@ -97,7 +97,7 @@ class SagemakerEmbeddingConfig(BaseEmbeddingConfig): response_data = raw_response.json() except Exception as e: raise SagemakerError( - message=f"Failed to parse response: {e!s}", + message=f"Failed to parse response: {e}", status_code=raw_response.status_code, ) diff --git a/litellm/llms/vertex_ai/agent_engine/transformation.py b/litellm/llms/vertex_ai/agent_engine/transformation.py index e785e7ec28e..9f01a9ee506 100644 --- a/litellm/llms/vertex_ai/agent_engine/transformation.py +++ b/litellm/llms/vertex_ai/agent_engine/transformation.py @@ -270,7 +270,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): total_tokens=total_tokens, ) except Exception as e: - verbose_logger.warning(f"Failed to calculate token usage: {e!s}") + verbose_logger.warning(f"Failed to calculate token usage: {e}") return None def transform_response( @@ -335,9 +335,9 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): return model_response except Exception as e: - verbose_logger.error(f"Error processing Vertex Agent Engine response: {e!s}") + verbose_logger.error(f"Error processing Vertex Agent Engine response: {e}") raise VertexAgentEngineError( - message=f"Error processing response: {e!s}", + message=f"Error processing response: {e}", status_code=raw_response.status_code, ) diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index b627444b181..81d084e7e03 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -221,7 +221,7 @@ def get_supports_system_message( supports_system_message = True except Exception as e: verbose_logger.warning( - f"Unable to identify if system message supported. Defaulting to 'False'. Received error message - {e!s}\nAdd it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json" + f"Unable to identify if system message supported. Defaulting to 'False'. Received error message - {e}\nAdd it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json" ) supports_system_message = False diff --git a/litellm/llms/vertex_ai/cost_calculator.py b/litellm/llms/vertex_ai/cost_calculator.py index 1cd4c0a9e97..a53e54e5fc2 100644 --- a/litellm/llms/vertex_ai/cost_calculator.py +++ b/litellm/llms/vertex_ai/cost_calculator.py @@ -114,7 +114,7 @@ def cost_per_character( prompt_cost = prompt_characters * model_info["input_cost_per_character"] except Exception as e: verbose_logger.debug( - f"litellm.litellm_core_utils.llm_cost_calc.google.py::cost_per_character(): Exception occured - {e!s}\nDefaulting to None" + f"litellm.litellm_core_utils.llm_cost_calc.google.py::cost_per_character(): Exception occured - {e}\nDefaulting to None" ) prompt_cost, _ = cost_per_token( model=model, @@ -152,7 +152,7 @@ def cost_per_character( completion_cost = completion_characters * model_info["output_cost_per_character"] except Exception as e: verbose_logger.debug( - f"litellm.litellm_core_utils.llm_cost_calc.google.py::cost_per_character(): Exception occured - {e!s}\nDefaulting to None" + f"litellm.litellm_core_utils.llm_cost_calc.google.py::cost_per_character(): Exception occured - {e}\nDefaulting to None" ) _, completion_cost = cost_per_token( model=model, diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index f8eea399bf6..49fdb2786e1 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -815,6 +815,6 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): "response": None, "error": { "code": "transformation_error", - "message": f"Failed to transform response: {e!s}", + "message": f"Failed to transform response: {e}", }, } diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 9d4d8a5a02e..76549d0fed2 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -356,9 +356,7 @@ def _get_gcs_object_content_type( headers["Authorization"] = f"Bearer {access_token}" except Exception as e: raise litellm.BadRequestError( - message=( - f"Unable to fetch GCS metadata with provided Vertex credentials/project. Original error: {e!s}" - ), + message=(f"Unable to fetch GCS metadata with provided Vertex credentials/project. Original error: {e}"), model=None, llm_provider="vertex_ai", ) @@ -844,7 +842,7 @@ def _gemini_convert_messages_with_history( f"{file_id or 'provided data'}, set this explicitly " f"using message[{msg_i}].content[{element_idx}].file.format " f"(or file.mime_type/content_type). " - f"Original error: {e!s}" + f"Original error: {e}" ), model=model, llm_provider="vertex_ai", diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 19c43d8000c..cadc8760601 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -2405,7 +2405,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): completion_response = GenerateContentResponseBody(**raw_response.json()) # type: ignore except Exception as e: raise VertexAIError( - message=f"Error converting to valid response block={e!s}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues", + message=f"Error converting to valid response block={e}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues", status_code=422, headers=raw_response.headers, ) @@ -2512,7 +2512,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): except Exception as e: raise VertexAIError( - message=f"Error converting to valid response block={e!s}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues", + message=f"Error converting to valid response block={e}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues", status_code=422, headers=raw_response.headers, ) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py index abad2bb73ea..d5e279ea240 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py @@ -127,7 +127,7 @@ class VertexAILlama3Config(OpenAIGPTConfig): except Exception as e: response_headers = getattr(raw_response, "headers", None) raise VertexAIError( - message=f"Unable to get json response - {e!s}, Original Response: {raw_response.text}", + message=f"Unable to get json response - {e}, Original Response: {raw_response.text}", status_code=raw_response.status_code, headers=response_headers, ) diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index b3ffa1d40be..2def1acb708 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -783,7 +783,7 @@ class VertexBase: except Exception as retry_error: verbose_logger.error( f"Reauthentication retry failed for project_id: {project_id}. " - f"Original error: {error!s}. Retry error: {retry_error!s}" + f"Original error: {error}. Retry error: {retry_error}" ) # Re-raise the original error for better context raise error @@ -837,7 +837,7 @@ class VertexBase: except Exception as retry_error: verbose_logger.error( f"Async reauthentication retry failed for project_id: {project_id}. " - f"Original error: {error!s}. Retry error: {retry_error!s}" + f"Original error: {error}. Retry error: {retry_error}" ) raise error @@ -897,7 +897,7 @@ class VertexBase: _credentials, credential_project_id = self.load_auth(credentials=credentials, project_id=project_id) except Exception as e: verbose_logger.exception( - f"Failed to load vertex credentials. Check to see if credentials containing partial/invalid information. Error: {e!s}" + f"Failed to load vertex credentials. Check to see if credentials containing partial/invalid information. Error: {e}" ) raise e diff --git a/litellm/llms/volcengine/embedding/transformation.py b/litellm/llms/volcengine/embedding/transformation.py index 5a0b59d411c..9923167ba31 100644 --- a/litellm/llms/volcengine/embedding/transformation.py +++ b/litellm/llms/volcengine/embedding/transformation.py @@ -162,7 +162,7 @@ class VolcEngineEmbeddingConfig(BaseEmbeddingConfig): try: response_json = raw_response.json() except Exception as e: - raise ValueError(f"Failed to parse Volcengine response as JSON: {e!s}") + raise ValueError(f"Failed to parse Volcengine response as JSON: {e}") # Volcengine response format matches OpenAI format closely # Just need to ensure all required fields are present diff --git a/litellm/llms/watsonx/audio_transcription/transformation.py b/litellm/llms/watsonx/audio_transcription/transformation.py index 6019b2e8355..a9cd85cb674 100644 --- a/litellm/llms/watsonx/audio_transcription/transformation.py +++ b/litellm/llms/watsonx/audio_transcription/transformation.py @@ -170,7 +170,7 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran try: raw_response_json = raw_response.json() except Exception as e: - raise ValueError(f"Error transforming response to json: {e!s}\nResponse: {raw_response.text}") + raise ValueError(f"Error transforming response to json: {e}\nResponse: {raw_response.text}") # Extract only valid fields for TranscriptionResponse.__init__() # TranscriptionResponse only accepts 'text' and 'usage' in __init__() diff --git a/litellm/llms/watsonx/rerank/transformation.py b/litellm/llms/watsonx/rerank/transformation.py index 6d9de3f481b..e1d5f2f3571 100644 --- a/litellm/llms/watsonx/rerank/transformation.py +++ b/litellm/llms/watsonx/rerank/transformation.py @@ -164,7 +164,7 @@ class IBMWatsonXRerankConfig(IBMWatsonXMixin, BaseRerankConfig): raw_response_json = raw_response.json() except Exception as e: raise self.get_error_class( - error_message=f"Failed to parse response: {e!s}", + error_message=f"Failed to parse response: {e}", status_code=raw_response.status_code, headers=raw_response.headers, ) diff --git a/litellm/main.py b/litellm/main.py index cea9d44fb1a..731a545a267 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8303,7 +8303,7 @@ async def ahealth_check( if mode is None: return { - "error": f"error:{e!s}. Missing `mode`. Set the `mode` for the model - https://docs.litellm.ai/docs/proxy/health#embedding-models \nstacktrace: {stack_trace}", + "error": f"error:{e}. Missing `mode`. Set the `mode` for the model - https://docs.litellm.ai/docs/proxy/health#embedding-models \nstacktrace: {stack_trace}", "exception": e, } @@ -8669,7 +8669,7 @@ def stream_chunk_builder( processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj) return response except Exception as e: - verbose_logger.exception(f"litellm.main.py::stream_chunk_builder() - Exception occurred - {e!s}") + verbose_logger.exception(f"litellm.main.py::stream_chunk_builder() - Exception occurred - {e}") raise litellm.APIError( status_code=500, message="Error building chunks for logging/streaming usage calculation", diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index e8f39daa758..a256653f0f9 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -1008,7 +1008,7 @@ class MCPRequestHandler: limits[source.team_id] = applicable return limits or None except Exception as e: # noqa: BLE001 # throttling metadata must never fail an allowed request - verbose_logger.warning(f"Failed to resolve per-team MCP rpm limits for admitted subject: {e!s}") + verbose_logger.warning(f"Failed to resolve per-team MCP rpm limits for admitted subject: {e}") return None @staticmethod @@ -1514,9 +1514,9 @@ class MCPRequestHandler: if isinstance(e, UnloadableEntitlementError): # A ceiling we KNOW exists and cannot read. Denying is the only answer that does not # widen this caller past what an operator configured, for both caller shapes. - verbose_logger.warning(f"Denying MCP access, entitlement unreadable: {e!s}") + verbose_logger.warning(f"Denying MCP access, entitlement unreadable: {e}") else: - verbose_logger.warning(f"Failed to get allowed MCP servers: {e!s}") + verbose_logger.warning(f"Failed to get allowed MCP servers: {e}") return [] @staticmethod @@ -1649,7 +1649,7 @@ class MCPRequestHandler: # Fault isolation is per SOURCE: an unresolvable team contributes nothing (fail closed for # it alone, access only narrows) while every other source stands. Raising would collapse the # whole union to deny-all over one momentarily-unreadable row. - verbose_logger.warning(f"MCP admitted-subject source team {team_id!r} unresolvable, skipping: {e!s}") + verbose_logger.warning(f"MCP admitted-subject source team {team_id!r} unresolvable, skipping: {e}") return None if team_obj is None: return None @@ -1682,10 +1682,10 @@ class MCPRequestHandler: proxy_logging_obj=proxy_logging_obj, ) except BudgetExceededError as e: - verbose_logger.info(f"MCP admitted-subject source team {team_id!r} over budget, not a grantor: {e!s}") + verbose_logger.info(f"MCP admitted-subject source team {team_id!r} over budget, not a grantor: {e}") return None except Exception as e: # noqa: BLE001 # per-source isolation: a budget-check fault narrows, never raises - verbose_logger.warning(f"MCP budget check failed for source team {team_id!r}, skipping source: {e!s}") + verbose_logger.warning(f"MCP budget check failed for source team {team_id!r}, skipping source: {e}") return None return team_obj @@ -1738,7 +1738,7 @@ class MCPRequestHandler: billed.org_id = source.org_id return billed except Exception as e: # noqa: BLE001 # attribution must never fail an authorized call - verbose_logger.warning(f"MCP billing attribution failed for {tool_name!r}, billing the user: {e!s}") + verbose_logger.warning(f"MCP billing attribution failed for {tool_name!r}, billing the user: {e}") return auth @staticmethod @@ -1946,9 +1946,9 @@ class MCPRequestHandler: # than the None (allow-all) key auth gets for an indeterminate fault. unreadable_entitlement = isinstance(e, UnloadableEntitlementError) if unreadable_entitlement: - verbose_logger.warning(f"Denying MCP tools, entitlement unreadable: {e!s}") + verbose_logger.warning(f"Denying MCP tools, entitlement unreadable: {e}") else: - verbose_logger.warning(f"Failed to get allowed tools for server: {e!s}") + verbose_logger.warning(f"Failed to get allowed tools for server: {e}") # Fail CLOSED for a keyless admitted subject: ANY error must deny the server's tools ([]), # not collapse to allow-all (None); key/JWT auth keeps its prior allow-all-on-error. Both # keyless_source AND the marker are needed: each source resolves through an UNMARKED auth, so @@ -1999,7 +1999,7 @@ class MCPRequestHandler: raise verbose_logger.warning( f"MCP org tool ceiling unresolvable for org_id={user_api_key_auth.org_id!r}; " - f"skipping org intersect, key/team/agent restrictions stand: {e!s}" + f"skipping org intersect, key/team/agent restrictions stand: {e}" ) return allowed_tools org_tools = ( @@ -2102,7 +2102,7 @@ class MCPRequestHandler: # Permission entries may be server_ids OR names/aliases — expand to ids. return global_mcp_server_manager.expand_permission_list(raw_server_ids) except Exception as e: - verbose_logger.warning(f"Failed to get key access group MCP server grants: {e!s}") + verbose_logger.warning(f"Failed to get key access group MCP server grants: {e}") return [] @staticmethod @@ -2180,7 +2180,7 @@ class MCPRequestHandler: all_servers = direct_mcp_servers + access_group_servers + tool_perm_servers + toolset_servers return list(set(all_servers)) except Exception as e: - verbose_logger.warning(f"Failed to get allowed MCP servers for key: {e!s}") + verbose_logger.warning(f"Failed to get allowed MCP servers for key: {e}") return [] @staticmethod @@ -2238,7 +2238,7 @@ class MCPRequestHandler: proxy_logging_obj=proxy_logging_obj, ) except Exception as e: # noqa: BLE001 # a team-resolution blip narrows access, never raises - verbose_logger.warning(f"Failed to resolve user teams for MCP grant: {e!s}") + verbose_logger.warning(f"Failed to resolve user teams for MCP grant: {e}") return [] if user_object is None or not user_object.teams: return [] @@ -2323,7 +2323,7 @@ class MCPRequestHandler: servers = await MCPRequestHandler._team_granted_servers(team_obj, team_access_group_servers) return list(servers) except Exception as e: - verbose_logger.warning(f"Failed to get allowed MCP servers for team: {e!s}") + verbose_logger.warning(f"Failed to get allowed MCP servers for team: {e}") return [] @staticmethod @@ -2462,7 +2462,7 @@ class MCPRequestHandler: # A NAMED-but-unreadable ceiling is a stronger fact than "unresolved" and denies everywhere. if isinstance(e, UnloadableEntitlementError): raise - verbose_logger.warning(f"Failed to get allowed MCP servers for org: {e!s}") + verbose_logger.warning(f"Failed to get allowed MCP servers for org: {e}") return None @staticmethod @@ -2490,7 +2490,7 @@ class MCPRequestHandler: route="/mcp", ) except Exception as e: # noqa: BLE001 # entitlement unknown, not known-absent: no ceiling, as before this level - verbose_logger.warning(f"Failed to resolve end_user for MCP permissions: {e!s}") + verbose_logger.warning(f"Failed to resolve end_user for MCP permissions: {e}") return None if end_user_obj is None: @@ -2554,7 +2554,7 @@ class MCPRequestHandler: all_servers = direct_mcp_servers + access_group_servers + tool_perm_servers return list(set(all_servers)) except Exception as e: - verbose_logger.warning(f"Failed to get allowed MCP servers for end_user: {e!s}") + verbose_logger.warning(f"Failed to get allowed MCP servers for end_user: {e}") return [] @staticmethod @@ -2637,7 +2637,7 @@ class MCPRequestHandler: ) return object_permission_id except Exception as e: # noqa: BLE001 # unknown whether entitled at all: no ceiling, as before - verbose_logger.warning(f"MCP user entitlement: link for {user_id!r} unresolved, no ceiling: {e!s}") + verbose_logger.warning(f"MCP user entitlement: link for {user_id!r} unresolved, no ceiling: {e}") return None @staticmethod @@ -2669,7 +2669,7 @@ class MCPRequestHandler: ) return list(set(direct_mcp_servers + access_group_servers + tool_perm_servers)) except Exception as e: # noqa: BLE001 # any resolution fault is an unresolved ceiling, never "no ceiling" - verbose_logger.warning(f"Failed to get allowed MCP servers for user: {e!s}") + verbose_logger.warning(f"Failed to get allowed MCP servers for user: {e}") return None @staticmethod @@ -2739,7 +2739,7 @@ class MCPRequestHandler: try: object_permissions = await MCPRequestHandler._get_user_object_permission(user_api_key_auth) except Exception as e: # noqa: BLE001 # an unresolved human entitlement must deny, not widen - verbose_logger.warning(f"MCP user tool ceiling unresolvable, denying tools on {server_id!r}: {e!s}") + verbose_logger.warning(f"MCP user tool ceiling unresolvable, denying tools on {server_id!r}: {e}") return [] if object_permissions is None or not object_permissions.mcp_tool_permissions: @@ -2785,7 +2785,7 @@ class MCPRequestHandler: ) return object_permission_id except Exception as e: # noqa: BLE001 # entitlement unknown, not known-absent: no ceiling, as before this level - verbose_logger.warning(f"Failed to resolve object_permission_id for agent {agent_id!r}: {e!s}") + verbose_logger.warning(f"Failed to resolve object_permission_id for agent {agent_id!r}: {e}") return None @staticmethod @@ -2869,7 +2869,7 @@ class MCPRequestHandler: all_servers = expanded_direct_servers + access_group_servers return list(set(all_servers)) except Exception as e: - verbose_logger.warning(f"Failed to get allowed MCP servers for agent: {e!s}") + verbose_logger.warning(f"Failed to get allowed MCP servers for agent: {e}") return [] @staticmethod @@ -2911,7 +2911,7 @@ class MCPRequestHandler: tools = global_mcp_server_manager.expand_tool_permissions(mcp_tool_permissions).get(server_id) return list(tools) if tools else None except Exception as e: - verbose_logger.warning(f"Failed to get agent tool permissions for server: {e!s}") + verbose_logger.warning(f"Failed to get agent tool permissions for server: {e}") return None @staticmethod @@ -2969,7 +2969,7 @@ class MCPRequestHandler: return list(server_ids) except Exception as e: - verbose_logger.warning(f"Failed to get MCP servers from access groups: {e!s}") + verbose_logger.warning(f"Failed to get MCP servers from access groups: {e}") return [] @staticmethod @@ -3029,7 +3029,7 @@ class MCPRequestHandler: return key_object_permission.mcp_access_groups or [] except Exception as e: - verbose_logger.warning(f"Failed to get MCP access groups for key: {e!s}") + verbose_logger.warning(f"Failed to get MCP access groups for key: {e}") return [] @staticmethod @@ -3077,7 +3077,7 @@ class MCPRequestHandler: return object_permissions.mcp_access_groups or [] except Exception as e: - verbose_logger.warning(f"Failed to get MCP access groups for team: {e!s}") + verbose_logger.warning(f"Failed to get MCP access groups for team: {e}") return [] @staticmethod diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 3c8e7d9f1ef..672396afd05 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -570,7 +570,7 @@ async def get_all_mcp_servers( decrypt_global_env_var_values(table.env_vars) return tables except Exception as e: - verbose_proxy_logger.debug(f"litellm.proxy._experimental.mcp_server.db.py::get_all_mcp_servers - {e!s}") + verbose_proxy_logger.debug(f"litellm.proxy._experimental.mcp_server.db.py::get_all_mcp_servers - {e}") return [] diff --git a/litellm/proxy/_experimental/mcp_server/elicitation_handler.py b/litellm/proxy/_experimental/mcp_server/elicitation_handler.py index 9927afa20d0..35681a9473e 100644 --- a/litellm/proxy/_experimental/mcp_server/elicitation_handler.py +++ b/litellm/proxy/_experimental/mcp_server/elicitation_handler.py @@ -79,7 +79,7 @@ async def handle_elicitation_request( verbose_logger.exception("MCP elicitation handler failed: %s", e) return ErrorData( code=-1, - message=f"Elicitation failed: {e!s}", + message=f"Elicitation failed: {e}", ) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index d8ab34a7ddb..0a6a0374d13 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -1951,7 +1951,7 @@ class MCPServerManager: verbose_logger.info(f"Successfully registered {registered_count} OpenAPI tools for server {server.name}") except Exception as e: - verbose_logger.error(f"Failed to register OpenAPI tools for server {server.name}: {e!s}") + verbose_logger.error(f"Failed to register OpenAPI tools for server {server.name}: {e}") raise e def _cleanup_server_tool_routing_artifacts(self, server: MCPServer) -> None: @@ -2326,7 +2326,7 @@ class MCPServerManager: verbose_logger.debug(f"Added MCP Server: {new_server.name}") except Exception as e: - verbose_logger.debug(f"Failed to add MCP server: {e!s}") + verbose_logger.debug(f"Failed to add MCP server: {e}") raise e async def update_server(self, mcp_server: LiteLLM_MCPServerTable): @@ -2360,7 +2360,7 @@ class MCPServerManager: verbose_logger.debug(f"Updated MCP Server: {new_server.name}") except Exception as e: - verbose_logger.debug(f"Failed to udpate MCP server: {e!s}") + verbose_logger.debug(f"Failed to udpate MCP server: {e}") raise e def get_all_mcp_server_ids(self) -> set[str]: @@ -2386,7 +2386,7 @@ class MCPServerManager: await user_api_key_cache.async_delete_cache(key=self.get_byom_submitted_servers_cache_key(user_id)) except Exception as e: # noqa: BLE001 - verbose_logger.warning(f"Failed to invalidate BYOM submitted MCP server cache: {e!s}") + verbose_logger.warning(f"Failed to invalidate BYOM submitted MCP server cache: {e}") async def _get_active_submitted_mcp_server_ids_for_user( self, user_api_key_auth: UserAPIKeyAuth | None @@ -2401,7 +2401,7 @@ class MCPServerManager: ) from litellm.proxy.proxy_server import prisma_client, user_api_key_cache except Exception as e: # noqa: BLE001 - verbose_logger.warning(f"Failed to load BYOM submitted MCP server cache dependencies: {e!s}") + verbose_logger.warning(f"Failed to load BYOM submitted MCP server cache dependencies: {e}") return [] byom_cache_key = self.get_byom_submitted_servers_cache_key(submitter_user_id) @@ -2411,7 +2411,7 @@ class MCPServerManager: if cached_submitted_server_ids is not None: submitted_server_ids = cast(list[str], cached_submitted_server_ids) except Exception as e: # noqa: BLE001 - verbose_logger.warning(f"Failed to read BYOM submitted MCP server cache: {e!s}") + verbose_logger.warning(f"Failed to read BYOM submitted MCP server cache: {e}") if submitted_server_ids is None: if prisma_client is None: @@ -2422,7 +2422,7 @@ class MCPServerManager: prisma_client, submitter_user_id ) except Exception as e: # noqa: BLE001 - verbose_logger.warning(f"Failed to read BYOM submitted MCP servers from database: {e!s}") + verbose_logger.warning(f"Failed to read BYOM submitted MCP servers from database: {e}") submitted_server_ids = [] try: await user_api_key_cache.async_set_cache( @@ -2431,7 +2431,7 @@ class MCPServerManager: ttl=60, ) except Exception as e: # noqa: BLE001 - verbose_logger.warning(f"Failed to write BYOM submitted MCP server cache: {e!s}") + verbose_logger.warning(f"Failed to write BYOM submitted MCP server cache: {e}") return [server_id for server_id in submitted_server_ids if self.get_mcp_server_by_id(server_id) is not None] @@ -2647,7 +2647,7 @@ class MCPServerManager: ) return tool_permissions except Exception as e: - verbose_logger.warning(f"Failed to resolve toolset permissions: {e!s}") + verbose_logger.warning(f"Failed to resolve toolset permissions: {e}") return {} def invalidate_toolset_cache(self, toolset_id: str | None = None) -> None: @@ -2764,7 +2764,7 @@ class MCPServerManager: return [] return await self._get_tools_from_server(server) except Exception as e: - verbose_logger.warning(f"Failed to get tools from server {server_id}: {e!s}") + verbose_logger.warning(f"Failed to get tools from server {server_id}: {e}") return [] async def list_tools( @@ -2822,7 +2822,7 @@ class MCPServerManager: return tools except Exception as e: verbose_logger.warning( - f"Failed to list tools from server {server.name}: {e!s}. Continuing with other servers." + f"Failed to list tools from server {server.name}: {e}. Continuing with other servers." ) return [] @@ -3476,12 +3476,12 @@ class MCPServerManager: www_authenticate=None if server.is_dcr_bridge else challenge_header, server_name=server.name, ) from e - verbose_logger.warning(f"Failed to get tools from server {server.name}: {e!s}") + verbose_logger.warning(f"Failed to get tools from server {server.name}: {e}") raise MCPServerListError(ServerListFault(tag="internal", status_code=e.status_code), server.name) from e except MCPServerListError: raise except Exception as e: - verbose_logger.warning(f"Failed to get tools from server {server.name}: {e!s}") + verbose_logger.warning(f"Failed to get tools from server {server.name}: {e}") raise_classified_list_failure(e, server.name, suppress_challenge=server.is_dcr_bridge) async def get_prompts_from_server( @@ -3532,7 +3532,7 @@ class MCPServerManager: return prefixed_or_original_prompts except Exception as e: - verbose_logger.warning(f"Failed to get prompts from server {server.name}: {e!s}") + verbose_logger.warning(f"Failed to get prompts from server {server.name}: {e}") return [] async def get_resources_from_server( @@ -3574,7 +3574,7 @@ class MCPServerManager: return prefixed_resources except Exception as e: - verbose_logger.warning(f"Failed to get resources from server {server.name}: {e!s}") + verbose_logger.warning(f"Failed to get resources from server {server.name}: {e}") return [] async def get_resource_templates_from_server( @@ -3618,7 +3618,7 @@ class MCPServerManager: return prefixed_templates except Exception as e: - verbose_logger.warning(f"Failed to get resource templates from server {server.name}: {e!s}") + verbose_logger.warning(f"Failed to get resource templates from server {server.name}: {e}") return [] async def read_resource_from_server( @@ -4215,10 +4215,10 @@ class MCPServerManager: verbose_logger.warning(f"Task cancelled while listing tools from {server_name}") raise MCPServerListError(ServerListFault(tag="internal"), server_name) from e except ConnectionError as e: - verbose_logger.warning(f"Connection error while listing tools from {server_name}: {e!s}") + verbose_logger.warning(f"Connection error while listing tools from {server_name}: {e}") raise MCPServerListError(ServerListFault(tag="unreachable"), server_name) from e except Exception as e: - verbose_logger.warning(f"Error listing tools from {server_name}: {e!s}") + verbose_logger.warning(f"Error listing tools from {server_name}: {e}") raise_classified_list_failure(e, server_name) _SHORT_PREFIX_MAX_REHASH_ATTEMPTS = 1024 @@ -4533,7 +4533,7 @@ class MCPServerManager: return result except Exception as e: - error_msg = f"Error calling OpenAPI tool {tool_name}: {e!s}" + error_msg = f"Error calling OpenAPI tool {tool_name}: {e}" verbose_logger.error(error_msg) return CallToolResult( content=[TextContent(type="text", text=error_msg)], @@ -4639,7 +4639,7 @@ class MCPServerManager: HTTPException, ) as e: # Re-raise guardrail exceptions to properly fail the MCP call - verbose_logger.error(f"Guardrail blocked MCP tool call pre call: {e!s}") + verbose_logger.error(f"Guardrail blocked MCP tool call pre call: {e}") raise e return hook_result @@ -4995,7 +4995,7 @@ class MCPServerManager: GuardrailRaisedException, HTTPException, ) as e: - verbose_logger.error(f"Guardrail blocked MCP tool call during result check: {e!s}") + verbose_logger.error(f"Guardrail blocked MCP tool call during result check: {e}") raise e # If proxy_logging_obj is None, the tool call result is at index 0 @@ -5194,7 +5194,7 @@ class MCPServerManager: GuardrailRaisedException, HTTPException, ) as e: - verbose_logger.error(f"Guardrail blocked MCP tool call during result check: {e!s}") + verbose_logger.error(f"Guardrail blocked MCP tool call during result check: {e}") raise e async def call_tool( @@ -5345,7 +5345,7 @@ class MCPServerManager: asyncio.create_task(self._initialize_tool_name_to_mcp_server_name_mapping()) except RuntimeError as e: # no running event loop verbose_logger.exception( - f"No running event loop - skipping tool name to MCP server name mapping initialization: {e!s}" + f"No running event loop - skipping tool name to MCP server name mapping initialization: {e}" ) async def _initialize_tool_name_to_mcp_server_name_mapping(self): @@ -5364,12 +5364,12 @@ class MCPServerManager: # at startup we have none, so an upstream 401 is normal. # Swallow it so we keep mapping the remaining servers. verbose_logger.debug( - f"Skipping tool name mapping for server {server.name} due to upstream auth error: {e!s}" + f"Skipping tool name mapping for server {server.name} due to upstream auth error: {e}" ) continue except Exception as e: verbose_logger.warning( - f"Failed to get tools from server {server.name} during tool name mapping initialization: {e!s}" + f"Failed to get tools from server {server.name} during tool name mapping initialization: {e}" ) continue for tool in tools: diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 6f4fde6fbfe..d0458db51c6 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -654,7 +654,7 @@ if MCP_AVAILABLE: return { "tools": [], "error": "server_error", - "message": f"Failed to get tools from server {server.name}: {e!s}", + "message": f"Failed to get tools from server {server.name}: {e}", } return { "tools": list_tools_result, @@ -866,7 +866,7 @@ if MCP_AVAILABLE: errors.append( f"{get_server_prefix(server)}: {classify_list_exception(e).tag}" if isinstance(e, (MCPServerListError, MCPUpstreamAuthError)) - else f"{get_server_prefix(server)}: {e!s}" + else f"{get_server_prefix(server)}: {e}" ) continue @@ -905,7 +905,7 @@ if MCP_AVAILABLE: return { "tools": [], "error": "unexpected_error", - "message": f"An unexpected error occurred: {e!s}", + "message": f"An unexpected error occurred: {e}", } @router.post("/tools/call", dependencies=[Depends(user_api_key_auth)]) @@ -1052,7 +1052,7 @@ if MCP_AVAILABLE: }, ) except BlockedPiiEntityError as e: - verbose_logger.error(f"BlockedPiiEntityError in MCP tool call: {e!s}") + verbose_logger.error(f"BlockedPiiEntityError in MCP tool call: {e}") raise HTTPException( status_code=400, detail={ @@ -1063,7 +1063,7 @@ if MCP_AVAILABLE: }, ) except GuardrailRaisedException as e: - verbose_logger.error(f"GuardrailRaisedException in MCP tool call: {e!s}") + verbose_logger.error(f"GuardrailRaisedException in MCP tool call: {e}") raise HTTPException( status_code=400, detail={ @@ -1082,15 +1082,15 @@ if MCP_AVAILABLE: # Locally generated denials (tool/server permission, IP filtering, BYOK) stay at error level # so restriction probing keeps full monitoring visibility; the relayed upstream 401 above is # the only status demoted to info. - verbose_logger.error(f"HTTPException in MCP tool call: {e!s}") + verbose_logger.error(f"HTTPException in MCP tool call: {e}") raise e except Exception as e: - verbose_logger.exception(f"Unexpected error in MCP tool call: {e!s}") + verbose_logger.exception(f"Unexpected error in MCP tool call: {e}") raise HTTPException( status_code=500, detail={ "error": "internal_server_error", - "message": f"An unexpected error occurred: {e!s}", + "message": f"An unexpected error occurred: {e}", }, ) diff --git a/litellm/proxy/_experimental/mcp_server/sampling_handler.py b/litellm/proxy/_experimental/mcp_server/sampling_handler.py index 779cc5861d4..e694c2da7e3 100644 --- a/litellm/proxy/_experimental/mcp_server/sampling_handler.py +++ b/litellm/proxy/_experimental/mcp_server/sampling_handler.py @@ -1292,5 +1292,5 @@ async def handle_sampling_create_message( verbose_logger.exception("MCP sampling handler failed: %s", e) return ErrorData( code=-1, - message=f"Sampling failed: {e!s}", + message=f"Sampling failed: {e}", ) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index fe47c264dfa..a894413019e 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -805,7 +805,7 @@ if MCP_AVAILABLE: } return ListToolsResult.model_validate({"tools": listing.tools, "_meta": outcome_meta}) except Exception as e: - verbose_logger.exception(f"Error in list_tools endpoint: {e!s}") + verbose_logger.exception(f"Error in list_tools endpoint: {e}") # Return empty list instead of failing completely # This prevents the HTTP stream from failing and allows the client to get a response return [] @@ -1080,26 +1080,26 @@ if MCP_AVAILABLE: isError=True, ) except BlockedPiiEntityError as e: - verbose_logger.error(f"BlockedPiiEntityError in MCP tool call: {e!s}") + verbose_logger.error(f"BlockedPiiEntityError in MCP tool call: {e}") return CallToolResult( content=[ TextContent( - text=f"Error: Blocked PII entity detected - {e!s}", + text=f"Error: Blocked PII entity detected - {e}", type="text", ) ], isError=True, ) except GuardrailRaisedException as e: - verbose_logger.error(f"GuardrailRaisedException in MCP tool call: {e!s}") + verbose_logger.error(f"GuardrailRaisedException in MCP tool call: {e}") return CallToolResult( - content=[TextContent(text=f"Error: Guardrail violation - {e!s}", type="text")], + content=[TextContent(text=f"Error: Guardrail violation - {e}", type="text")], isError=True, ) except HTTPException as e: - verbose_logger.error(f"HTTPException in MCP tool call: {e!s}") + verbose_logger.error(f"HTTPException in MCP tool call: {e}") return CallToolResult( - content=[TextContent(text=f"Error: {e.detail!s}", type="text")], + content=[TextContent(text=f"Error: {e.detail}", type="text")], isError=True, ) except MCPUpstreamAuthError as e: @@ -1121,7 +1121,7 @@ if MCP_AVAILABLE: except Exception as e: verbose_logger.exception(f"MCP mcp_server_tool_call - error: {e}") return CallToolResult( - content=[TextContent(text=f"Error: {e!s}", type="text")], + content=[TextContent(text=f"Error: {e}", type="text")], isError=True, ) @@ -1173,7 +1173,7 @@ if MCP_AVAILABLE: verbose_logger.info(f"MCP list_prompts - Successfully returned {len(prompts)} prompts") return prompts except Exception as e: - verbose_logger.exception(f"Error in list_prompts endpoint: {e!s}") + verbose_logger.exception(f"Error in list_prompts endpoint: {e}") # Return empty list instead of failing completely # This prevents the HTTP stream from failing and allows the client to get a response return [] @@ -1265,7 +1265,7 @@ if MCP_AVAILABLE: verbose_logger.info(f"MCP list_resources - Successfully returned {len(resources)} resources") return resources except Exception as e: - verbose_logger.exception(f"Error in list_resources endpoint: {e!s}") + verbose_logger.exception(f"Error in list_resources endpoint: {e}") return [] finally: if _session_reset_token is not None: @@ -1310,7 +1310,7 @@ if MCP_AVAILABLE: ) return resource_templates except Exception as e: - verbose_logger.exception(f"Error in list_resource_templates endpoint: {e!s}") + verbose_logger.exception(f"Error in list_resource_templates endpoint: {e}") return [] finally: if _session_reset_token is not None: @@ -2036,7 +2036,7 @@ if MCP_AVAILABLE: verbose_logger.debug(f"MCP list_tools: omitting {server.name}; it needs upstream auth") return [], classify_list_exception(e) except Exception as e: - verbose_logger.exception(f"Error getting tools from server {server.name}: {e!s}") + verbose_logger.exception(f"Error getting tools from server {server.name}: {e}") return [], classify_list_exception(e) # Fetch tools from all servers in parallel @@ -2169,7 +2169,7 @@ if MCP_AVAILABLE: verbose_logger.debug(f"Successfully fetched {len(prompts)} prompts from server {server.name}") except Exception as e: - verbose_logger.exception(f"Error getting prompts from server {server.name}: {e!s}") + verbose_logger.exception(f"Error getting prompts from server {server.name}: {e}") # Continue with other servers instead of failing completely verbose_logger.info(f"Successfully fetched {len(all_prompts)} prompts total from all MCP servers") @@ -2221,7 +2221,7 @@ if MCP_AVAILABLE: verbose_logger.debug(f"Successfully fetched {len(resources)} resources from server {server.name}") except Exception as e: - verbose_logger.exception(f"Error getting resources from server {server.name}: {e!s}") + verbose_logger.exception(f"Error getting resources from server {server.name}: {e}") verbose_logger.info(f"Successfully fetched {len(all_resources)} resources total from all MCP servers") @@ -2359,7 +2359,7 @@ if MCP_AVAILABLE: verbose_logger.debug(f"Successfully fetched {len(listing.tools)} tools from managed MCP servers") return listing except Exception as e: - verbose_logger.exception(f"Error getting tools from managed MCP servers: {e!s}") + verbose_logger.exception(f"Error getting tools from managed MCP servers: {e}") # Continue with an empty listing instead of failing completely return AggregateToolListing(tools=[], outcomes={}) @@ -2398,7 +2398,7 @@ if MCP_AVAILABLE: ) verbose_logger.debug(f"Successfully fetched {len(managed_prompts)} prompts from managed MCP servers") except Exception as e: - verbose_logger.exception(f"Error getting tools from managed MCP servers: {e!s}") + verbose_logger.exception(f"Error getting tools from managed MCP servers: {e}") # Continue with empty managed tools list instead of failing completely return managed_prompts @@ -2428,7 +2428,7 @@ if MCP_AVAILABLE: ) verbose_logger.debug(f"Successfully fetched {len(managed_resources)} resources from managed MCP servers") except Exception as e: - verbose_logger.exception(f"Error getting resources from managed MCP servers: {e!s}") + verbose_logger.exception(f"Error getting resources from managed MCP servers: {e}") return managed_resources @@ -3335,8 +3335,8 @@ if MCP_AVAILABLE: result = tool.handler(**arguments) return [TextContent(text=str(result), type="text")] except Exception as e: - verbose_logger.exception(f"Error executing local tool {name}: {e!s}") - return [TextContent(text=f"Error: {e!s}", type="text")] + verbose_logger.exception(f"Error executing local tool {name}: {e}") + return [TextContent(text=f"Error: {e}", type="text")] def _get_mcp_servers_in_path(path: str) -> list[str] | None: """ diff --git a/litellm/proxy/_experimental/mcp_server/toolset_db.py b/litellm/proxy/_experimental/mcp_server/toolset_db.py index 62733edf378..a321c40b9e2 100644 --- a/litellm/proxy/_experimental/mcp_server/toolset_db.py +++ b/litellm/proxy/_experimental/mcp_server/toolset_db.py @@ -55,7 +55,7 @@ async def list_mcp_toolsets( rows = await MCPToolsetRepository(prisma_client).table.find_many(where=where) return [_toolset_from_row(r) for r in rows] except Exception as e: - verbose_proxy_logger.warning(f"litellm.proxy._experimental.mcp_server.toolset_db::list_mcp_toolsets - {e!s}") + verbose_proxy_logger.warning(f"litellm.proxy._experimental.mcp_server.toolset_db::list_mcp_toolsets - {e}") return [] diff --git a/litellm/proxy/a2a/discovery.py b/litellm/proxy/a2a/discovery.py index 66c661972a8..ffd85331bfd 100644 --- a/litellm/proxy/a2a/discovery.py +++ b/litellm/proxy/a2a/discovery.py @@ -122,11 +122,11 @@ async def fetch_well_known_card( # dict so production (``user_url_validation=True``) doesn't 500. response = await async_safe_get(client, url, headers=headers or {}) except SSRFError as exc: - last_error = f"{url}: {exc!s}" + last_error = f"{url}: {exc}" verbose_proxy_logger.debug("A2A discovery blocked by SSRF guard for %s: %s", url, exc) continue except Exception as exc: - last_error = f"{url}: {exc!s}" + last_error = f"{url}: {exc}" verbose_proxy_logger.debug("A2A discovery failed for %s: %s", url, exc) continue @@ -138,7 +138,7 @@ async def fetch_well_known_card( try: card = response.json() except Exception as exc: - last_error = f"{url}: invalid JSON ({exc!s})" + last_error = f"{url}: invalid JSON ({exc})" continue if not isinstance(card, dict): diff --git a/litellm/proxy/a2a/endpoints.py b/litellm/proxy/a2a/endpoints.py index bcc07629ab1..cd5024a8456 100644 --- a/litellm/proxy/a2a/endpoints.py +++ b/litellm/proxy/a2a/endpoints.py @@ -104,7 +104,7 @@ async def discover_agent_card( raise HTTPException(status_code=400, detail=str(exc)) except Exception as exc: verbose_proxy_logger.exception("Unexpected error during A2A discovery: %s", exc) - raise HTTPException(status_code=500, detail=f"Discovery failed: {exc!s}") + raise HTTPException(status_code=500, detail=f"Discovery failed: {exc}") return JSONResponse( content={"url": request.url, "agent_card": card}, diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py index 79808c06daa..6d48b31658b 100644 --- a/litellm/proxy/agent_endpoints/a2a_endpoints.py +++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py @@ -441,7 +441,7 @@ async def _handle_stream_message( "message": getattr( proxy_exc, "message", - f"Streaming error: {proxy_exc!s}", + f"Streaming error: {proxy_exc}", ), }, } @@ -491,7 +491,7 @@ async def _handle_stream_message( "id": request_id, "error": { "code": -32603, - "message": f"Streaming error: {e!s}", + "message": f"Streaming error: {e}", }, } ) @@ -974,4 +974,4 @@ async def invoke_agent_a2a( ) except Exception: pass - return _jsonrpc_error(body.get("id"), -32603, f"Internal error: {e!s}", 500) + return _jsonrpc_error(body.get("id"), -32603, f"Internal error: {e}", 500) diff --git a/litellm/proxy/agent_endpoints/agent_registry.py b/litellm/proxy/agent_endpoints/agent_registry.py index 60367178c7f..5ae992648d7 100644 --- a/litellm/proxy/agent_endpoints/agent_registry.py +++ b/litellm/proxy/agent_endpoints/agent_registry.py @@ -271,7 +271,7 @@ class AgentRegistry: created_agent_dict["object_permission"] = created_agent.object_permission.dict() return AgentResponse(**created_agent_dict) # type: ignore except Exception as e: - raise Exception(f"Error adding agent to DB: {e!s}") + raise Exception(f"Error adding agent to DB: {e}") async def delete_agent_from_db(self, agent_id: str, prisma_client: PrismaClient) -> Mapping[str, object]: """ @@ -281,7 +281,7 @@ class AgentRegistry: deleted_agent = await agents_table(prisma_client).delete(where={"agent_id": agent_id}) return dict(deleted_agent) except Exception as e: - raise Exception(f"Error deleting agent from DB: {e!s}") + raise Exception(f"Error deleting agent from DB: {e}") async def patch_agent_in_db( self, @@ -363,7 +363,7 @@ class AgentRegistry: patched_agent_dict["object_permission"] = patched_agent.object_permission.dict() return AgentResponse(**patched_agent_dict) # type: ignore except Exception as e: - raise Exception(f"Error patching agent in DB: {e!s}") + raise Exception(f"Error patching agent in DB: {e}") async def update_agent_in_db( self, @@ -450,7 +450,7 @@ class AgentRegistry: updated_agent_dict["object_permission"] = updated_agent.object_permission.dict() return AgentResponse(**updated_agent_dict) # type: ignore except Exception as e: - raise Exception(f"Error updating agent in DB: {e!s}") + raise Exception(f"Error updating agent in DB: {e}") @staticmethod async def get_all_agents_from_db( @@ -478,7 +478,7 @@ class AgentRegistry: return agents except Exception as e: - raise Exception(f"Error getting agents from DB: {e!s}") + raise Exception(f"Error getting agents from DB: {e}") def get_agent_by_id( self, @@ -494,7 +494,7 @@ class AgentRegistry: return None except Exception as e: - raise Exception(f"Error getting agent from DB: {e!s}") + raise Exception(f"Error getting agent from DB: {e}") def get_agent_by_name(self, agent_name: str) -> AgentResponse | None: """ @@ -507,7 +507,7 @@ class AgentRegistry: return None except Exception as e: - raise Exception(f"Error getting agent from DB: {e!s}") + raise Exception(f"Error getting agent from DB: {e}") global_agent_registry = AgentRegistry() diff --git a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py index 8acff11b009..6999228c83d 100644 --- a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py +++ b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py @@ -59,7 +59,7 @@ class AgentRequestHandler: return list(set(allowed_agents)) except Exception as e: - verbose_logger.warning(f"Failed to get allowed agents: {e!s}") + verbose_logger.warning(f"Failed to get allowed agents: {e}") return [] @staticmethod @@ -179,7 +179,7 @@ class AgentRequestHandler: return list(set(all_agents)) except Exception as e: - verbose_logger.warning(f"Failed to get allowed agents for key: {e!s}") + verbose_logger.warning(f"Failed to get allowed agents for key: {e}") return [] @staticmethod @@ -255,7 +255,7 @@ class AgentRequestHandler: # litellm-dashboard is the default UI team and will never have agents; # skip noisy warnings for it. if user_api_key_auth.team_id != UI_TEAM_ID: - verbose_logger.warning(f"Failed to get allowed agents for team: {e!s}") + verbose_logger.warning(f"Failed to get allowed agents for team: {e}") return [] @staticmethod @@ -310,7 +310,7 @@ class AgentRequestHandler: return list(agent_ids) except Exception as e: - verbose_logger.warning(f"Failed to get agents from access groups: {e!s}") + verbose_logger.warning(f"Failed to get agents from access groups: {e}") return [] @staticmethod @@ -369,7 +369,7 @@ class AgentRequestHandler: return key_object_permission.agent_access_groups or [] except Exception as e: - verbose_logger.warning(f"Failed to get agent access groups for key: {e!s}") + verbose_logger.warning(f"Failed to get agent access groups for key: {e}") return [] @staticmethod @@ -412,5 +412,5 @@ class AgentRequestHandler: return object_permissions.agent_access_groups or [] except Exception as e: - verbose_logger.warning(f"Failed to get agent access groups for team: {e!s}") + verbose_logger.warning(f"Failed to get agent access groups for team: {e}") return [] diff --git a/litellm/proxy/agent_endpoints/endpoints.py b/litellm/proxy/agent_endpoints/endpoints.py index 1efbdeb0132..db5341dbe5a 100644 --- a/litellm/proxy/agent_endpoints/endpoints.py +++ b/litellm/proxy/agent_endpoints/endpoints.py @@ -316,8 +316,8 @@ async def get_agents( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.agent_endpoints.get_agents(): Exception occurred - {e!s}") - raise HTTPException(status_code=500, detail={"error": f"Internal server error: {e!s}"}) + verbose_proxy_logger.exception(f"litellm.proxy.agent_endpoints.get_agents(): Exception occurred - {e}") + raise HTTPException(status_code=500, detail={"error": f"Internal server error: {e}"}) #### CRUD ENDPOINTS FOR AGENTS #### diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py index 03fdede0cf4..bf797b92850 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py @@ -121,7 +121,7 @@ async def get_marketplace(): verbose_proxy_logger.exception(f"Error generating marketplace: {e}") raise HTTPException( status_code=500, - detail={"error": f"Failed to generate marketplace: {e!s}"}, + detail={"error": f"Failed to generate marketplace: {e}"}, ) @@ -304,7 +304,7 @@ async def register_plugin( verbose_proxy_logger.exception(f"Error registering plugin: {e}") raise HTTPException( status_code=500, - detail={"error": f"Registration failed: {e!s}"}, + detail={"error": f"Registration failed: {e}"}, ) diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 4b566caf2b1..5535928dfac 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -142,7 +142,7 @@ async def anthropic_response( _usage = _blocked_response_usage(e.original_response) _anthropic_response = AnthropicMessagesResponse( - id=f"msg_{uuid.uuid4()!s}", + id=f"msg_{uuid.uuid4()}", type="message", role="assistant", content=[{"type": "text", "text": e.message}], @@ -189,7 +189,7 @@ async def anthropic_response( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.anthropic_response(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.anthropic_response(): Exception occured - {e}") # Extract model_id from request metadata (same as success path) litellm_metadata = data.get("litellm_metadata", {}) or {} @@ -209,7 +209,7 @@ async def anthropic_response( litellm_logging_obj=None, ) - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), @@ -301,8 +301,8 @@ async def count_tokens( detail=detail, ) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.anthropic_endpoints.count_tokens(): Exception occurred - {e!s}") - raise HTTPException(status_code=500, detail={"error": f"Internal server error: {e!s}"}) + verbose_proxy_logger.exception(f"litellm.proxy.anthropic_endpoints.count_tokens(): Exception occurred - {e}") + raise HTTPException(status_code=500, detail={"error": f"Internal server error: {e}"}) @router.post( diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index f876b303510..52943737eed 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -246,7 +246,7 @@ def _is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None except Exception as e: # If we can't determine the cost, assume it has cost (conservative approach) - verbose_proxy_logger.debug(f"Error checking cost for model {model_name}: {e!s}, assuming it has cost") + verbose_proxy_logger.debug(f"Error checking cost for model {model_name}: {e}, assuming it has cost") return False # All models checked have zero cost @@ -973,7 +973,7 @@ async def get_default_end_user_budget( return _budget_obj except Exception as e: - verbose_proxy_logger.error(f"Error fetching default end user budget: {e!s}") + verbose_proxy_logger.error(f"Error fetching default end user budget: {e}") return None @@ -2238,7 +2238,7 @@ async def get_team_object_by_alias( verbose_proxy_logger.exception("Error looking up team by alias: %s", team_alias) raise HTTPException( status_code=500, - detail={"error": f"Error looking up team by alias '{team_alias}': {e!s}"}, + detail={"error": f"Error looking up team by alias '{team_alias}': {e}"}, ) @@ -2324,7 +2324,7 @@ async def get_org_object_by_alias( verbose_proxy_logger.exception("Error looking up organization by alias: %s", org_alias) raise HTTPException( status_code=500, - detail={"error": f"Error looking up organization by alias '{org_alias}': {e!s}"}, + detail={"error": f"Error looking up organization by alias '{org_alias}': {e}"}, ) diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index e96d3db65ff..681647814e7 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -95,7 +95,7 @@ class UserAPIKeyAuthExceptionHandler: use_x_forwarded_for=general_settings.get("use_x_forwarded_for", False), ) verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - {e!s}\nRequester IP Address:{requester_ip}", + f"litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - {e}\nRequester IP Address:{requester_ip}", extra={"requester_ip": requester_ip}, ) @@ -150,7 +150,7 @@ class UserAPIKeyAuthExceptionHandler: ) if isinstance(e, HTTPException): raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e!s})"), + message=getattr(e, "detail", f"Authentication Error({e})"), type=ProxyErrorTypes.auth_error, param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_401_UNAUTHORIZED), diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index dfa5b22d285..a03ed13180c 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -582,7 +582,7 @@ def route_in_additonal_public_routes(current_route: str): return False except Exception as e: - verbose_proxy_logger.error(f"route_in_additonal_public_routes: {e!s}") + verbose_proxy_logger.error(f"route_in_additonal_public_routes: {e}") return False @@ -619,7 +619,7 @@ def get_request_route(request: Request) -> str: return raw_path except Exception as e: verbose_proxy_logger.debug( - f"error on get_request_route: {e!s}, defaulting to request.url.path={request.url.path}" + f"error on get_request_route: {e}, defaulting to request.url.path={request.url.path}" ) return str(request.url.path) @@ -639,7 +639,7 @@ def get_request_route_template(request: Request) -> str | None: template = getattr(route, "path", None) return template if isinstance(template, str) and template else None except Exception as e: - verbose_proxy_logger.debug(f"error on get_request_route_template: {e!s}") + verbose_proxy_logger.debug(f"error on get_request_route_template: {e}") return None diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index b1ccdc87830..cf4b47e3180 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -777,8 +777,8 @@ class JWTHandler: return userinfo except Exception as e: - verbose_proxy_logger.error(f"Error fetching OIDC UserInfo: {e!s}") - raise Exception(f"Failed to fetch OIDC UserInfo: {e!s}") + verbose_proxy_logger.error(f"Error fetching OIDC UserInfo: {e}") + raise Exception(f"Failed to fetch OIDC UserInfo: {e}") _unscoped_jwt_warning_emitted = False @@ -987,7 +987,7 @@ class JWTHandler: code=status.HTTP_401_UNAUTHORIZED, ) except Exception as e: - raise Exception(f"Validation fails: {e!s}") + raise Exception(f"Validation fails: {e}") return self._apply_issuer_claim_mappings( token=payload, @@ -1032,7 +1032,7 @@ class JWTHandler: code=status.HTTP_401_UNAUTHORIZED, ) except Exception as e: - raise Exception(f"Validation fails: {e!s}") + raise Exception(f"Validation fails: {e}") raise Exception("Invalid JWT Submitted") diff --git a/litellm/proxy/auth/litellm_license.py b/litellm/proxy/auth/litellm_license.py index a25f3e58d2c..1f61ef7ea28 100644 --- a/litellm/proxy/auth/litellm_license.py +++ b/litellm/proxy/auth/litellm_license.py @@ -48,7 +48,7 @@ class LicenseCheck: else: self.public_key = None except Exception as e: - verbose_proxy_logger.error(f"Error reading public key: {e!s}") + verbose_proxy_logger.error(f"Error reading public key: {e}") def _verify(self, license_str: str) -> bool: verbose_proxy_logger.debug( @@ -84,7 +84,7 @@ class LicenseCheck: return premium except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.auth.litellm_license.py::_verify - Unable to verify License={license_str} via api. - {e!s}" + f"litellm.proxy.auth.litellm_license.py::_verify - Unable to verify License={license_str} via api. - {e}" ) return False @@ -187,6 +187,6 @@ class LicenseCheck: except Exception as e: verbose_proxy_logger.debug( - f"litellm.proxy.auth.litellm_license.py::verify_license_without_api_request - Unable to verify License locally. - {e!s}" + f"litellm.proxy.auth.litellm_license.py::verify_license_without_api_request - Unable to verify License locally. - {e}" ) return False diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 72450453174..2905eb86c0f 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -1481,7 +1481,7 @@ async def _user_api_key_auth_builder( except Exception as e: if isinstance(e, litellm.BudgetExceededError): raise e - verbose_proxy_logger.debug(f"Unable to find user in db. Error - {e!s}") + verbose_proxy_logger.debug(f"Unable to find user in db. Error - {e}") ### CHECK IF ADMIN ### # note: never string compare api keys, this is vulenerable to a time attack. Use secrets.compare_digest instead @@ -1729,7 +1729,7 @@ async def _user_api_key_auth_builder( ) except Exception as e: verbose_logger.debug( - f"litellm.proxy.auth.user_api_key_auth.py::user_api_key_auth() - Unable to get user from db/cache. Setting user_obj to None. Exception received - {e!s}" + f"litellm.proxy.auth.user_api_key_auth.py::user_api_key_auth() - Unable to get user from db/cache. Setting user_obj to None. Exception received - {e}" ) user_obj = None @@ -2754,7 +2754,7 @@ async def _lookup_end_user_and_apply_budget( except Exception as e: if isinstance(e, litellm.BudgetExceededError): raise e - verbose_proxy_logger.debug(f"Unable to find user in db. Error - {e!s}") + verbose_proxy_logger.debug(f"Unable to find user in db. Error - {e}") return valid_token, end_user_object diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index 8b2a437009d..0c2764db33f 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -340,7 +340,7 @@ async def create_batch( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.create_batch(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.create_batch(): Exception occured - {e}") raise handle_exception_on_proxy(e) @@ -592,7 +592,7 @@ async def retrieve_batch( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.retrieve_batch(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.retrieve_batch(): Exception occured - {e}") raise handle_exception_on_proxy(e) @@ -773,7 +773,7 @@ async def list_batches( original_exception=e, request_data={"after": after, "limit": limit}, ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.retrieve_batch(): Exception occured - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.retrieve_batch(): Exception occured - {e}") raise handle_exception_on_proxy(e) @@ -982,7 +982,7 @@ async def cancel_batch( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.create_batch(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.create_batch(): Exception occured - {e}") raise handle_exception_on_proxy(e) diff --git a/litellm/proxy/caching_routes.py b/litellm/proxy/caching_routes.py index e64f3e9e7e3..50b2f63e18a 100644 --- a/litellm/proxy/caching_routes.py +++ b/litellm/proxy/caching_routes.py @@ -43,7 +43,7 @@ def _extract_cache_params() -> dict[str, Any]: cleaned_params = HealthCheckCacheParams(**cache_params).model_dump() if cache_params else {} return masker.mask_dict(cleaned_params) except (AttributeError, TypeError) as e: - verbose_proxy_logger.debug(f"Error extracting cache params: {e!s}") + verbose_proxy_logger.debug(f"Error extracting cache params: {e}") return {} @@ -158,7 +158,7 @@ async def cache_delete(request: Request): except Exception as e: raise HTTPException( status_code=500, - detail=f"Cache Delete Failed({e!s})", + detail=f"Cache Delete Failed({e})", ) @@ -173,7 +173,7 @@ def _get_redis_client_info(cache_instance) -> tuple[list, int]: client_list = cache_instance.client_list() return client_list, len(client_list) except Exception as e: - verbose_proxy_logger.warning(f"CLIENT LIST command failed (likely restricted on managed Redis): {e!s}") + verbose_proxy_logger.warning(f"CLIENT LIST command failed (likely restricted on managed Redis): {e}") return ["CLIENT LIST command not available on this Redis instance"], -1 @@ -209,7 +209,7 @@ async def cache_redis_info(): except Exception as e: raise HTTPException( status_code=503, - detail=f"Service Unhealthy ({e!s})", + detail=f"Service Unhealthy ({e})", ) @@ -245,5 +245,5 @@ async def cache_flushall(): except Exception as e: raise HTTPException( status_code=503, - detail=f"Service Unhealthy ({e!s})", + detail=f"Service Unhealthy ({e})", ) diff --git a/litellm/proxy/client/cli/commands/chat.py b/litellm/proxy/client/cli/commands/chat.py index f0c91be686d..3e86c79a90b 100644 --- a/litellm/proxy/client/cli/commands/chat.py +++ b/litellm/proxy/client/cli/commands/chat.py @@ -386,5 +386,5 @@ def _stream_response( console.print(f"[red]{e.response.text}[/red]") return None except Exception as e: - console.print(f"\n[red]Error: {e!s}[/red]") + console.print(f"\n[red]Error: {e}[/red]") return None diff --git a/litellm/proxy/client/cli/commands/credentials.py b/litellm/proxy/client/cli/commands/credentials.py index 8187f811778..cdfa4c5cd69 100644 --- a/litellm/proxy/client/cli/commands/credentials.py +++ b/litellm/proxy/client/cli/commands/credentials.py @@ -71,7 +71,7 @@ def create(ctx: click.Context, credential_name: str, info: str, values: str): credential_info = json.loads(info) credential_values = json.loads(values) except json.JSONDecodeError as e: - raise click.BadParameter(f"Invalid JSON: {e!s}") + raise click.BadParameter(f"Invalid JSON: {e}") try: response = client.create(credential_name, credential_info, credential_values) diff --git a/litellm/proxy/client/cli/commands/keys.py b/litellm/proxy/client/cli/commands/keys.py index ec5dca25518..8ebed1749f4 100644 --- a/litellm/proxy/client/cli/commands/keys.py +++ b/litellm/proxy/client/cli/commands/keys.py @@ -122,7 +122,7 @@ def generate( aliases_dict = json.loads(aliases) if aliases else None config_dict = json.loads(config) if config else None except json.JSONDecodeError as e: - raise click.BadParameter(f"Invalid JSON: {e!s}") + raise click.BadParameter(f"Invalid JSON: {e}") try: response = client.generate( models=models_list, @@ -316,7 +316,7 @@ def _import_keys_to_destination( except Exception as e: failed_count += 1 key_alias = key.get("key_alias", "N/A") - click.echo(f"Failed to import key {key_alias}: {e!s}", err=True) + click.echo(f"Failed to import key {key_alias}: {e}", err=True) return imported_count, failed_count @@ -389,5 +389,5 @@ def import_keys( click.echo(e.response.text, err=True) raise click.Abort() except Exception as e: - click.echo(f"Error: {e!s}", err=True) + click.echo(f"Error: {e}", err=True) raise click.Abort() diff --git a/litellm/proxy/client/cli/commands/teams.py b/litellm/proxy/client/cli/commands/teams.py index 442ac40a775..2d88e4bbce2 100644 --- a/litellm/proxy/client/cli/commands/teams.py +++ b/litellm/proxy/client/cli/commands/teams.py @@ -76,7 +76,7 @@ def list(ctx: click.Context): click.echo(f"Details: {error_body.get('detail', 'Unknown error')}", err=True) raise click.Abort() except Exception as e: - click.echo(f"Error: {e!s}", err=True) + click.echo(f"Error: {e}", err=True) raise click.Abort() @@ -99,7 +99,7 @@ def available(ctx: click.Context): error_body = e.response.json() click.echo(f"Details: {error_body.get('detail', 'Unknown error')}", err=True) except Exception as e: - click.echo(f"Error: {e!s}", err=True) + click.echo(f"Error: {e}", err=True) raise click.Abort() @@ -158,5 +158,5 @@ def assign_key(ctx: click.Context, team_id: str | None): click.echo(f"Details: {error_body.get('detail', 'Unknown error')}", err=True) raise click.Abort() except Exception as e: - click.echo(f"Error: {e!s}", err=True) + click.echo(f"Error: {e}", err=True) raise click.Abort() diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 4d4f459a080..cb688860280 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -908,7 +908,7 @@ def _log_llm_api_exception(e: Exception) -> None: "litellm.proxy.proxy_server._handle_llm_api_exception(): client disconnected, upstream LLM request cancelled" ) return - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - {e}") async def _cancel_llm_call_on_client_disconnect( @@ -2696,7 +2696,7 @@ class ProxyBaseLLMRequestProcessing: status_code=http_status_error.response.status_code, detail={"error": error_text}, ) - error_msg = f"{e!s}" + error_msg = f"{e}" # Check for AttributeError in the exception chain. # The AttributeError may be wrapped in multiple layers # (e.g. AttributeError -> OpenAIException -> APIConnectionError), @@ -2898,7 +2898,7 @@ class ProxyBaseLLMRequestProcessing: raise except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.async_data_generator(): Exception occured - {e!s}" + f"litellm.proxy.proxy_server.async_data_generator(): Exception occured - {e}" ) transformed_exception = await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, @@ -2914,7 +2914,7 @@ class ProxyBaseLLMRequestProcessing: if isinstance(e, HTTPException): raise e error_traceback = _redact_string(traceback.format_exc()) - error_msg = f"{e!s}\n\n{error_traceback}" + error_msg = f"{e}\n\n{error_traceback}" proxy_exception = ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), diff --git a/litellm/proxy/common_utils/custom_openapi_spec.py b/litellm/proxy/common_utils/custom_openapi_spec.py index 55d6f083fdd..a884eab462a 100644 --- a/litellm/proxy/common_utils/custom_openapi_spec.py +++ b/litellm/proxy/common_utils/custom_openapi_spec.py @@ -273,7 +273,7 @@ class CustomOpenAPISpec: except Exception as e: # If schema addition fails, continue without it - verbose_proxy_logger.debug(f"Failed to add {operation_name} request schema: {e!s}") + verbose_proxy_logger.debug(f"Failed to add {operation_name} request schema: {e}") return openapi_schema @@ -302,7 +302,7 @@ class CustomOpenAPISpec: operation_name="chat completion", ) except ImportError as e: - verbose_proxy_logger.debug(f"Failed to import ProxyChatCompletionRequest: {e!s}") + verbose_proxy_logger.debug(f"Failed to import ProxyChatCompletionRequest: {e}") return openapi_schema @staticmethod @@ -328,7 +328,7 @@ class CustomOpenAPISpec: operation_name="embedding", ) except ImportError as e: - verbose_proxy_logger.debug(f"Failed to import EmbeddingRequest: {e!s}") + verbose_proxy_logger.debug(f"Failed to import EmbeddingRequest: {e}") return openapi_schema @staticmethod @@ -356,7 +356,7 @@ class CustomOpenAPISpec: operation_name="responses API", ) except ImportError as e: - verbose_proxy_logger.debug(f"Failed to import ResponsesAPIRequestParams: {e!s}") + verbose_proxy_logger.debug(f"Failed to import ResponsesAPIRequestParams: {e}") return openapi_schema @staticmethod diff --git a/litellm/proxy/common_utils/debug_utils.py b/litellm/proxy/common_utils/debug_utils.py index 7d3150a3e72..7d2b303a7ca 100644 --- a/litellm/proxy/common_utils/debug_utils.py +++ b/litellm/proxy/common_utils/debug_utils.py @@ -653,7 +653,7 @@ async def configure_gc_thresholds_endpoint( ) except Exception as e: verbose_proxy_logger.error(f"Failed to set GC thresholds: {e}") - raise HTTPException(status_code=500, detail=f"Failed to set GC thresholds: {e!s}") + raise HTTPException(status_code=500, detail=f"Failed to set GC thresholds: {e}") # Get current object count to show immediate impact current_count = gc.get_count()[0] @@ -783,4 +783,4 @@ def init_verbose_loggers(): except Exception as e: import logging - logging.warning(f"Failed to init verbose loggers: {e!s}") + logging.warning(f"Failed to init verbose loggers: {e}") diff --git a/litellm/proxy/common_utils/encrypt_decrypt_utils.py b/litellm/proxy/common_utils/encrypt_decrypt_utils.py index b7b8bfd1eea..651e59ef959 100644 --- a/litellm/proxy/common_utils/encrypt_decrypt_utils.py +++ b/litellm/proxy/common_utils/encrypt_decrypt_utils.py @@ -145,7 +145,7 @@ def decrypt_value_helper( # if it's not str - do not decrypt it, return the value return value except Exception as e: - error_message = f"Error decrypting value for key: {key}, Did your master_key/salt key change recently? \nError: {e!s}\nSet permanent salt key - https://docs.litellm.ai/docs/proxy/prod#5-set-litellm-salt-key" + error_message = f"Error decrypting value for key: {key}, Did your master_key/salt key change recently? \nError: {e}\nSet permanent salt key - https://docs.litellm.ai/docs/proxy/prod#5-set-litellm-salt-key" if exception_type == "debug": verbose_proxy_logger.debug(error_message) return value if return_original_value else None diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 0dd910e1901..67212539cc4 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -98,9 +98,9 @@ async def _read_request_body(request: Request | None) -> dict: # Above the configured size, skip the repair and raise the 400 now. repair_limit_bytes = MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB * 1024 * 1024 if repair_limit_bytes > 0 and len(body) > repair_limit_bytes: - verbose_proxy_logger.error(f"Invalid JSON payload received: {e!s}") + verbose_proxy_logger.error(f"Invalid JSON payload received: {e}") raise ProxyException( - message=f"Invalid JSON payload: {e!s}", + message=f"Invalid JSON payload: {e}", type="invalid_request_error", param="request_body", code=status.HTTP_400_BAD_REQUEST, @@ -120,9 +120,9 @@ async def _read_request_body(request: Request | None) -> dict: parsed_body = json.loads(body_str) except json.JSONDecodeError: # If both orjson and json.loads fail, throw a proper error - verbose_proxy_logger.error(f"Invalid JSON payload received: {e!s}") + verbose_proxy_logger.error(f"Invalid JSON payload received: {e}") raise ProxyException( - message=f"Invalid JSON payload: {e!s}", + message=f"Invalid JSON payload: {e}", type="invalid_request_error", param="request_body", code=status.HTTP_400_BAD_REQUEST, @@ -134,7 +134,7 @@ async def _read_request_body(request: Request | None) -> dict: except (json.JSONDecodeError, orjson.JSONDecodeError, ProxyException) as e: # Re-raise ProxyException as-is - verbose_proxy_logger.error(f"Invalid JSON payload received: {e!s}") + verbose_proxy_logger.error(f"Invalid JSON payload received: {e}") raise except Exception as e: # Catch unexpected errors to avoid crashes @@ -426,7 +426,7 @@ def extract_nested_form_metadata(form_data: dict[str, Any], prefix: str = "litel verbose_proxy_logger.warning(f"Cannot set value - parent is not a dict for key: {key}") except Exception as e: - verbose_proxy_logger.error(f"Error parsing metadata key '{key}': {e!s}") + verbose_proxy_logger.error(f"Error parsing metadata key '{key}': {e}") continue return metadata diff --git a/litellm/proxy/common_utils/load_config_utils.py b/litellm/proxy/common_utils/load_config_utils.py index 225f7cfdf6c..56aedb76590 100644 --- a/litellm/proxy/common_utils/load_config_utils.py +++ b/litellm/proxy/common_utils/load_config_utils.py @@ -34,9 +34,9 @@ def get_file_contents_from_s3(bucket_name, object_key): except ImportError as e: # this is most likely if a user is not using the litellm docker container - verbose_proxy_logger.error(f"ImportError: {e!s}") + verbose_proxy_logger.error(f"ImportError: {e}") except Exception as e: - verbose_proxy_logger.error(f"Error retrieving file contents: {e!s}") + verbose_proxy_logger.error(f"Error retrieving file contents: {e}") return None @@ -57,7 +57,7 @@ async def get_config_file_contents_from_gcs(bucket_name, object_key): return config except Exception as e: - verbose_proxy_logger.error(f"Error retrieving file contents: {e!s}") + verbose_proxy_logger.error(f"Error retrieving file contents: {e}") return None @@ -111,10 +111,10 @@ def download_python_file_from_s3( return True except ImportError as e: - verbose_proxy_logger.error(f"ImportError: {e!s}") + verbose_proxy_logger.error(f"ImportError: {e}") return False except Exception as e: - verbose_proxy_logger.exception(f"Error downloading Python file: {e!s}") + verbose_proxy_logger.exception(f"Error downloading Python file: {e}") return False @@ -158,7 +158,7 @@ async def download_python_file_from_gcs( return True except Exception as e: - verbose_proxy_logger.exception(f"Error downloading Python file from GCS: {e!s}") + verbose_proxy_logger.exception(f"Error downloading Python file from GCS: {e}") return False diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py index b5ff2ffa9cc..d7c70bdb20c 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py +++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py @@ -68,7 +68,7 @@ class SpendLogCleanup: return True except ValueError as e: verbose_proxy_logger.warning( - f"Invalid maximum_spend_logs_retention_period value: {retention_setting}, error: {e!s}" + f"Invalid maximum_spend_logs_retention_period value: {retention_setting}, error: {e}" ) return False diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py index 778ea729e32..4daab1caf96 100644 --- a/litellm/proxy/fine_tuning_endpoints/endpoints.py +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -199,9 +199,7 @@ async def create_fine_tuning_job( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.create_fine_tuning_job(): Exception occurred - {e!s}" - ) + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.create_fine_tuning_job(): Exception occurred - {e}") raise handle_exception_on_proxy(e) @@ -340,7 +338,7 @@ async def retrieve_fine_tuning_job( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.retrieve_fine_tuning_job(): Exception occurred - {e!s}" + f"litellm.proxy.proxy_server.retrieve_fine_tuning_job(): Exception occurred - {e}" ) raise handle_exception_on_proxy(e) @@ -468,9 +466,7 @@ async def list_fine_tuning_jobs( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.list_fine_tuning_jobs(): Exception occurred - {e!s}" - ) + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.list_fine_tuning_jobs(): Exception occurred - {e}") raise handle_exception_on_proxy(e) @@ -608,7 +604,5 @@ async def cancel_fine_tuning_job( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.cancel_fine_tuning_job(): Exception occurred - {e!s}" - ) + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.cancel_fine_tuning_job(): Exception occurred - {e}") raise handle_exception_on_proxy(e) diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index 3d8ed8dc1e2..12373b7fb97 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -1420,7 +1420,7 @@ async def get_category_yaml(category_name: str): "file_type": file_type, } except Exception as e: - raise HTTPException(status_code=500, detail=f"Error reading category file: {e!s}") + raise HTTPException(status_code=500, detail=f"Error reading category file: {e}") @router.get( @@ -1452,7 +1452,7 @@ async def get_major_airlines(): airlines = json.load(f) return {"airlines": airlines} except Exception as e: - raise HTTPException(status_code=500, detail=f"Error reading major_airlines.json: {e!s}") from e + raise HTTPException(status_code=500, detail=f"Error reading major_airlines.json: {e}") from e @router.post( @@ -1540,10 +1540,10 @@ async def validate_blocked_words_file(request: dict[str, str]): "message": f"Valid YAML file with {len(blocked_words_list)} blocked word(s)", } except yaml.YAMLError as e: - return {"valid": False, "error": f"Invalid YAML syntax: {e!s}"} + return {"valid": False, "error": f"Invalid YAML syntax: {e}"} except Exception as e: verbose_proxy_logger.exception("Error validating blocked words file") - return {"valid": False, "error": f"Validation error: {e!s}"} + return {"valid": False, "error": f"Validation error: {e}"} def _get_field_type_from_annotation(field_annotation: Any) -> str: diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index f8fedb22872..5aae14b83e6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -2266,4 +2266,4 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): raise except Exception as e: verbose_proxy_logger.error("Bedrock Guardrail: Failed to apply guardrail: %s", str(e)) - raise Exception(f"Bedrock guardrail failed: {e!s}") + raise Exception(f"Bedrock guardrail failed: {e}") diff --git a/litellm/proxy/guardrails/guardrail_hooks/custom_code/primitives.py b/litellm/proxy/guardrails/guardrail_hooks/custom_code/primitives.py index c019d445fd4..43a3671ad97 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/custom_code/primitives.py +++ b/litellm/proxy/guardrails/guardrail_hooks/custom_code/primitives.py @@ -480,10 +480,10 @@ async def http_request( return _http_success_response(e.response) except httpx.RequestError as e: verbose_proxy_logger.warning(f"Custom code http_request error: {e}") - return _http_error_response(f"Request failed: {e!s}") + return _http_error_response(f"Request failed: {e}") except Exception as e: verbose_proxy_logger.warning(f"Custom code http_request unexpected error: {e}") - return _http_error_response(f"Unexpected error: {e!s}") + return _http_error_response(f"Unexpected error: {e}") async def _execute_http_request( diff --git a/litellm/proxy/guardrails/guardrail_hooks/deepkeep/deepkeep.py b/litellm/proxy/guardrails/guardrail_hooks/deepkeep/deepkeep.py index c7ed4028218..96e7e605349 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/deepkeep/deepkeep.py +++ b/litellm/proxy/guardrails/guardrail_hooks/deepkeep/deepkeep.py @@ -228,7 +228,7 @@ class DeepKeepGuardrail(CustomGuardrail): **({"http_status_code": http_status_code} if http_status_code else {}), ) verbose_proxy_logger.error("DeepKeep guardrail API error: %s", str(error)) - raise DeepKeepGuardrailAPIError(f"DeepKeep guardrail API failed: {error!s}") + raise DeepKeepGuardrailAPIError(f"DeepKeep guardrail API failed: {error}") @staticmethod def _build_return_inputs( diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py index a694ef897ab..f1f37263eae 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py @@ -357,7 +357,7 @@ class GenericGuardrailAPI(CustomGuardrail): **({"http_status_code": http_status_code} if http_status_code else {}), ) verbose_proxy_logger.error("Generic Guardrail API: failed to make request: %s", str(error)) - raise Exception(f"Generic Guardrail API failed: {error!s}") + raise Exception(f"Generic Guardrail API failed: {error}") @log_guardrail_information async def apply_guardrail( diff --git a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py index dba82eeb32c..90d131893c6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py +++ b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py @@ -299,8 +299,8 @@ class LassoGuardrail(CustomGuardrail): except Exception as e: if isinstance(e, HTTPException): raise e - verbose_proxy_logger.error(f"Error in post-call Lasso masking: {e!s}") - raise LassoGuardrailAPIError(f"Failed to apply post-call masking: {e!s}") + verbose_proxy_logger.error(f"Error in post-call Lasso masking: {e}") + raise LassoGuardrailAPIError(f"Failed to apply post-call masking: {e}") else: # Use the same data for conversation_id consistency (no cache access needed) await self._run_lasso_guardrail(response_data, cache=global_cache, message_type="COMPLETION") @@ -599,7 +599,7 @@ class LassoGuardrail(CustomGuardrail): # Log error with context verbose_proxy_logger.error( - f"Error calling Lasso API: {error!s}", + f"Error calling Lasso API: {error}", extra={ "guardrail_name": getattr(self, "guardrail_name", "unknown"), "message_type": message_type, @@ -620,7 +620,7 @@ class LassoGuardrail(CustomGuardrail): raise LassoGuardrailAPIError(f"API error: {error.response.status_code}") # Generic error handling - raise LassoGuardrailAPIError(f"Failed to verify request safety with Lasso API: {error!s}") + raise LassoGuardrailAPIError(f"Failed to verify request safety with Lasso API: {error}") def _log_masking_applied( self, diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py index 5650a6b07ac..c6900c38cbf 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py @@ -750,7 +750,7 @@ class ContentFilterGuardrail(CustomGuardrail): except FileNotFoundError: raise FileNotFoundError(f"Blocked words file not found: {file_path}") except Exception as e: - raise Exception(f"Error loading blocked words file {file_path}: {e!s}") + raise Exception(f"Error loading blocked words file {file_path}: {e}") def _find_pattern_spans(self, text: str, pattern_entry: dict[str, Any]) -> list[tuple[int, int]]: """Return all match spans for a pattern, applying contextual rules if required.""" diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py index d30de723443..8292f575c74 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py @@ -168,7 +168,7 @@ def get_available_content_categories() -> list[dict[str, str]]: # Skip files that can't be loaded but log the error for debugging from litellm._logging import verbose_proxy_logger - verbose_proxy_logger.warning(f"Failed to load category file {filename}: {e!s}") + verbose_proxy_logger.warning(f"Failed to load category file {filename}: {e}") continue elif filename.endswith(".json"): # JSON category files (e.g. harm_toxic_abuse.json) - no YAML header, use filename diff --git a/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py b/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py index 642c1dcbca8..b2f91083cc0 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py +++ b/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py @@ -163,7 +163,7 @@ class NomaGuardrail(CustomGuardrail): try: asyncio.create_task(coro) except Exception as e: - verbose_proxy_logger.error(f"Failed to create background Noma task: {e!s}") + verbose_proxy_logger.error(f"Failed to create background Noma task: {e}") async def _process_user_message_check( self, @@ -348,7 +348,7 @@ class NomaGuardrail(CustomGuardrail): return "guardrail_failed_to_respond" except Exception as e: - verbose_proxy_logger.error(f"Error determining NOMA guardrail status: {e!s}") + verbose_proxy_logger.error(f"Error determining NOMA guardrail status: {e}") return "guardrail_failed_to_respond" def _should_only_sensitive_data_failed(self, classification_obj: dict) -> bool: @@ -513,7 +513,7 @@ class NomaGuardrail(CustomGuardrail): try: await self._process_user_message_check(request_data, user_auth) except Exception as e: - verbose_proxy_logger.error(f"Noma background user message check failed: {e!s}") + verbose_proxy_logger.error(f"Noma background user message check failed: {e}") async def _check_llm_response_background( self, @@ -525,7 +525,7 @@ class NomaGuardrail(CustomGuardrail): try: await self._process_llm_response_check(request_data, response, user_auth) except Exception as e: - verbose_proxy_logger.error(f"Noma background response check failed: {e!s}") + verbose_proxy_logger.error(f"Noma background response check failed: {e}") async def _handle_verdict_background( self, @@ -547,7 +547,7 @@ class NomaGuardrail(CustomGuardrail): msg = f"Noma guardrail allowed {type} message: {message}" verbose_proxy_logger.info(msg) except Exception as e: - verbose_proxy_logger.error(f"Noma background verdict handling failed: {e!s}") + verbose_proxy_logger.error(f"Noma background verdict handling failed: {e}") async def async_pre_call_hook( self, @@ -570,7 +570,7 @@ class NomaGuardrail(CustomGuardrail): try: self._create_background_noma_check(self._check_user_message_background(data, user_api_key_dict)) except Exception as e: - verbose_proxy_logger.error(f"Failed to start background Noma pre-call check: {e!s}") + verbose_proxy_logger.error(f"Failed to start background Noma pre-call check: {e}") return data try: @@ -594,7 +594,7 @@ class NomaGuardrail(CustomGuardrail): event_type=GuardrailEventHooks.pre_call, ) - verbose_proxy_logger.error(f"Noma pre-call hook failed: {e!s}") + verbose_proxy_logger.error(f"Noma pre-call hook failed: {e}") if self.block_failures: raise @@ -618,7 +618,7 @@ class NomaGuardrail(CustomGuardrail): try: self._create_background_noma_check(self._check_user_message_background(data, user_api_key_dict)) except Exception as e: - verbose_proxy_logger.error(f"Failed to start background Noma moderation check: {e!s}") + verbose_proxy_logger.error(f"Failed to start background Noma moderation check: {e}") return data try: @@ -642,7 +642,7 @@ class NomaGuardrail(CustomGuardrail): event_type=GuardrailEventHooks.during_call, ) - verbose_proxy_logger.error(f"Noma moderation hook failed: {e!s}") + verbose_proxy_logger.error(f"Noma moderation hook failed: {e}") if self.block_failures: raise @@ -665,7 +665,7 @@ class NomaGuardrail(CustomGuardrail): self._check_llm_response_background(data, response, user_api_key_dict) ) except Exception as e: - verbose_proxy_logger.error(f"Failed to start background Noma post-call check: {e!s}") + verbose_proxy_logger.error(f"Failed to start background Noma post-call check: {e}") return response try: @@ -689,7 +689,7 @@ class NomaGuardrail(CustomGuardrail): event_type=GuardrailEventHooks.post_call, ) - verbose_proxy_logger.error(f"Noma post-call hook failed: {e!s}") + verbose_proxy_logger.error(f"Noma post-call hook failed: {e}") if self.block_failures: raise return response @@ -828,7 +828,7 @@ class NomaGuardrail(CustomGuardrail): except Exception as e: if self.block_failures: raise - verbose_proxy_logger.error(f"Noma streaming post-call hook failed: {e!s}") + verbose_proxy_logger.error(f"Noma streaming post-call hook failed: {e}") for chunk in all_chunks: yield chunk return diff --git a/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py b/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py index b86273f754a..37ce84b8e6e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py +++ b/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py @@ -118,7 +118,7 @@ class OnyxGuardrail(CustomGuardrail): payload = parsed.get("response", {}) except Exception as e: verbose_proxy_logger.error( - f"Error in converting request_data to ModelResponse: {e!s}", + f"Error in converting request_data to ModelResponse: {e}", extra={ "conversation_id": conversation_id, "input_type": input_type, @@ -133,7 +133,7 @@ class OnyxGuardrail(CustomGuardrail): raise e except Exception as e: verbose_proxy_logger.error( - f"Error in apply_guardrail guard: {e!s}", + f"Error in apply_guardrail guard: {e}", extra={"conversation_id": conversation_id, "input_type": input_type}, ) return inputs diff --git a/litellm/proxy/guardrails/guardrail_hooks/ovalix/ovalix.py b/litellm/proxy/guardrails/guardrail_hooks/ovalix/ovalix.py index 947a81c1b79..fe2e87ff661 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/ovalix/ovalix.py +++ b/litellm/proxy/guardrails/guardrail_hooks/ovalix/ovalix.py @@ -250,7 +250,7 @@ class OvalixGuardrail(CustomGuardrail): verbose_proxy_logger.exception("Ovalix apply_guardrail checkpoint call failed: %s", e) raise GuardrailRaisedException( guardrail_name=self.guardrail_name, - message=f"Ovalix guardrail error: {e!s}", + message=f"Ovalix guardrail error: {e}", should_wrap_with_default_message=False, ) from e diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index 5d134fd01c2..782ffef61cf 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -231,7 +231,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): return " ".join(text_parts) if text_parts else "" except (AttributeError, IndexError) as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS: Error extracting response text: {e!s}") + verbose_proxy_logger.error(f"PANW Prisma AIRS: Error extracting response text: {e}") return "" async def _call_panw_api( @@ -433,7 +433,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): } except httpx.TimeoutException as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS: Timeout error: {e!s}") + verbose_proxy_logger.error(f"PANW Prisma AIRS: Timeout error: {e}") return { "action": "block", "category": "timeout_error", @@ -441,7 +441,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): } except httpx.RequestError as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS: Network/request error: {e!s}") + verbose_proxy_logger.error(f"PANW Prisma AIRS: Network/request error: {e}") return { "action": "block", "category": "network_error", @@ -449,7 +449,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): } except Exception as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS: Unexpected error: {e!s}") + verbose_proxy_logger.error(f"PANW Prisma AIRS: Unexpected error: {e}") return {"action": "block", "category": "api_error", "_is_transient": True} @staticmethod @@ -1056,7 +1056,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS scan failed: {e!s}") + verbose_proxy_logger.error(f"PANW Prisma AIRS scan failed: {e}") raise HTTPException( status_code=500, detail={ @@ -1170,7 +1170,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS scan failed: {e!s}") + verbose_proxy_logger.error(f"PANW Prisma AIRS scan failed: {e}") raise HTTPException( status_code=500, detail={ @@ -1366,7 +1366,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): error_obj["code"] = e.status_code yield f"data: {json.dumps({'error': error_obj})}\n\n" except Exception as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS streaming error: {e!s}") + verbose_proxy_logger.error(f"PANW Prisma AIRS streaming error: {e}") yield f"data: {json.dumps({'error': {'message': 'Security scan failed - streaming response blocked for safety', 'type': 'guardrail_scan_error', 'code': 500, 'guardrail': self.guardrail_name}})}\n\n" async def _scan_tool_calls_for_guardrail( diff --git a/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py b/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py index f96fca11abc..77767c8c61b 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py +++ b/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py @@ -457,7 +457,7 @@ class PillarGuardrail(CustomGuardrail): raise e # Handle API communication errors based on fallback_on_error setting - verbose_proxy_logger.error(f"Pillar Guardrail: API communication failed - {e!s}") + verbose_proxy_logger.error(f"Pillar Guardrail: API communication failed - {e}") return self._handle_api_error(e, data) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 9d0b8d2777c..7a38c4087c6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -139,9 +139,9 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): except FileNotFoundError: raise Exception(f"File not found. file_path={ad_hoc_recognizers}") except json.JSONDecodeError as e: - raise Exception(f"Error decoding JSON file: {e!s}, file_path={ad_hoc_recognizers}") + raise Exception(f"Error decoding JSON file: {e}, file_path={ad_hoc_recognizers}") except Exception as e: - raise Exception(f"An error occurred: {e!s}, file_path={ad_hoc_recognizers}") + raise Exception(f"An error occurred: {e}, file_path={ad_hoc_recognizers}") self.validate_environment( presidio_analyzer_api_base=presidio_analyzer_api_base, presidio_anonymizer_api_base=presidio_anonymizer_api_base, @@ -1124,7 +1124,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): yield mock_response_stream except Exception as e: - verbose_proxy_logger.error(f"Error masking streaming PII output: {e!s}") + verbose_proxy_logger.error(f"Error masking streaming PII output: {e}") for chunk in all_chunks: yield chunk @@ -1253,7 +1253,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): yield mock_response_stream except Exception as e: - verbose_proxy_logger.error(f"Error in PII streaming processing: {e!s}") + verbose_proxy_logger.error(f"Error in PII streaming processing: {e}") for chunk in remaining_chunks: yield chunk diff --git a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py index 9b55fcd8062..0f3a817b12c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py +++ b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py @@ -326,7 +326,7 @@ class PromptSecurityGuardrail(CustomGuardrail): except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error processing image: {e!s}") + verbose_proxy_logger.error(f"Error processing image: {e}") @staticmethod def _resolve_key_alias_from_request_data(request_data: dict) -> str | None: @@ -481,8 +481,8 @@ class PromptSecurityGuardrail(CustomGuardrail): except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error sanitizing image file: {e!s}") - raise HTTPException(status_code=500, detail=f"File sanitization failed: {e!s}") + verbose_proxy_logger.error(f"Error sanitizing image file: {e}") + raise HTTPException(status_code=500, detail=f"File sanitization failed: {e}") async def _process_document_item(self, item: dict, user_api_key_alias: str | None) -> dict: """Process and sanitize document/file items.""" @@ -554,8 +554,8 @@ class PromptSecurityGuardrail(CustomGuardrail): except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error sanitizing document: {e!s}") - raise HTTPException(status_code=500, detail=f"Document sanitization failed: {e!s}") + verbose_proxy_logger.error(f"Error sanitizing document: {e}") + raise HTTPException(status_code=500, detail=f"Document sanitization failed: {e}") async def process_message_files(self, messages: list, user_api_key_alias: str | None = None) -> list: """Process messages and sanitize any file content (images, documents, PDFs, etc.).""" diff --git a/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py b/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py index 09e5ffff193..9e16e9d5786 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py +++ b/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py @@ -351,7 +351,7 @@ class ZscalerAIGuard(CustomGuardrail): return self._handle_response(response, direction) except Exception as e: verbose_proxy_logger.error(f"{e}. Blocking request.") - user_facing_error = self._create_user_facing_error(f"{e!s}") + user_facing_error = self._create_user_facing_error(f"{e}") raise HTTPException(status_code=500, detail=user_facing_error) @staticmethod diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index aaaef95f4a4..b0e16c0ed2e 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -285,7 +285,7 @@ class GuardrailRegistry: return guardrail_dict except Exception as e: - raise Exception(f"Error adding guardrail to DB: {e!s}") + raise Exception(f"Error adding guardrail to DB: {e}") async def delete_guardrail_from_db(self, guardrail_id: str, prisma_client: PrismaClient): """ @@ -297,7 +297,7 @@ class GuardrailRegistry: return {"message": f"Guardrail {guardrail_id} deleted successfully"} except Exception as e: - raise Exception(f"Error deleting guardrail from DB: {e!s}") + raise Exception(f"Error deleting guardrail from DB: {e}") async def update_guardrail_in_db(self, guardrail_id: str, guardrail: Guardrail, prisma_client: PrismaClient): """ @@ -328,7 +328,7 @@ class GuardrailRegistry: # Convert to dict and return return dict(updated_guardrail) except Exception as e: - raise Exception(f"Error updating guardrail in DB: {e!s}") + raise Exception(f"Error updating guardrail in DB: {e}") @staticmethod async def get_all_guardrails_from_db( @@ -350,7 +350,7 @@ class GuardrailRegistry: return guardrails except Exception as e: - raise Exception(f"Error getting guardrails from DB: {e!s}") + raise Exception(f"Error getting guardrails from DB: {e}") async def get_guardrail_by_id_from_db(self, guardrail_id: str, prisma_client: PrismaClient) -> Guardrail | None: """ @@ -366,7 +366,7 @@ class GuardrailRegistry: return Guardrail(**(dict(guardrail))) # type: ignore except Exception as e: - raise Exception(f"Error getting guardrail from DB: {e!s}") + raise Exception(f"Error getting guardrail from DB: {e}") async def get_guardrail_by_name_from_db(self, guardrail_name: str, prisma_client: PrismaClient) -> Guardrail | None: """ @@ -382,7 +382,7 @@ class GuardrailRegistry: return Guardrail(**(dict(guardrail))) # type: ignore except Exception as e: - raise Exception(f"Error getting guardrail from DB: {e!s}") + raise Exception(f"Error getting guardrail from DB: {e}") class InMemoryGuardrailHandler: diff --git a/litellm/proxy/guardrails/init_guardrails.py b/litellm/proxy/guardrails/init_guardrails.py index 71ffc9d36ef..036ee5dca78 100644 --- a/litellm/proxy/guardrails/init_guardrails.py +++ b/litellm/proxy/guardrails/init_guardrails.py @@ -141,5 +141,5 @@ def initialize_guardrails( return litellm.guardrail_name_config_map except Exception as e: - verbose_proxy_logger.exception(f"error initializing guardrails {e!s}") + verbose_proxy_logger.exception(f"error initializing guardrails {e}") raise e diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 03645c0b2fa..c3bce5e8370 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -425,11 +425,11 @@ async def health_services_endpoint( } except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.health_services_endpoint(): Exception occured - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.health_services_endpoint(): Exception occured - {e}") verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e!s})"), + message=getattr(e, "detail", f"Authentication Error({e})"), type=ProxyErrorTypes.auth_error, param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), @@ -1069,7 +1069,7 @@ async def health_endpoint( ) return _post_process(router_result) except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.py::health_endpoint(): Exception occured - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.py::health_endpoint(): Exception occured - {e}") verbose_proxy_logger.debug(traceback.format_exc()) raise e @@ -1110,7 +1110,7 @@ async def health_check_history_endpoint( verbose_proxy_logger.error(f"Error getting health check history: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail={"error": f"Failed to retrieve health check history: {e!s}"}, + detail={"error": f"Failed to retrieve health check history: {e}"}, ) @@ -1142,7 +1142,7 @@ async def latest_health_checks_endpoint( verbose_proxy_logger.error(f"Error getting latest health checks: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail={"error": f"Failed to retrieve latest health checks: {e!s}"}, + detail={"error": f"Failed to retrieve latest health checks: {e}"}, ) @@ -1185,7 +1185,7 @@ async def shared_health_check_status_endpoint( verbose_proxy_logger.error(f"Error getting shared health check status: {e}") raise HTTPException( status_code=fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR, - detail={"error": f"Failed to retrieve shared health check status: {e!s}"}, + detail={"error": f"Failed to retrieve shared health check status: {e}"}, ) @@ -1473,7 +1473,7 @@ async def _get_health_readiness_details( "is_detailed_debug": is_detailed_debug, } except Exception as e: - raise HTTPException(status_code=503, detail=f"Service Unhealthy ({e!s})") + raise HTTPException(status_code=503, detail=f"Service Unhealthy ({e})") def _allow_public_health_readiness_details() -> bool: @@ -1897,10 +1897,8 @@ async def test_model_connection( except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.debug( - f"litellm.proxy.health_endpoints.test_model_connection(): Exception occurred - {e!s}" - ) + verbose_proxy_logger.debug(f"litellm.proxy.health_endpoints.test_model_connection(): Exception occurred - {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail={"error": f"Failed to test connection: {e!s}"}, + detail={"error": f"Failed to test connection: {e}"}, ) diff --git a/litellm/proxy/hooks/azure_content_safety.py b/litellm/proxy/hooks/azure_content_safety.py index 75ce0dff59c..3c7713b2819 100644 --- a/litellm/proxy/hooks/azure_content_safety.py +++ b/litellm/proxy/hooks/azure_content_safety.py @@ -123,7 +123,7 @@ class _PROXY_AzureContentSafety( raise e except Exception as e: verbose_proxy_logger.error( - f"litellm.proxy.hooks.azure_content_safety.py::async_pre_call_hook(): Exception occured - {e!s}" + f"litellm.proxy.hooks.azure_content_safety.py::async_pre_call_hook(): Exception occured - {e}" ) verbose_proxy_logger.debug(traceback.format_exc()) diff --git a/litellm/proxy/hooks/batch_rate_limiter.py b/litellm/proxy/hooks/batch_rate_limiter.py index 651ede6f5bc..ce4ff2cb370 100644 --- a/litellm/proxy/hooks/batch_rate_limiter.py +++ b/litellm/proxy/hooks/batch_rate_limiter.py @@ -600,7 +600,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): ) raise except Exception as e: - verbose_proxy_logger.error(f"Error counting input file usage for {file_id}: {e!s}") + verbose_proxy_logger.error(f"Error counting input file usage for {file_id}: {e}") raise async def _enforce_batch_file_model_access( @@ -704,7 +704,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): detail={ "error": ( "Batch input file references a model the caller is " - f"not authorized to use: model={model_to_check}, reason={e!s}" + f"not authorized to use: model={model_to_check}, reason={e}" ) }, ) @@ -734,7 +734,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): from litellm.proxy.proxy_server import llm_router, proxy_logging_obj except ImportError as e: raise ValueError( - f"Cannot import proxy_server dependencies: {e!s}. Managed files require proxy_server to be initialized." + f"Cannot import proxy_server dependencies: {e}. Managed files require proxy_server to be initialized." ) # Get the managed files hook @@ -846,6 +846,6 @@ class _PROXY_BatchRateLimiter(CustomLogger): # Re-raise HTTP exceptions (rate limit exceeded) raise except Exception as e: - verbose_proxy_logger.error(f"Error in batch rate limiting: {e!s}", exc_info=True) + verbose_proxy_logger.error(f"Error in batch rate limiting: {e}", exc_info=True) # Don't block the request if rate limiting fails return data diff --git a/litellm/proxy/hooks/batch_redis_get.py b/litellm/proxy/hooks/batch_redis_get.py index effafdbcf35..377cd8d3d45 100644 --- a/litellm/proxy/hooks/batch_redis_get.py +++ b/litellm/proxy/hooks/batch_redis_get.py @@ -84,7 +84,7 @@ class _PROXY_BatchRedisRequests(CustomLogger): raise e except Exception as e: verbose_proxy_logger.error( - f"litellm.proxy.hooks.batch_redis_get.py::async_pre_call_hook(): Exception occured - {e!s}" + f"litellm.proxy.hooks.batch_redis_get.py::async_pre_call_hook(): Exception occured - {e}" ) verbose_proxy_logger.debug(traceback.format_exc()) diff --git a/litellm/proxy/hooks/cache_control_check.py b/litellm/proxy/hooks/cache_control_check.py index a5c26e0dad8..f2a0f06b95b 100644 --- a/litellm/proxy/hooks/cache_control_check.py +++ b/litellm/proxy/hooks/cache_control_check.py @@ -52,5 +52,5 @@ class _PROXY_CacheControlCheck(CustomLogger): raise e except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.cache_control_check.py::async_pre_call_hook(): Exception occured - {e!s}" + f"litellm.proxy.hooks.cache_control_check.py::async_pre_call_hook(): Exception occured - {e}" ) diff --git a/litellm/proxy/hooks/dynamic_rate_limiter.py b/litellm/proxy/hooks/dynamic_rate_limiter.py index d08c3488348..5d890b6787c 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter.py @@ -68,7 +68,7 @@ class DynamicRateLimiterCache: await self.cache.async_set_cache_sadd(key=key_name, value=value, ttl=self.ttl) except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.hooks.dynamic_rate_limiter.py::async_set_cache_sadd(): Exception occured - {e!s}" + f"litellm.proxy.hooks.dynamic_rate_limiter.py::async_set_cache_sadd(): Exception occured - {e}" ) raise e @@ -172,7 +172,7 @@ class _PROXY_DynamicRateLimitHandler(CustomLogger): ) except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.hooks.dynamic_rate_limiter.py::check_available_usage: Exception occurred - {e!s}" + f"litellm.proxy.hooks.dynamic_rate_limiter.py::check_available_usage: Exception occurred - {e}" ) return None, None, None, None, None @@ -263,6 +263,6 @@ class _PROXY_DynamicRateLimitHandler(CustomLogger): ) except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.hooks.dynamic_rate_limiter.py::async_post_call_success_hook(): Exception occured - {e!s}" + f"litellm.proxy.hooks.dynamic_rate_limiter.py::async_post_call_success_hook(): Exception occured - {e}" ) return response diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index cee11ff22ae..773abed1785 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -282,7 +282,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): return max_saturation except Exception as e: - verbose_proxy_logger.error(f"Error checking saturation for {model}: {e!s}") + verbose_proxy_logger.error(f"Error checking saturation for {model}: {e}") # Fail open: assume not saturated on error return 0.0 @@ -640,7 +640,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error in dynamic rate limiter: {e!s}, allowing request") + verbose_proxy_logger.error(f"Error in dynamic rate limiter: {e}, allowing request") # Fail open on unexpected errors return None @@ -676,7 +676,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): return response except Exception as e: - verbose_proxy_logger.exception(f"Error in dynamic rate limiter v3 post-call hook: {e!s}") + verbose_proxy_logger.exception(f"Error in dynamic rate limiter v3 post-call hook: {e}") return response async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -791,4 +791,4 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): ) except Exception as e: - verbose_proxy_logger.exception(f"Error in dynamic rate limiter success event: {e!s}") + verbose_proxy_logger.exception(f"Error in dynamic rate limiter success event: {e}") diff --git a/litellm/proxy/hooks/litellm_skills/main.py b/litellm/proxy/hooks/litellm_skills/main.py index 2fc9779e5fd..983a59657ce 100644 --- a/litellm/proxy/hooks/litellm_skills/main.py +++ b/litellm/proxy/hooks/litellm_skills/main.py @@ -587,7 +587,7 @@ class SkillsInjectionHook(CustomLogger): return result or "Code executed successfully" except Exception as e: - return f"Code execution failed: {e!s}" + return f"Code execution failed: {e}" async def _execute_skill_tool( self, @@ -821,7 +821,7 @@ print('No executable skill module found') except Exception as e: verbose_proxy_logger.error(f"SkillsInjectionHook: Code execution failed: {e}") - return f"Code execution failed: {e!s}" + return f"Code execution failed: {e}" def _attach_files_to_response( self, diff --git a/litellm/proxy/hooks/max_budget_limiter.py b/litellm/proxy/hooks/max_budget_limiter.py index 0a1a09d0792..4a768b4e7de 100644 --- a/litellm/proxy/hooks/max_budget_limiter.py +++ b/litellm/proxy/hooks/max_budget_limiter.py @@ -75,5 +75,5 @@ class _PROXY_MaxBudgetLimiter(CustomLogger): raise e except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.max_budget_limiter.py::async_pre_call_hook(): Exception occured - {e!s}" + f"litellm.proxy.hooks.max_budget_limiter.py::async_pre_call_hook(): Exception occured - {e}" ) diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index b41fd960aac..04f34d0e9cf 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -776,7 +776,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): litellm_parent_otel_span=litellm_parent_otel_span, ) # save in cache for up to 1 min. except Exception as e: - verbose_proxy_logger.exception(f"Inside Parallel Request Limiter: An exception occurred - {e!s}") + verbose_proxy_logger.exception(f"Inside Parallel Request Limiter: An exception occurred - {e}") async def get_internal_user_object( self, diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 7253a684b3c..98f1e650845 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -490,7 +490,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): parallel_request_limiter=self, ) except Exception as e: - verbose_proxy_logger.debug(f"Could not load batch rate limiter: {e!s}") + verbose_proxy_logger.debug(f"Could not load batch rate limiter: {e}") return self._batch_rate_limiter def _get_current_time(self) -> datetime: @@ -808,7 +808,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) all_cache_values.extend(group_cache_values) except Exception as e: - verbose_proxy_logger.warning(f"Redis Lua script failed for hash tag {hash_tag}: {e!s}") + verbose_proxy_logger.warning(f"Redis Lua script failed for hash tag {hash_tag}: {e}") # Fallback to in-memory cache for this group group_cache_values = await self.in_memory_cache_sliding_window( keys=group_keys, @@ -1055,7 +1055,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) counts = [max(0, int(value)) for value in raw_counts] except Exception as e: # noqa: BLE001 - any Redis/Lua failure degrades to the local mirror, never a 500 - verbose_proxy_logger.warning(f"parallel_count_script failed, using local mirror: {e!s}") + verbose_proxy_logger.warning(f"parallel_count_script failed, using local mirror: {e}") counts = await self._read_local_gauge_counts(gauge_keys, parent_otel_span) else: counts = await self._read_local_gauge_counts(gauge_keys, parent_otel_span) @@ -1085,7 +1085,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ], ) except Exception as e: # noqa: BLE001 - any Redis/Lua failure degrades to in-memory enforcement, never a 500 - verbose_proxy_logger.warning(f"parallel_acquire_script failed, falling back to in-memory gauge: {e!s}") + verbose_proxy_logger.warning(f"parallel_acquire_script failed, falling back to in-memory gauge: {e}") async with self._check_and_increment_lock: return await self._acquire_parallel_slots_in_memory(gauges, slot_id, parent_otel_span) if int(raw[0]) == 1: @@ -1212,9 +1212,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) return except Exception as e: # noqa: BLE001 - any Redis/Lua failure degrades to the in-memory release, never a 500 - verbose_proxy_logger.warning( - f"parallel_release_script failed, falling back to in-memory release: {e!s}" - ) + verbose_proxy_logger.warning(f"parallel_release_script failed, falling back to in-memory release: {e}") async with self._check_and_increment_lock: for counter_key in counter_keys: @@ -2240,7 +2238,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): return False except Exception as e: - verbose_proxy_logger.debug(f"Error checking model failure status: {e!s}, defaulting to enforce limits") + verbose_proxy_logger.debug(f"Error checking model failure status: {e}, defaulting to enforce limits") # Fail safe: enforce limits if we can't check return True @@ -2746,7 +2744,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) except Exception as e: - verbose_proxy_logger.warning(f"TTL preservation failed, falling back to regular pipeline: {e!s}") + verbose_proxy_logger.warning(f"TTL preservation failed, falling back to regular pipeline: {e}") # Fallback to regular pipeline on error await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline( increment_list=pipeline_operations, @@ -3003,7 +3001,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) except Exception as e: - verbose_proxy_logger.exception(f"Error in rate limit success event: {e!s}") + verbose_proxy_logger.exception(f"Error in rate limit success event: {e}") async def async_logging_hook( self, @@ -3120,7 +3118,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if stash is not None and reserved_tokens > 0: stash.reservation_released = True except Exception as e: - verbose_proxy_logger.exception(f"Error in rate limit failure event: {e!s}") + verbose_proxy_logger.exception(f"Error in rate limit failure event: {e}") async def async_release_max_parallel_requests_on_disconnect( self, @@ -3185,7 +3183,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) except Exception as e: - verbose_proxy_logger.exception(f"Error in rate limit post-call hook: {e!s}") + verbose_proxy_logger.exception(f"Error in rate limit post-call hook: {e}") async def async_post_call_failure_hook( self, diff --git a/litellm/proxy/hooks/prompt_injection_detection.py b/litellm/proxy/hooks/prompt_injection_detection.py index 3e8518d55dc..e7192b9b063 100644 --- a/litellm/proxy/hooks/prompt_injection_detection.py +++ b/litellm/proxy/hooks/prompt_injection_detection.py @@ -197,7 +197,7 @@ class _OPTIONAL_PromptInjectionDetection(CustomLogger): raise e except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {e!s}" + f"litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {e}" ) async def async_moderation_hook( # type: ignore diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 857429fa89f..0319a680714 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -308,7 +308,7 @@ class _ProxyDBLogger(CustomLogger): f"Cost tracking failed for model={model}.\nDebug info - {cost_tracking_failure_debug_info}\nAdd custom pricing - https://docs.litellm.ai/docs/proxy/custom_pricing" ) except Exception as e: - error_msg = f"Error in tracking cost callback - {e!s}\n Traceback:{traceback.format_exc()}" + error_msg = f"Error in tracking cost callback - {e}\n Traceback:{traceback.format_exc()}" model = kwargs.get("model", "") metadata = get_litellm_metadata_from_kwargs(kwargs=kwargs) litellm_metadata = kwargs.get("litellm_params", {}).get("litellm_metadata", {}) diff --git a/litellm/proxy/hooks/user_management_event_hooks.py b/litellm/proxy/hooks/user_management_event_hooks.py index 444c39340a0..b242f763fcb 100644 --- a/litellm/proxy/hooks/user_management_event_hooks.py +++ b/litellm/proxy/hooks/user_management_event_hooks.py @@ -71,7 +71,7 @@ class UserManagementEventHooks: ) ) except Exception as e: - verbose_proxy_logger.warning(f"Unable to create audit log for user on `/user/new` - {e!s}") + verbose_proxy_logger.warning(f"Unable to create audit log for user on `/user/new` - {e}") @staticmethod async def async_send_user_invitation_email( diff --git a/litellm/proxy/image_endpoints/endpoints.py b/litellm/proxy/image_endpoints/endpoints.py index 7666ad0f065..36f702e1b4b 100644 --- a/litellm/proxy/image_endpoints/endpoints.py +++ b/litellm/proxy/image_endpoints/endpoints.py @@ -185,7 +185,7 @@ async def image_generation( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.image_generation(): Exception occured - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.image_generation(): Exception occured - {e}") verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -195,7 +195,7 @@ async def image_generation( code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) else: - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), diff --git a/litellm/proxy/management_endpoints/cache_settings_endpoints.py b/litellm/proxy/management_endpoints/cache_settings_endpoints.py index e08bc13a14d..14a42811b07 100644 --- a/litellm/proxy/management_endpoints/cache_settings_endpoints.py +++ b/litellm/proxy/management_endpoints/cache_settings_endpoints.py @@ -386,7 +386,7 @@ class CacheSettingsManager: verbose_proxy_logger.info("Cache settings initialized from database") except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.management_endpoints.cache_settings_endpoints.py::CacheSettingsManager::init_cache_settings_in_db - {e!s}" + f"litellm.proxy.management_endpoints.cache_settings_endpoints.py::CacheSettingsManager::init_cache_settings_in_db - {e}" ) @staticmethod @@ -480,8 +480,8 @@ async def get_cache_settings( redis_type_descriptions=REDIS_TYPE_DESCRIPTIONS, ) except Exception as e: - verbose_proxy_logger.error(f"Error fetching cache settings: {e!s}") - raise HTTPException(status_code=500, detail=f"Error fetching cache settings: {e!s}") + verbose_proxy_logger.error(f"Error fetching cache settings: {e}") + raise HTTPException(status_code=500, detail=f"Error fetching cache settings: {e}") @router.post( @@ -539,10 +539,10 @@ async def test_cache_connection( return CacheTestResponse(**result) except Exception as e: - verbose_proxy_logger.error(f"Error testing cache connection: {e!s}") + verbose_proxy_logger.error(f"Error testing cache connection: {e}") return CacheTestResponse( status="failed", - message=f"Cache connection test failed: {e!s}", + message=f"Cache connection test failed: {e}", error=str(e), ) @@ -652,5 +652,5 @@ async def update_cache_settings( "settings": _redact_credentials(cache_settings), } except Exception as e: - verbose_proxy_logger.error(f"Error updating cache settings: {e!s}") - raise HTTPException(status_code=500, detail=f"Error updating cache settings: {e!s}") + verbose_proxy_logger.error(f"Error updating cache settings: {e}") + raise HTTPException(status_code=500, detail=f"Error updating cache settings: {e}") diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 9bd8db16769..0dc85f98786 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -993,10 +993,10 @@ async def get_daily_activity( ) except Exception as e: - verbose_proxy_logger.exception(f"Error fetching daily activity: {e!s}") + verbose_proxy_logger.exception(f"Error fetching daily activity: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail={"error": f"Failed to fetch analytics: {e!s}"}, + detail={"error": f"Failed to fetch analytics: {e}"}, ) @@ -1082,8 +1082,8 @@ async def get_daily_activity_aggregated( ) except Exception as e: - verbose_proxy_logger.exception(f"Error fetching aggregated daily activity: {e!s}") + verbose_proxy_logger.exception(f"Error fetching aggregated daily activity: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail={"error": f"Failed to fetch analytics: {e!s}"}, + detail={"error": f"Failed to fetch analytics: {e}"}, ) diff --git a/litellm/proxy/management_endpoints/cost_tracking_settings.py b/litellm/proxy/management_endpoints/cost_tracking_settings.py index 9d985e48a60..f2295452f4d 100644 --- a/litellm/proxy/management_endpoints/cost_tracking_settings.py +++ b/litellm/proxy/management_endpoints/cost_tracking_settings.py @@ -129,7 +129,7 @@ async def get_cost_discount_config( return {"values": cost_discount_config} except Exception as e: - verbose_proxy_logger.error(f"Error fetching cost discount config: {e!s}") + verbose_proxy_logger.error(f"Error fetching cost discount config: {e}") return {"values": {}} @@ -224,10 +224,10 @@ async def update_cost_discount_config( "values": cost_discount_config, } except Exception as e: - verbose_proxy_logger.error(f"Error updating cost discount config: {e!s}") + verbose_proxy_logger.error(f"Error updating cost discount config: {e}") raise HTTPException( status_code=500, - detail={"error": f"Failed to update cost discount config: {e!s}"}, + detail={"error": f"Failed to update cost discount config: {e}"}, ) @@ -262,7 +262,7 @@ async def get_cost_margin_config( return {"values": cost_margin_config} except Exception as e: - verbose_proxy_logger.error(f"Error fetching cost margin config: {e!s}") + verbose_proxy_logger.error(f"Error fetching cost margin config: {e}") return {"values": {}} @@ -398,10 +398,10 @@ async def update_cost_margin_config( "values": cost_margin_config, } except Exception as e: - verbose_proxy_logger.error(f"Error updating cost margin config: {e!s}") + verbose_proxy_logger.error(f"Error updating cost margin config: {e}") raise HTTPException( status_code=500, - detail={"error": f"Failed to update cost margin config: {e!s}"}, + detail={"error": f"Failed to update cost margin config: {e}"}, ) @@ -484,7 +484,7 @@ async def estimate_cost( raise HTTPException( status_code=404, detail={ - "error": f"Could not calculate cost for model '{request.model}' (resolved to '{resolved_model}'): {e!s}" + "error": f"Could not calculate cost for model '{request.model}' (resolved to '{resolved_model}'): {e}" }, ) diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index 09977fdce40..ff384190d31 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -103,7 +103,7 @@ async def block_user(data: BlockUsers): return {"blocked_users": records} except Exception as e: - verbose_proxy_logger.error(f"An error occurred - {e!s}") + verbose_proxy_logger.error(f"An error occurred - {e}") raise HTTPException(status_code=500, detail={"error": str(e)}) @@ -390,7 +390,7 @@ async def new_end_user( return _to_customer_response(end_user_record) except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.management_endpoints.customer_endpoints.new_end_user(): Exception occured - {e!s}" + f"litellm.proxy.management_endpoints.customer_endpoints.new_end_user(): Exception occured - {e}" ) if "Unique constraint failed on the fields: (`user_id`)" in str(e): raise ProxyException( @@ -455,7 +455,7 @@ async def end_user_info( except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.management_endpoints.customer_endpoints.end_user_info(): Exception occured - {e!s}" + f"litellm.proxy.management_endpoints.customer_endpoints.end_user_info(): Exception occured - {e}" ) raise handle_exception_on_proxy(e) @@ -636,7 +636,7 @@ async def update_end_user( # update based on remaining passed in values except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.update_end_user(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.update_end_user(): Exception occured - {e}") raise handle_exception_on_proxy(e) @@ -711,7 +711,7 @@ async def delete_end_user( # update based on remaining passed in values except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.delete_end_user(): Exception occured - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.delete_end_user(): Exception occured - {e}") raise handle_exception_on_proxy(e) @@ -767,7 +767,7 @@ async def list_end_user( except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.management_endpoints.customer_endpoints.list_end_user(): Exception occured - {e!s}" + f"litellm.proxy.management_endpoints.customer_endpoints.list_end_user(): Exception occured - {e}" ) raise handle_exception_on_proxy(e) diff --git a/litellm/proxy/management_endpoints/fallback_management_endpoints.py b/litellm/proxy/management_endpoints/fallback_management_endpoints.py index f765cf379e4..3df5384b551 100644 --- a/litellm/proxy/management_endpoints/fallback_management_endpoints.py +++ b/litellm/proxy/management_endpoints/fallback_management_endpoints.py @@ -182,10 +182,10 @@ async def create_fallback( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error creating fallback: {e!s}", exc_info=True) + verbose_proxy_logger.error(f"Error creating fallback: {e}", exc_info=True) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail={"error": f"Failed to create fallback: {e!s}"}, + detail={"error": f"Failed to create fallback: {e}"}, ) @@ -239,10 +239,10 @@ async def get_fallback( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error getting fallback: {e!s}", exc_info=True) + verbose_proxy_logger.error(f"Error getting fallback: {e}", exc_info=True) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail={"error": f"Failed to get fallback: {e!s}"}, + detail={"error": f"Failed to get fallback: {e}"}, ) @@ -350,8 +350,8 @@ async def delete_fallback( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error deleting fallback: {e!s}", exc_info=True) + verbose_proxy_logger.error(f"Error deleting fallback: {e}", exc_info=True) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail={"error": f"Failed to delete fallback: {e!s}"}, + detail={"error": f"Failed to delete fallback: {e}"}, ) diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index d87a0b3d096..8e31b1f6e62 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -330,7 +330,7 @@ async def _add_user_to_team( except HTTPException as e: if e.status_code == 400 and ("already exists" in str(e) or "doesn't exist" in str(e)): verbose_proxy_logger.debug( - f"litellm.proxy.management_endpoints.internal_user_endpoints.new_user(): User already exists in team - {e!s}" + f"litellm.proxy.management_endpoints.internal_user_endpoints.new_user(): User already exists in team - {e}" ) else: verbose_proxy_logger.error( @@ -348,7 +348,7 @@ async def _add_user_to_team( and ProxyErrorTypes.team_member_already_in_team in e.type ): verbose_proxy_logger.debug( - f"litellm.proxy.management_endpoints.internal_user_endpoints.new_user(): User already exists in team - {e!s}" + f"litellm.proxy.management_endpoints.internal_user_endpoints.new_user(): User already exists in team - {e}" ) else: verbose_proxy_logger.error( @@ -605,7 +605,7 @@ async def new_user( return new_user_response except Exception as e: - verbose_proxy_logger.exception(f"/user/new: Exception occured - {e!s}") + verbose_proxy_logger.exception(f"/user/new: Exception occured - {e}") raise handle_exception_on_proxy(e) @@ -900,7 +900,7 @@ async def user_info( return response_data except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.user_info(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.user_info(): Exception occured - {e}") raise handle_exception_on_proxy(e) @@ -1050,7 +1050,7 @@ async def user_info_v2( object_permission=user_data.get("object_permission"), ) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.user_info_v2(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.user_info_v2(): Exception occured - {e}") raise handle_exception_on_proxy(e) @@ -1320,7 +1320,7 @@ async def _invalidate_cached_user_entitlement(user_id: str | None, object_permis try: await user_api_key_cache.async_delete_cache(key=key) except Exception as e: # noqa: BLE001 # a cache we cannot clear still expires; never fail the write - verbose_proxy_logger.warning(f"Failed to invalidate cached entitlement key {key!r}: {e!s}") + verbose_proxy_logger.warning(f"Failed to invalidate cached entitlement key {key!r}: {e}") async def _update_single_user_helper( @@ -1569,11 +1569,11 @@ async def user_update( ) return response except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.user_update(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.user_update(): Exception occured - {e}") verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e!s})"), + message=getattr(e, "detail", f"Authentication Error({e})"), type=ProxyErrorTypes.auth_error, param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), @@ -2395,7 +2395,7 @@ async def add_internal_user_to_organization( return new_membership except Exception as e: - raise Exception(f"Failed to add user to organization: {e!s}") + raise Exception(f"Failed to add user to organization: {e}") async def _resolve_org_filter_for_user_search( @@ -2593,8 +2593,8 @@ async def ui_view_users( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error searching users: {e!s}") - raise HTTPException(status_code=500, detail=f"Error searching users: {e!s}") + verbose_proxy_logger.exception(f"Error searching users: {e}") + raise HTTPException(status_code=500, detail=f"Error searching users: {e}") # Using shared metric helper implementations from common_daily_activity @@ -2716,10 +2716,10 @@ async def get_user_daily_activity( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"/spend/daily/analytics: Exception occured - {e!s}") + verbose_proxy_logger.exception(f"/spend/daily/analytics: Exception occured - {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail={"error": f"Failed to fetch analytics: {e!s}"}, + detail={"error": f"Failed to fetch analytics: {e}"}, ) @@ -2808,8 +2808,8 @@ async def get_user_daily_activity_aggregated( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"/user/daily/activity/aggregated: Exception occured - {e!s}") + verbose_proxy_logger.exception(f"/user/daily/activity/aggregated: Exception occured - {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail={"error": f"Failed to fetch analytics: {e!s}"}, + detail={"error": f"Failed to fetch analytics: {e}"}, ) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index fb9e6fd739e..d4403b3f5db 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -940,7 +940,7 @@ async def _common_key_generation_helper( data = apply_enterprise_key_management_params(data, team_table) except Exception as e: verbose_proxy_logger.debug( - f"litellm.proxy.proxy_server.generate_key_fn(): Enterprise key management params not applied - {e!s}" + f"litellm.proxy.proxy_server.generate_key_fn(): Enterprise key management params not applied - {e}" ) # TODO: @ishaan-jaff: Migrate all budget tracking to use LiteLLM_BudgetTable @@ -1732,7 +1732,7 @@ async def generate_key_fn( ) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.generate_key_fn(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.generate_key_fn(): Exception occured - {e}") raise handle_exception_on_proxy(e) @@ -1934,9 +1934,7 @@ def prepare_metadata_fields(data: BaseModel, non_default_values: dict, existing_ casted_metadata[k] = v except Exception as e: - verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.prepare_metadata_fields(): Exception occured - {e!s}" - ) + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.prepare_metadata_fields(): Exception occured - {e}") non_default_values["metadata"] = encrypt_callback_vars(casted_metadata) return non_default_values @@ -2799,10 +2797,10 @@ async def update_key_fn( return {"key": key, **response["data"]} # update based on remaining passed in values except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.update_key_fn(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.update_key_fn(): Exception occured - {e}") if isinstance(e, HTTPException): raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e!s})"), + message=getattr(e, "detail", f"Authentication Error({e})"), type=ProxyErrorTypes.auth_error, param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), @@ -3368,7 +3366,7 @@ async def delete_key_fn( return {"deleted_keys": deleted_keys} except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.delete_key_fn(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.delete_key_fn(): Exception occured - {e}") raise handle_exception_on_proxy(e) @@ -3908,7 +3906,7 @@ async def generate_key_helper_fn( # If it's not valid JSON/YAML, keep as is or set to empty dict key_data["router_settings"] = {} except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.generate_key_helper_fn(): Exception occured - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.generate_key_helper_fn(): Exception occured - {e}") verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise e @@ -4115,7 +4113,7 @@ async def delete_verification_tokens( raise Exception("DB not connected. prisma_client is None") except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.delete_verification_tokens(): Exception occured - {e!s}" + f"litellm.proxy.proxy_server.delete_verification_tokens(): Exception occured - {e}" ) verbose_proxy_logger.debug(traceback.format_exc()) raise e @@ -4387,7 +4385,7 @@ async def _rotate_master_key( }, ) except Exception as e: - verbose_proxy_logger.error(f"Failed to re-encrypt credential {cred.credential_name}: {e!s}") + verbose_proxy_logger.error(f"Failed to re-encrypt credential {cred.credential_name}: {e}") # Continue with next credential instead of failing entire rotation continue verbose_proxy_logger.debug(f"Successfully re-encrypted {len(credentials)} credentials with new master key") @@ -5451,7 +5449,7 @@ async def list_keys( verbose_proxy_logger.exception(f"Error in list_keys: {e}") if isinstance(e, HTTPException): raise ProxyException( - message=getattr(e, "detail", f"error({e!s})"), + message=getattr(e, "detail", f"error({e})"), type=ProxyErrorTypes.internal_server_error, param=getattr(e, "param", "None"), code=getattr(e, "status_code", fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR), @@ -5603,7 +5601,7 @@ async def key_aliases( verbose_proxy_logger.exception(f"Error in key_aliases: {e}") if isinstance(e, HTTPException): raise ProxyException( - message=getattr(e, "detail", f"error({e!s})"), + message=getattr(e, "detail", f"error({e})"), type=ProxyErrorTypes.internal_server_error, param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), @@ -6340,7 +6338,7 @@ async def key_health( except Exception as e: raise ProxyException( - message=f"Key health check failed: {e!s}", + message=f"Key health check failed: {e}", type=ProxyErrorTypes.internal_server_error, param=getattr(e, "param", "None"), code=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -6425,7 +6423,7 @@ async def test_key_logging( return LoggingCallbackStatus( callbacks=logging_callbacks, status="unhealthy", - details=f"Logging test failed: {e!s}", + details=f"Logging test failed: {e}", ) await asyncio.sleep(2) # wait for callbacks to run, callbacks use batching so wait for the flush event @@ -6556,5 +6554,5 @@ def validate_model_max_budget(model_max_budget: dict | None) -> None: BudgetConfig(**_info) except Exception as e: raise ValueError( - f"Invalid model_max_budget: {e!s}. Example of valid model_max_budget: https://docs.litellm.ai/docs/proxy/users" + f"Invalid model_max_budget: {e}. Example of valid model_max_budget: https://docs.litellm.ai/docs/proxy/users" ) diff --git a/litellm/proxy/management_endpoints/management_v1/budgets.py b/litellm/proxy/management_endpoints/management_v1/budgets.py index 8ecd7b1fa30..1bd47a940be 100644 --- a/litellm/proxy/management_endpoints/management_v1/budgets.py +++ b/litellm/proxy/management_endpoints/management_v1/budgets.py @@ -191,7 +191,7 @@ async def list_budgets( raise except Exception as e: # noqa: BLE001 # a driver error answers as a problem document, not the OpenAI error shape verbose_proxy_logger.exception( - f"litellm.proxy.management_endpoints.management_v1.budgets.list_budgets(): Exception occured - {e!s}" + f"litellm.proxy.management_endpoints.management_v1.budgets.list_budgets(): Exception occured - {e}" ) raise ManagementProblem( ProblemDetail( diff --git a/litellm/proxy/management_endpoints/management_v1/spend_logs.py b/litellm/proxy/management_endpoints/management_v1/spend_logs.py index 1927e94d01b..403e2760fb9 100644 --- a/litellm/proxy/management_endpoints/management_v1/spend_logs.py +++ b/litellm/proxy/management_endpoints/management_v1/spend_logs.py @@ -188,7 +188,7 @@ async def list_spend_log_end_users( except Exception as e: verbose_proxy_logger.exception( "litellm.proxy.management_endpoints.management_v1.spend_logs.list_spend_log_end_users(): " - f"Exception occured - {e!s}" + f"Exception occured - {e}" ) raise ManagementProblem( ProblemDetail( diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 2ae9da576b2..da86a7f06f2 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -399,7 +399,7 @@ if MCP_AVAILABLE: try: encrypted_payload = encrypt_value_helper(payload_json) except Exception as e: - verbose_proxy_logger.debug(f"Failed to encrypt temporary MCP server payload for Redis cache: {e!s}") + verbose_proxy_logger.debug(f"Failed to encrypt temporary MCP server payload for Redis cache: {e}") return if not isinstance(encrypted_payload, str): @@ -413,7 +413,7 @@ if MCP_AVAILABLE: ttl=max(1, ttl_seconds), ) except Exception as e: - verbose_proxy_logger.debug(f"Failed to write temporary MCP server to Redis cache: {e!s}") + verbose_proxy_logger.debug(f"Failed to write temporary MCP server to Redis cache: {e}") async def _get_temporary_mcp_server_from_redis( server_id: str, @@ -435,7 +435,7 @@ if MCP_AVAILABLE: key=f"{TEMPORARY_MCP_SERVER_REDIS_KEY_PREFIX}:{server_id}" ) except Exception as e: - verbose_proxy_logger.debug(f"Failed reading temporary MCP server from Redis cache: {e!s}") + verbose_proxy_logger.debug(f"Failed reading temporary MCP server from Redis cache: {e}") return None if not isinstance(cached_server, str): @@ -454,7 +454,7 @@ if MCP_AVAILABLE: try: loaded = json.loads(decrypted_json) except Exception as e: - verbose_proxy_logger.debug(f"Invalid decrypted temporary MCP payload in Redis cache: {e!s}") + verbose_proxy_logger.debug(f"Invalid decrypted temporary MCP payload in Redis cache: {e}") return None if not isinstance(loaded, dict): return None @@ -463,7 +463,7 @@ if MCP_AVAILABLE: try: return MCPServer.model_validate(payload_dict) except Exception as e: - verbose_proxy_logger.debug(f"Invalid temporary MCP server payload in Redis cache: {e!s}") + verbose_proxy_logger.debug(f"Invalid temporary MCP server payload in Redis cache: {e}") return None async def get_cached_temporary_mcp_server( @@ -1183,10 +1183,10 @@ if MCP_AVAILABLE: touched_by=user_api_key_dict.user_id or user_api_key_dict.team_id, ) except Exception as e: - verbose_proxy_logger.exception(f"Error registering mcp server: {e!s}") + verbose_proxy_logger.exception(f"Error registering mcp server: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail={"error": f"Error registering mcp server: {e!s}"}, + detail={"error": f"Error registering mcp server: {e}"}, ) # Do NOT add to runtime registry — pending servers are not active return _redact_mcp_credentials(new_mcp_server) @@ -1483,10 +1483,10 @@ if MCP_AVAILABLE: touched_by=user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME, ) except Exception as e: - verbose_proxy_logger.exception(f"Error creating mcp server: {e!s}") + verbose_proxy_logger.exception(f"Error creating mcp server: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail={"error": f"Error creating mcp server: {e!s}"}, + detail={"error": f"Error creating mcp server: {e}"}, ) # Registry refresh is best-effort: the row is already committed, so a @@ -1498,7 +1498,7 @@ if MCP_AVAILABLE: await global_mcp_server_manager.reload_servers_from_database() except Exception as e: verbose_proxy_logger.exception( - f"MCP server {new_mcp_server.server_id} created but in-memory registry refresh failed: {e!s}" + f"MCP server {new_mcp_server.server_id} created but in-memory registry refresh failed: {e}" ) return _redact_mcp_credentials(new_mcp_server) @@ -1559,10 +1559,10 @@ if MCP_AVAILABLE: ttl_seconds=TEMPORARY_MCP_SERVER_TTL_SECONDS, ) except Exception as e: - verbose_proxy_logger.exception(f"Error caching temporary mcp server: {e!s}") + verbose_proxy_logger.exception(f"Error caching temporary mcp server: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail={"error": f"Error caching temporary mcp server: {e!s}"}, + detail={"error": f"Error caching temporary mcp server: {e}"}, ) return _redact_mcp_credentials(temp_record) diff --git a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py index 2ac0b32ec13..b294b2674e4 100644 --- a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py @@ -439,10 +439,10 @@ async def create_model_group( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error creating access group '{data.access_group}': {e!s}") + verbose_proxy_logger.exception(f"Error creating access group '{data.access_group}': {e}") raise HTTPException( status_code=500, - detail={"error": f"Failed to create access group: {e!s}"}, + detail={"error": f"Failed to create access group: {e}"}, ) @@ -489,10 +489,10 @@ async def list_access_groups( return ListAccessGroupsResponse(access_groups=access_groups_list) except Exception as e: - verbose_proxy_logger.exception(f"Error listing access groups: {e!s}") + verbose_proxy_logger.exception(f"Error listing access groups: {e}") raise HTTPException( status_code=500, - detail={"error": f"Failed to list access groups: {e!s}"}, + detail={"error": f"Failed to list access groups: {e}"}, ) @@ -546,10 +546,10 @@ async def get_access_group_info( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error getting access group info for '{access_group}': {e!s}") + verbose_proxy_logger.exception(f"Error getting access group info for '{access_group}': {e}") raise HTTPException( status_code=500, - detail={"error": f"Failed to get access group info: {e!s}"}, + detail={"error": f"Failed to get access group info: {e}"}, ) @@ -627,7 +627,7 @@ async def update_access_group( except Exception as e: raise HTTPException( status_code=500, - detail={"error": f"Failed to check access group existence: {e!s}"}, + detail={"error": f"Failed to check access group existence: {e}"}, ) # Validation: Check if all new models exist (only if using model_names path) @@ -699,10 +699,10 @@ async def update_access_group( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error updating access group '{access_group}': {e!s}") + verbose_proxy_logger.exception(f"Error updating access group '{access_group}': {e}") raise HTTPException( status_code=500, - detail={"error": f"Failed to update access group: {e!s}"}, + detail={"error": f"Failed to update access group: {e}"}, ) @@ -759,7 +759,7 @@ async def delete_access_group( except Exception as e: raise HTTPException( status_code=500, - detail={"error": f"Failed to check access group existence: {e!s}"}, + detail={"error": f"Failed to check access group existence: {e}"}, ) try: @@ -800,8 +800,8 @@ async def delete_access_group( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error deleting access group '{access_group}': {e!s}") + verbose_proxy_logger.exception(f"Error deleting access group '{access_group}': {e}") raise HTTPException( status_code=500, - detail={"error": f"Failed to delete access group: {e!s}"}, + detail={"error": f"Failed to delete access group: {e}"}, ) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 35b64963d4d..50109b02189 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -355,13 +355,13 @@ async def patch_model( return updated_model except Exception as e: - verbose_proxy_logger.exception(f"Error in patch_model: {e!s}") + verbose_proxy_logger.exception(f"Error in patch_model: {e}") if isinstance(e, (HTTPException, ProxyException)): raise e raise ProxyException( - message=f"Error updating model: {e!s}", + message=f"Error updating model: {e}", type=ProxyErrorTypes.internal_server_error, code=status.HTTP_500_INTERNAL_SERVER_ERROR, param=None, @@ -462,13 +462,13 @@ async def _set_model_blocked_status( return updated_model except Exception as e: - verbose_proxy_logger.exception(f"Error in model {action}: {e!s}") + verbose_proxy_logger.exception(f"Error in model {action}: {e}") if isinstance(e, (HTTPException, ProxyException)): raise e raise ProxyException( - message=f"Error updating model blocked status: {e!s}", + message=f"Error updating model blocked status: {e}", type=ProxyErrorTypes.internal_server_error, code=status.HTTP_500_INTERNAL_SERVER_ERROR, param=None, @@ -1223,10 +1223,10 @@ async def delete_model( ) except Exception as e: - verbose_proxy_logger.exception(f"Failed to delete model. Due to error - {e!s}") + verbose_proxy_logger.exception(f"Failed to delete model. Due to error - {e}") if isinstance(e, HTTPException): raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e!s})"), + message=getattr(e, "detail", f"Authentication Error({e})"), type=ProxyErrorTypes.auth_error, param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), @@ -1429,10 +1429,10 @@ async def add_new_model( return model_response except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.add_new_model(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.add_new_model(): Exception occured - {e}") if isinstance(e, HTTPException): raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e!s})"), + message=getattr(e, "detail", f"Authentication Error({e})"), type=ProxyErrorTypes.auth_error, param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), @@ -1582,10 +1582,10 @@ async def update_model( return model_response except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.update_model(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.update_model(): Exception occured - {e}") if isinstance(e, HTTPException): raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e!s})"), + message=getattr(e, "detail", f"Authentication Error({e})"), type=ProxyErrorTypes.auth_error, param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), @@ -1675,13 +1675,13 @@ async def update_public_model_groups( } except Exception as e: - verbose_proxy_logger.exception(f"Error updating public model groups: {e!s}") + verbose_proxy_logger.exception(f"Error updating public model groups: {e}") if isinstance(e, HTTPException): raise e raise ProxyException( - message=f"Error updating public model groups: {e!s}", + message=f"Error updating public model groups: {e}", type=ProxyErrorTypes.internal_server_error, code=status.HTTP_500_INTERNAL_SERVER_ERROR, param=None, @@ -1743,13 +1743,13 @@ async def update_useful_links( } except Exception as e: - verbose_proxy_logger.exception(f"Error updating public model groups: {e!s}") + verbose_proxy_logger.exception(f"Error updating public model groups: {e}") if isinstance(e, HTTPException): raise e raise ProxyException( - message=f"Error updating public model groups: {e!s}", + message=f"Error updating public model groups: {e}", type=ProxyErrorTypes.internal_server_error, code=status.HTTP_500_INTERNAL_SERVER_ERROR, param=None, @@ -1970,5 +1970,5 @@ async def clear_cache() -> frozenset[str] | None: ) return still_desired_ids except Exception as e: - verbose_proxy_logger.exception(f"Failed to clear cache and reload models. Due to error - {e!s}") + verbose_proxy_logger.exception(f"Failed to clear cache and reload models. Due to error - {e}") return None diff --git a/litellm/proxy/management_endpoints/organization_endpoints.py b/litellm/proxy/management_endpoints/organization_endpoints.py index 3ec728c7f79..949c35e4182 100644 --- a/litellm/proxy/management_endpoints/organization_endpoints.py +++ b/litellm/proxy/management_endpoints/organization_endpoints.py @@ -1261,7 +1261,7 @@ async def organization_member_add( verbose_proxy_logger.exception(f"Error adding member to organization: {e}") if isinstance(e, HTTPException): raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e!s})"), + message=getattr(e, "detail", f"Authentication Error({e})"), type=ProxyErrorTypes.auth_error, param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), diff --git a/litellm/proxy/management_endpoints/router_settings_endpoints.py b/litellm/proxy/management_endpoints/router_settings_endpoints.py index 061b820093c..0adc0610c60 100644 --- a/litellm/proxy/management_endpoints/router_settings_endpoints.py +++ b/litellm/proxy/management_endpoints/router_settings_endpoints.py @@ -120,7 +120,7 @@ async def get_router_settings( routing_strategy_descriptions=ROUTING_STRATEGY_DESCRIPTIONS, ) except Exception as e: - verbose_proxy_logger.error(f"Error fetching router settings: {e!s}") + verbose_proxy_logger.error(f"Error fetching router settings: {e}") raise @@ -168,5 +168,5 @@ async def get_router_fields( routing_strategy_descriptions=ROUTING_STRATEGY_DESCRIPTIONS, ) except Exception as e: - verbose_proxy_logger.error(f"Error fetching router fields: {e!s}") + verbose_proxy_logger.error(f"Error fetching router fields: {e}") raise diff --git a/litellm/proxy/management_endpoints/tag_management_endpoints.py b/litellm/proxy/management_endpoints/tag_management_endpoints.py index fecb14b08d3..8e701fa9e20 100644 --- a/litellm/proxy/management_endpoints/tag_management_endpoints.py +++ b/litellm/proxy/management_endpoints/tag_management_endpoints.py @@ -201,7 +201,7 @@ async def _get_model_names(prisma_client: "PrismaClient", model_ids: Sequence[st models = await _table(ModelRepository(prisma_client)).find_many(where={"model_id": {"in": model_ids}}) return {model.model_id: model.model_name for model in models} except Exception as e: - verbose_proxy_logger.error(f"Error getting model names: {e!s}") + verbose_proxy_logger.error(f"Error getting model names: {e}") return {} @@ -331,7 +331,7 @@ async def new_tag( "tag": tag_config, } except Exception as e: - verbose_proxy_logger.exception(f"Error creating tag: {e!s}") + verbose_proxy_logger.exception(f"Error creating tag: {e}") raise HTTPException(status_code=500, detail=str(e)) @@ -372,7 +372,7 @@ async def _add_tag_to_deployment(deployment: "Deployment", tag: str): data={"litellm_params": json.dumps(existing_params)}, ) except Exception as e: - verbose_proxy_logger.exception(f"Error adding tag to deployment: {e!s}") + verbose_proxy_logger.exception(f"Error adding tag to deployment: {e}") raise HTTPException(status_code=500, detail=str(e)) @@ -461,7 +461,7 @@ async def update_tag( "tag": tag_config, } except Exception as e: - verbose_proxy_logger.exception(f"Error updating tag: {e!s}") + verbose_proxy_logger.exception(f"Error updating tag: {e}") raise HTTPException(status_code=500, detail=str(e)) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 152c27202b4..84b4e298bf4 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -257,7 +257,7 @@ async def add_team_callbacks( except ProxyException as e: raise e except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.add_team_callbacks(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.add_team_callbacks(): Exception occured - {e}") raise ProxyException( message="Internal Server Error, " + str(e), type=ProxyErrorTypes.internal_server_error.value, @@ -373,7 +373,7 @@ async def disable_team_logging( except ProxyException: raise except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.disable_team_logging(): Exception occurred - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.disable_team_logging(): Exception occurred - {e}") verbose_proxy_logger.debug(traceback.format_exc()) raise ProxyException( message="Internal Server Error, " + str(e), @@ -465,11 +465,11 @@ async def get_team_callbacks( except ProxyException: raise except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.get_team_callbacks(): Exception occurred - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.get_team_callbacks(): Exception occurred - {e}") verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( - message=getattr(e, "detail", f"Internal Server Error({e!s})"), + message=getattr(e, "detail", f"Internal Server Error({e})"), type=ProxyErrorTypes.internal_server_error.value, param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index def58794040..54ef697d16e 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -2411,7 +2411,7 @@ async def _process_team_members( except Exception as e: raise HTTPException( status_code=500, - detail={"error": f"Unable to add user - {data.member}, to team - {data.team_id}, for reason - {e!s}"}, + detail={"error": f"Unable to add user - {data.member}, to team - {data.team_id}, for reason - {e}"}, ) updated_users.append(updated_user) if updated_tm is not None: @@ -2433,7 +2433,7 @@ async def _process_team_members( except Exception as e: raise HTTPException( status_code=500, - detail={"error": f"Unable to add user - {m}, to team - {data.team_id}, for reason - {e!s}"}, + detail={"error": f"Unable to add user - {m}, to team - {data.team_id}, for reason - {e}"}, ) updated_users.append(updated_user) if updated_tm is not None: @@ -3936,7 +3936,7 @@ async def team_info( ) if isinstance(e, HTTPException): raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e!s})"), + message=getattr(e, "detail", f"Authentication Error({e})"), type=ProxyErrorTypes.auth_error, param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), @@ -4808,7 +4808,7 @@ async def list_team( ) except Exception as e: team_exception = f"""Invalid team object for team_id: {team.team_id}. team_object={team.model_dump()}. - Error: {e!s} + Error: {e} """ verbose_proxy_logger.exception(team_exception) continue @@ -4925,7 +4925,7 @@ async def ui_view_teams( return teams except Exception as e: - raise HTTPException(status_code=500, detail=f"Error searching teams: {e!s}") + raise HTTPException(status_code=500, detail=f"Error searching teams: {e}") def add_new_models_to_team(team_obj: LiteLLM_TeamTable, new_models: list[str]) -> list[str]: diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index d274879f82a..bc05f72ae14 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -2197,7 +2197,7 @@ async def cli_sso_callback( raise except Exception as e: verbose_proxy_logger.error(f"Error with CLI SSO callback: {e}") - raise HTTPException(status_code=500, detail=f"Failed to process CLI SSO: {e!s}") + raise HTTPException(status_code=500, detail=f"Failed to process CLI SSO: {e}") @router.get("/sso/cli/poll/{key_id}", tags=["experimental"], include_in_schema=False) @@ -2320,7 +2320,7 @@ async def cli_poll_key( raise except Exception as e: verbose_proxy_logger.error(f"Error polling for CLI JWT: {e}") - raise HTTPException(status_code=500, detail=f"Error checking session status: {e!s}") + raise HTTPException(status_code=500, detail=f"Error checking session status: {e}") async def insert_sso_user( @@ -4479,7 +4479,7 @@ async def debug_sso_callback(request: Request): # Try to convert to string or another JSON serializable format filtered_result[key] = str(value) except Exception as e: - filtered_result[key] = f"Complex value (not displayable): {e!s}" + filtered_result[key] = f"Complex value (not displayable): {e}" # Defense-in-depth: ensure no bearer tokens leak into the rendered HTML even if # a non-conforming IdP places them in its userinfo response. diff --git a/litellm/proxy/management_endpoints/user_agent_analytics_endpoints.py b/litellm/proxy/management_endpoints/user_agent_analytics_endpoints.py index 20f4e91b030..939fe7300f7 100644 --- a/litellm/proxy/management_endpoints/user_agent_analytics_endpoints.py +++ b/litellm/proxy/management_endpoints/user_agent_analytics_endpoints.py @@ -150,7 +150,7 @@ async def get_distinct_user_agent_tags( except Exception as e: raise HTTPException( status_code=500, - detail=f"Failed to fetch distinct user agent tags: {e!s}", + detail=f"Failed to fetch distinct user agent tags: {e}", ) @@ -243,7 +243,7 @@ async def get_daily_active_users( except Exception as e: raise HTTPException( status_code=500, - detail=f"Failed to fetch DAU analytics: {e!s}", + detail=f"Failed to fetch DAU analytics: {e}", ) @@ -364,7 +364,7 @@ async def get_weekly_active_users( except Exception as e: raise HTTPException( status_code=500, - detail=f"Failed to fetch WAU analytics: {e!s}", + detail=f"Failed to fetch WAU analytics: {e}", ) @@ -485,7 +485,7 @@ async def get_monthly_active_users( except Exception as e: raise HTTPException( status_code=500, - detail=f"Failed to fetch MAU analytics: {e!s}", + detail=f"Failed to fetch MAU analytics: {e}", ) @@ -585,12 +585,12 @@ async def get_tag_summary( except ValueError as e: raise HTTPException( status_code=400, - detail=f"Invalid date format. Use YYYY-MM-DD: {e!s}", + detail=f"Invalid date format. Use YYYY-MM-DD: {e}", ) except Exception as e: raise HTTPException( status_code=500, - detail=f"Failed to fetch tag summary analytics: {e!s}", + detail=f"Failed to fetch tag summary analytics: {e}", ) @@ -740,5 +740,5 @@ async def get_per_user_analytics( except Exception as e: raise HTTPException( status_code=500, - detail=f"Failed to fetch per-user analytics: {e!s}", + detail=f"Failed to fetch per-user analytics: {e}", ) diff --git a/litellm/proxy/ocr_endpoints/endpoints.py b/litellm/proxy/ocr_endpoints/endpoints.py index df8b0725257..ca25be9d92c 100644 --- a/litellm/proxy/ocr_endpoints/endpoints.py +++ b/litellm/proxy/ocr_endpoints/endpoints.py @@ -55,7 +55,7 @@ async def _parse_multipart_form(request: Request) -> dict[str, Any]: form = await request.form() except Exception as e: raise ValueError( - f"Failed to parse multipart form data: {e!s}. " + f"Failed to parse multipart form data: {e}. " "When using curl with --form/-F, do NOT set the Content-Type header " "manually — curl will set it automatically with the required boundary." ) diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 4e4718272bd..5d4c3c04818 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -549,7 +549,7 @@ async def create_file( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.create_file(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.create_file(): Exception occured - {e}") if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "message", str(e.detail)), @@ -558,7 +558,7 @@ async def create_file( code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) else: - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), @@ -690,7 +690,7 @@ async def get_file_content( ) except ValueError as e: raise ProxyException( - message=f"Storage backend error: {e!s}", + message=f"Storage backend error: {e}", type="invalid_request_error", param="file_id", code=400, @@ -845,7 +845,7 @@ async def get_file_content( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.retrieve_file_content(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.retrieve_file_content(): Exception occured - {e}") verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -855,7 +855,7 @@ async def get_file_content( code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) else: - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), @@ -1032,7 +1032,7 @@ async def get_file( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.retrieve_file(): Exception occured - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.retrieve_file(): Exception occured - {e}") verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -1042,7 +1042,7 @@ async def get_file( code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) else: - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), @@ -1238,7 +1238,7 @@ async def delete_file( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.delete_file(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.delete_file(): Exception occured - {e}") if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "message", str(e.detail)), @@ -1247,7 +1247,7 @@ async def delete_file( code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) else: - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), @@ -1427,7 +1427,7 @@ async def list_files( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.list_files(): Exception occured - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.list_files(): Exception occured - {e}") verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -1437,7 +1437,7 @@ async def list_files( code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) else: - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 1395fc9d32f..0d9b0ab9c49 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -857,14 +857,14 @@ async def handle_bedrock_count_tokens( except BedrockError as e: # Convert BedrockError to HTTPException for FastAPI - verbose_proxy_logger.error(f"BedrockError in handle_bedrock_count_tokens: {e!s}") + verbose_proxy_logger.error(f"BedrockError in handle_bedrock_count_tokens: {e}") raise HTTPException(status_code=e.status_code, detail={"error": e.message}) except HTTPException: # Re-raise HTTP exceptions as-is raise except Exception as e: - verbose_proxy_logger.error(f"Error in handle_bedrock_count_tokens: {e!s}") - raise HTTPException(status_code=500, detail={"error": f"CountTokens processing error: {e!s}"}) + verbose_proxy_logger.error(f"Error in handle_bedrock_count_tokens: {e}") + raise HTTPException(status_code=500, detail={"error": f"CountTokens processing error: {e}"}) async def bedrock_llm_proxy_route( diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index ddf86e9cd80..5d045ff2852 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -935,7 +935,7 @@ class AnthropicPassthroughLoggingHandler: index=0, message={ "role": "assistant", - "content": f"Error creating batch job: {e!s}", + "content": f"Error creating batch job: {e}", "tool_calls": None, "function_call": None, "provider_specific_fields": { diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py index 93bcac704e5..397f1d94a34 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py @@ -203,7 +203,7 @@ class AssemblyAIPassthroughLoggingHandler: return response.json() except Exception as e: - verbose_proxy_logger.exception(f"[Non blocking logging error] Error getting AssemblyAI transcript: {e!s}") + verbose_proxy_logger.exception(f"[Non blocking logging error] Error getting AssemblyAI transcript: {e}") return None def _poll_assembly_for_transcript_response( @@ -275,7 +275,7 @@ class AssemblyAIPassthroughLoggingHandler: return None except Exception as e: - verbose_proxy_logger.exception(f"[Non blocking logging error] Error getting AssemblyAI model info: {e!s}") + verbose_proxy_logger.exception(f"[Non blocking logging error] Error getting AssemblyAI model info: {e}") return None @staticmethod diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index e878f2a544d..63414a1c19e 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -183,7 +183,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): return cost except Exception as e: - verbose_proxy_logger.warning(f"Error calculating image generation cost: {e!s}") + verbose_proxy_logger.warning(f"Error calculating image generation cost: {e}") return 0.0 @staticmethod @@ -217,7 +217,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): return cost except Exception as e: - verbose_proxy_logger.warning(f"Error calculating image editing cost: {e!s}") + verbose_proxy_logger.warning(f"Error calculating image editing cost: {e}") return 0.0 @staticmethod @@ -445,7 +445,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): } except Exception as e: - verbose_proxy_logger.error(f"Error in OpenAI passthrough cost tracking: {e!s}") + verbose_proxy_logger.error(f"Error in OpenAI passthrough cost tracking: {e}") # Fall back to base handler without cost tracking base_handler = OpenAIPassthroughLoggingHandler() return base_handler.passthrough_chat_handler( @@ -514,7 +514,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): return complete_streaming_response except Exception as e: - verbose_proxy_logger.error(f"Error building complete streaming response: {e!s}") + verbose_proxy_logger.error(f"Error building complete streaming response: {e}") return None @staticmethod @@ -608,7 +608,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): } except Exception as e: - verbose_proxy_logger.error(f"Error in OpenAI streaming passthrough cost tracking: {e!s}") + verbose_proxy_logger.error(f"Error in OpenAI streaming passthrough cost tracking: {e}") return { "result": None, "kwargs": {}, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index a2f17eb8911..233127c3fef 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -759,7 +759,7 @@ class VertexPassthroughLoggingHandler: index=0, message={ "role": "assistant", - "content": f"Error creating batch prediction job: {e!s}", + "content": f"Error creating batch prediction job: {e}", "tool_calls": None, "function_call": None, "provider_specific_fields": { diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index b957618d776..b8aba215d10 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -294,8 +294,8 @@ async def chat_completion_pass_through_endpoint( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.completion(): Exception occured - {e!s}") - error_msg = f"{e!s}" + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.completion(): Exception occured - {e}") + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), @@ -1502,7 +1502,7 @@ async def pass_through_request( ) else: verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.pass_through_endpoint(): Exception occured - {e!s}" + f"litellm.proxy.proxy_server.pass_through_endpoint(): Exception occured - {e}" ) ######################################################### @@ -1544,7 +1544,7 @@ async def pass_through_request( headers=custom_headers, ) else: - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index 010cf8a7561..9a4a28c7678 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -89,7 +89,7 @@ class PassThroughStreamingHandler: yield chunk except Exception as e: - verbose_proxy_logger.error(f"Error in chunk_processor: {e!s}") + verbose_proxy_logger.error(f"Error in chunk_processor: {e}") raise finally: # GeneratorExit (raised on client disconnect) is not caught by @@ -115,7 +115,7 @@ class PassThroughStreamingHandler: ) ) except Exception as e: - verbose_proxy_logger.error(f"Error scheduling chunk_processor logging: {e!s}") + verbose_proxy_logger.error(f"Error scheduling chunk_processor logging: {e}") @staticmethod async def _route_streaming_logging_to_handler( @@ -165,7 +165,7 @@ class PassThroughStreamingHandler: **kwargs, ) except Exception as e: - verbose_proxy_logger.error(f"Error in _route_streaming_logging_to_handler: {e!s}") + verbose_proxy_logger.error(f"Error in _route_streaming_logging_to_handler: {e}") @staticmethod def _build_passthrough_logging_result( diff --git a/litellm/proxy/policy_engine/attachment_registry.py b/litellm/proxy/policy_engine/attachment_registry.py index ed0d98c6e6a..797f72f7667 100644 --- a/litellm/proxy/policy_engine/attachment_registry.py +++ b/litellm/proxy/policy_engine/attachment_registry.py @@ -60,8 +60,8 @@ class AttachmentRegistry: self._attachments.append(attachment) verbose_proxy_logger.debug(f"Loaded attachment for policy: {attachment.policy}") except Exception as e: - verbose_proxy_logger.error(f"Error loading attachment: {e!s}") - raise ValueError(f"Invalid attachment: {e!s}") from e + verbose_proxy_logger.error(f"Error loading attachment: {e}") + raise ValueError(f"Invalid attachment: {e}") from e self._config_attachments = tuple(self._attachments) self._initialized = True @@ -318,7 +318,7 @@ class AttachmentRegistry: ) except Exception as e: verbose_proxy_logger.exception(f"Error adding attachment to DB: {e}") - raise Exception(f"Error adding attachment to DB: {e!s}") + raise Exception(f"Error adding attachment to DB: {e}") async def delete_attachment_from_db( self, @@ -354,7 +354,7 @@ class AttachmentRegistry: return {"message": f"Attachment {attachment_id} deleted successfully"} except Exception as e: verbose_proxy_logger.exception(f"Error deleting attachment from DB: {e}") - raise Exception(f"Error deleting attachment from DB: {e!s}") + raise Exception(f"Error deleting attachment from DB: {e}") async def get_attachment_by_id_from_db( self, @@ -394,7 +394,7 @@ class AttachmentRegistry: ) except Exception as e: verbose_proxy_logger.exception(f"Error getting attachment from DB: {e}") - raise Exception(f"Error getting attachment from DB: {e!s}") + raise Exception(f"Error getting attachment from DB: {e}") async def get_all_attachments_from_db( self, @@ -432,7 +432,7 @@ class AttachmentRegistry: ] except Exception as e: verbose_proxy_logger.exception(f"Error getting attachments from DB: {e}") - raise Exception(f"Error getting attachments from DB: {e!s}") + raise Exception(f"Error getting attachments from DB: {e}") async def sync_attachments_from_db( self, @@ -468,7 +468,7 @@ class AttachmentRegistry: ) except Exception as e: verbose_proxy_logger.exception(f"Error syncing attachments from DB: {e}") - raise Exception(f"Error syncing attachments from DB: {e!s}") + raise Exception(f"Error syncing attachments from DB: {e}") # Global singleton instance diff --git a/litellm/proxy/policy_engine/init_policies.py b/litellm/proxy/policy_engine/init_policies.py index 9fb700770d2..1facec0898f 100644 --- a/litellm/proxy/policy_engine/init_policies.py +++ b/litellm/proxy/policy_engine/init_policies.py @@ -167,7 +167,7 @@ async def init_policies( policy_registry.load_policies(policies_config) verbose_proxy_logger.info(f"Successfully loaded {len(policies_config)} policies") except Exception as e: - verbose_proxy_logger.error(f"Failed to load policies: {e!s}") + verbose_proxy_logger.error(f"Failed to load policies: {e}") raise # Load attachments if provided @@ -176,7 +176,7 @@ async def init_policies( attachment_registry.load_attachments(policy_attachments_config) verbose_proxy_logger.info(f"Successfully loaded {len(policy_attachments_config)} policy attachments") except Exception as e: - verbose_proxy_logger.error(f"Failed to load policy attachments: {e!s}") + verbose_proxy_logger.error(f"Failed to load policy attachments: {e}") raise return validation_result diff --git a/litellm/proxy/policy_engine/policy_registry.py b/litellm/proxy/policy_engine/policy_registry.py index 32dfc44b8ba..07a4c2abac6 100644 --- a/litellm/proxy/policy_engine/policy_registry.py +++ b/litellm/proxy/policy_engine/policy_registry.py @@ -187,8 +187,8 @@ class PolicyRegistry: self._policies[policy_name] = policy verbose_proxy_logger.debug(f"Loaded policy: {policy_name}") except Exception as e: - verbose_proxy_logger.error(f"Error loading policy '{policy_name}': {e!s}") - raise ValueError(f"Invalid policy '{policy_name}': {e!s}") from e + verbose_proxy_logger.error(f"Error loading policy '{policy_name}': {e}") + raise ValueError(f"Invalid policy '{policy_name}': {e}") from e self._config_policies = dict(self._policies) self._sources = {policy_name: "config" for policy_name in self._policies} @@ -433,7 +433,7 @@ class PolicyRegistry: return _row_to_policy_db_response(created_policy) except Exception as e: verbose_proxy_logger.exception(f"Error adding policy to DB: {e}") - raise Exception(f"Error adding policy to DB: {e!s}") + raise Exception(f"Error adding policy to DB: {e}") async def update_policy_in_db( self, @@ -497,7 +497,7 @@ class PolicyRegistry: return _row_to_policy_db_response(updated_policy) except Exception as e: verbose_proxy_logger.exception(f"Error updating policy in DB: {e}") - raise Exception(f"Error updating policy in DB: {e!s}") + raise Exception(f"Error updating policy in DB: {e}") async def delete_policy_from_db( self, @@ -547,7 +547,7 @@ class PolicyRegistry: return result except Exception as e: verbose_proxy_logger.exception(f"Error deleting policy from DB: {e}") - raise Exception(f"Error deleting policy from DB: {e!s}") + raise Exception(f"Error deleting policy from DB: {e}") async def get_policy_by_id_from_db( self, @@ -573,7 +573,7 @@ class PolicyRegistry: return _row_to_policy_db_response(policy) except Exception as e: verbose_proxy_logger.exception(f"Error getting policy from DB: {e}") - raise Exception(f"Error getting policy from DB: {e!s}") + raise Exception(f"Error getting policy from DB: {e}") def get_policy_by_id_for_request(self, policy_id: str) -> tuple[str, Policy] | None: """ @@ -620,7 +620,7 @@ class PolicyRegistry: return [_row_to_policy_db_response(p) for p in policies] except Exception as e: verbose_proxy_logger.exception(f"Error getting policies from DB: {e}") - raise Exception(f"Error getting policies from DB: {e!s}") + raise Exception(f"Error getting policies from DB: {e}") async def sync_policies_from_db( self, @@ -689,7 +689,7 @@ class PolicyRegistry: ) except Exception as e: verbose_proxy_logger.exception(f"Error syncing policies from DB: {e}") - raise Exception(f"Error syncing policies from DB: {e!s}") + raise Exception(f"Error syncing policies from DB: {e}") async def resolve_guardrails_from_db( self, @@ -742,7 +742,7 @@ class PolicyRegistry: return sorted(resolved_policy.guardrails) except Exception as e: verbose_proxy_logger.exception(f"Error resolving guardrails from DB: {e}") - raise Exception(f"Error resolving guardrails from DB: {e!s}") + raise Exception(f"Error resolving guardrails from DB: {e}") async def get_versions_by_policy_name( self, @@ -772,7 +772,7 @@ class PolicyRegistry: ) except Exception as e: verbose_proxy_logger.exception(f"Error getting versions: {e}") - raise Exception(f"Error getting versions: {e!s}") + raise Exception(f"Error getting versions: {e}") async def create_new_version( self, @@ -858,7 +858,7 @@ class PolicyRegistry: return _row_to_policy_db_response(created) except Exception as e: verbose_proxy_logger.exception(f"Error creating new version: {e}") - raise Exception(f"Error creating new version: {e!s}") + raise Exception(f"Error creating new version: {e}") async def update_version_status( self, @@ -963,7 +963,7 @@ class PolicyRegistry: return _row_to_policy_db_response(updated) except Exception as e: verbose_proxy_logger.exception(f"Error updating version status: {e}") - raise Exception(f"Error updating version status: {e!s}") + raise Exception(f"Error updating version status: {e}") async def compare_versions( self, @@ -1016,7 +1016,7 @@ class PolicyRegistry: ) except Exception as e: verbose_proxy_logger.exception(f"Error comparing versions: {e}") - raise Exception(f"Error comparing versions: {e!s}") + raise Exception(f"Error comparing versions: {e}") async def delete_all_versions( self, @@ -1047,7 +1047,7 @@ class PolicyRegistry: return {"message": message} except Exception as e: verbose_proxy_logger.exception(f"Error deleting all versions: {e}") - raise Exception(f"Error deleting all versions: {e!s}") + raise Exception(f"Error deleting all versions: {e}") # Global singleton instance diff --git a/litellm/proxy/policy_engine/policy_validator.py b/litellm/proxy/policy_engine/policy_validator.py index 824f009c474..67f7b37472c 100644 --- a/litellm/proxy/policy_engine/policy_validator.py +++ b/litellm/proxy/policy_engine/policy_validator.py @@ -78,7 +78,7 @@ class PolicyValidator: guardrails = IN_MEMORY_GUARDRAIL_HANDLER.list_in_memory_guardrails() return {g.get("guardrail_name", "") for g in guardrails if g.get("guardrail_name")} except Exception as e: - verbose_proxy_logger.warning(f"Could not get guardrails from registry: {e!s}") + verbose_proxy_logger.warning(f"Could not get guardrails from registry: {e}") return set() async def check_team_alias_exists(self, team_alias: str) -> bool: @@ -100,7 +100,7 @@ class PolicyValidator: ) return team is not None except Exception as e: - verbose_proxy_logger.warning(f"Could not check team alias '{team_alias}': {e!s}") + verbose_proxy_logger.warning(f"Could not check team alias '{team_alias}': {e}") return True # Assume valid on error async def check_key_alias_exists(self, key_alias: str) -> bool: @@ -122,7 +122,7 @@ class PolicyValidator: ) return key is not None except Exception as e: - verbose_proxy_logger.warning(f"Could not check key alias '{key_alias}': {e!s}") + verbose_proxy_logger.warning(f"Could not check key alias '{key_alias}': {e}") return True # Assume valid on error def check_model_exists(self, model: str) -> bool: @@ -151,7 +151,7 @@ class PolicyValidator: return False except Exception as e: - verbose_proxy_logger.warning(f"Could not check model '{model}': {e!s}") + verbose_proxy_logger.warning(f"Could not check model '{model}': {e}") return True # Assume valid on error @staticmethod @@ -436,7 +436,7 @@ class PolicyValidator: PolicyValidationError( policy_name=policy_name, error_type=PolicyValidationErrorType.INVALID_SYNTAX, - message=f"Failed to parse policy: {e!s}", + message=f"Failed to parse policy: {e}", ) ) diff --git a/litellm/proxy/prompts/prompt_endpoints.py b/litellm/proxy/prompts/prompt_endpoints.py index c0c8ef2de54..89087a3fdd5 100644 --- a/litellm/proxy/prompts/prompt_endpoints.py +++ b/litellm/proxy/prompts/prompt_endpoints.py @@ -1304,7 +1304,7 @@ async def convert_prompt_file_to_json( } except Exception as e: - raise HTTPException(status_code=500, detail=f"Error converting prompt file: {e!s}") + raise HTTPException(status_code=500, detail=f"Error converting prompt file: {e}") finally: # Clean up temp file diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 901ca39326b..ac45898ce0b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3848,7 +3848,7 @@ class ProxyConfig: with open(file_path, "r") as file: return yaml.safe_load(file) or {} except Exception as e: - raise Exception(f"Error loading yaml file {file_path}: {e!s}") + raise Exception(f"Error loading yaml file {file_path}: {e}") async def _get_config_from_file(self, config_file_path: str | None = None) -> dict: """ @@ -4286,7 +4286,7 @@ class ProxyConfig: search_tool_typed: SearchToolTypedDict = SearchToolTypedDict(**search_tool) # type: ignore search_tools_parsed.append(search_tool_typed) except Exception as e: - verbose_proxy_logger.error(f"Error parsing search tool {search_tool_name}: {e!s}") + verbose_proxy_logger.error(f"Error parsing search tool {search_tool_name}: {e}") continue return search_tools_parsed if search_tools_parsed else None @@ -5499,7 +5499,7 @@ class ProxyConfig: self._add_deployment(db_models=models_list) except Exception as e: - verbose_proxy_logger.exception(f"Error adding/deleting model to llm_router: {e!s}") + verbose_proxy_logger.exception(f"Error adding/deleting model to llm_router: {e}") if llm_router is not None: llm_model_list = llm_router.get_model_list() @@ -6143,7 +6143,7 @@ class ProxyConfig: return new_models except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy_server.py::add_deployment() - Error getting new models from DB - {e!s}" + f"litellm.proxy_server.py::add_deployment() - Error getting new models from DB - {e}" ) return None @@ -6200,7 +6200,7 @@ class ProxyConfig: await self._init_non_llm_objects_in_db(prisma_client=prisma_client) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - {e}") return still_desired_ids @@ -6375,9 +6375,7 @@ class ProxyConfig: uppercase_sso_settings = {key.upper(): value for key, value in sso_settings.sso_settings.items()} self._decrypt_and_set_db_env_variables(environment_variables=uppercase_sso_settings) except Exception as e: - verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.py::ProxyConfig:_init_sso_settings_in_db - {e!s}" - ) + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_sso_settings_in_db - {e}") async def _init_hashicorp_vault_config_override(self, prisma_client: PrismaClient): """ @@ -6534,7 +6532,7 @@ class ProxyConfig: ) except Exception as e: - verbose_proxy_logger.exception(f"Error in _check_and_reload_model_cost_map: {e!s}") + verbose_proxy_logger.exception(f"Error in _check_and_reload_model_cost_map: {e}") async def _check_and_reload_anthropic_beta_headers(self, prisma_client: PrismaClient): """ @@ -6631,7 +6629,7 @@ class ProxyConfig: ) except Exception as e: - verbose_proxy_logger.exception(f"Error in _check_and_reload_anthropic_beta_headers: {e!s}") + verbose_proxy_logger.exception(f"Error in _check_and_reload_anthropic_beta_headers: {e}") def _get_prompt_spec_for_db_prompt(self, db_prompt): """ @@ -6660,7 +6658,7 @@ class ProxyConfig: prompt_spec = self._get_prompt_spec_for_db_prompt(db_prompt=prompt) IN_MEMORY_PROMPT_REGISTRY.initialize_prompt(prompt=prompt_spec) except Exception as e: - verbose_proxy_logger.debug(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_prompts_in_db - {e!s}") + verbose_proxy_logger.debug(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_prompts_in_db - {e}") async def _init_guardrails_in_db(self, prisma_client: PrismaClient): from litellm.proxy.guardrails.guardrail_registry import ( @@ -6687,7 +6685,7 @@ class ProxyConfig: # pod. Config-loaded entries are never touched. IN_MEMORY_GUARDRAIL_HANDLER.reconcile_db_guardrails(db_guardrail_ids=db_guardrail_ids) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_guardrails_in_db - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_guardrails_in_db - {e}") async def _init_policies_in_db(self, prisma_client: PrismaClient): """ @@ -6711,7 +6709,7 @@ class ProxyConfig: verbose_proxy_logger.debug("Successfully synced policies and attachments from DB") except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_policies_in_db - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_policies_in_db - {e}") async def _init_tool_policy_in_db(self, prisma_client: PrismaClient): """ @@ -6725,9 +6723,7 @@ class ProxyConfig: await registry.sync_tool_policy_from_db(prisma_client=prisma_client) verbose_proxy_logger.debug("Successfully synced tool policy from DB") except Exception as e: - verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.py::ProxyConfig:_init_tool_policy_in_db - {e!s}" - ) + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_tool_policy_in_db - {e}") async def _init_vector_stores_in_db(self, prisma_client: PrismaClient): from litellm.vector_stores.vector_store_registry import VectorStoreRegistry @@ -6745,7 +6741,7 @@ class ProxyConfig: litellm.vector_store_registry.add_vector_store_to_registry(vector_store=vector_store) except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.py::ProxyConfig:_init_vector_stores_in_db - {e!s}" + f"litellm.proxy.proxy_server.py::ProxyConfig:_init_vector_stores_in_db - {e}" ) async def _init_vector_store_indexes_in_db(self, prisma_client: PrismaClient): @@ -6769,7 +6765,7 @@ class ProxyConfig: litellm.vector_store_index_registry.upsert_vector_store_index(vector_store_index=vector_store_index) except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.py::ProxyConfig:_init_vector_stores_in_db - {e!s}" + f"litellm.proxy.proxy_server.py::ProxyConfig:_init_vector_stores_in_db - {e}" ) async def _init_mcp_servers_in_db(self): @@ -6794,7 +6790,7 @@ class ProxyConfig: await backfill_null_oauth2_flows(prisma_client) except Exception as e: # noqa: BLE001 verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db backfill - {e!s}" + f"litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db backfill - {e}" ) try: @@ -6802,15 +6798,13 @@ class ProxyConfig: await backfill_discovery_stamped_issuers(prisma_client) except Exception as e: # noqa: BLE001 verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db issuer stamp backfill - {e!s}" + f"litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db issuer stamp backfill - {e}" ) try: await global_mcp_server_manager.reload_servers_from_database() except Exception as e: - verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db - {e!s}" - ) + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db - {e}") async def init_mcp_servers_from_db(self) -> None: if self._should_load_db_object(object_type="mcp"): @@ -6838,7 +6832,7 @@ class ProxyConfig: await global_mcp_server_manager.reload_servers_from_database() except Exception as e: # noqa: BLE001 # scheduled job: a reload failure must not kill the recurring retry verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.py::ProxyConfig:reload_mcp_servers_from_db - {e!s}" + f"litellm.proxy.proxy_server.py::ProxyConfig:reload_mcp_servers_from_db - {e}" ) async def _init_agents_in_db(self, prisma_client: PrismaClient): @@ -6850,7 +6844,7 @@ class ProxyConfig: db_agents = await AGENT_REGISTRY.get_all_agents_from_db(prisma_client=prisma_client) AGENT_REGISTRY.load_agents_from_db_and_config(db_agents=db_agents) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_agents_in_db - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_agents_in_db - {e}") async def _init_search_tools_in_db(self, prisma_client: PrismaClient): """ @@ -6890,9 +6884,7 @@ class ProxyConfig: ) except Exception as e: - verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.py::ProxyConfig:_init_search_tools_in_db - {e!s}" - ) + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_search_tools_in_db - {e}") @staticmethod def _merge_config_and_db_search_tools( @@ -6958,7 +6950,7 @@ class ProxyConfig: CredentialAccessor.upsert_credentials(credentials) # upsert credentials that are in the all-up list except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy_server.py::get_credentials() - Error getting credentials from DB - {e!s}" + f"litellm.proxy_server.py::get_credentials() - Error getting credentials from DB - {e}" ) return [] @@ -7138,14 +7130,14 @@ async def async_assistants_data_generator(response, user_api_key_dict: UserAPIKe try: yield f"data: {c}\n\n" except Exception as e: - yield f"data: {e!s}\n\n" + yield f"data: {e}\n\n" # Streaming is done, yield the [DONE] chunk done_message = "[DONE]" yield f"data: {done_message}\n\n" except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.async_assistants_data_generator(): Exception occured - {e!s}" + f"litellm.proxy.proxy_server.async_assistants_data_generator(): Exception occured - {e}" ) await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, @@ -7586,7 +7578,7 @@ async def async_data_generator( try: yield _format_streaming_sse_chunk(chunk=chunk) except Exception as e: - yield f"data: {e!s}\n\n" + yield f"data: {e}\n\n" if pending_fallback_event: yield _format_fallback_metadata_sse_event( @@ -7624,7 +7616,7 @@ async def async_data_generator( client_disconnected = True raise except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.async_data_generator(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.async_data_generator(): Exception occured - {e}") await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, @@ -9375,8 +9367,8 @@ async def completion( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.completion(): Exception occured - {e!s}") - error_msg = f"{e!s}" + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.completion(): Exception occured - {e}") + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), @@ -9614,7 +9606,7 @@ async def moderations( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.moderations(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.moderations(): Exception occured - {e}") if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "message", str(e)), @@ -9623,7 +9615,7 @@ async def moderations( code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) else: - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), @@ -9760,7 +9752,7 @@ async def audio_speech( original_exception=e, request_data=data, ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.audio_speech(): Exception occured - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.audio_speech(): Exception occured - {e}") verbose_proxy_logger.debug(traceback.format_exc()) raise e @@ -9902,7 +9894,7 @@ async def audio_transcriptions( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.audio_transcription(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.audio_transcription(): Exception occured - {e}") if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "message", str(e.detail)), @@ -9911,7 +9903,7 @@ async def audio_transcriptions( code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) else: - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), @@ -10188,7 +10180,7 @@ async def get_assistants( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.get_assistants(): Exception occured - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.get_assistants(): Exception occured - {e}") verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10198,7 +10190,7 @@ async def get_assistants( code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) else: - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), @@ -10279,7 +10271,7 @@ async def create_assistant( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.create_assistant(): Exception occured - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.create_assistant(): Exception occured - {e}") verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10289,7 +10281,7 @@ async def create_assistant( code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) else: - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), @@ -10368,7 +10360,7 @@ async def delete_assistant( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.delete_assistant(): Exception occured - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.delete_assistant(): Exception occured - {e}") verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10378,7 +10370,7 @@ async def delete_assistant( code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) else: - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), @@ -10457,7 +10449,7 @@ async def create_threads( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.create_threads(): Exception occured - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.create_threads(): Exception occured - {e}") verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10467,7 +10459,7 @@ async def create_threads( code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) else: - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), @@ -10544,7 +10536,7 @@ async def get_thread( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.get_thread(): Exception occured - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.get_thread(): Exception occured - {e}") verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10554,7 +10546,7 @@ async def get_thread( code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) else: - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), @@ -10635,7 +10627,7 @@ async def add_messages( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.add_messages(): Exception occured - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.add_messages(): Exception occured - {e}") verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10645,7 +10637,7 @@ async def add_messages( code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) else: - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), @@ -10722,7 +10714,7 @@ async def get_messages( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.get_messages(): Exception occured - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.get_messages(): Exception occured - {e}") verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10732,7 +10724,7 @@ async def get_messages( code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) else: - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), @@ -10823,7 +10815,7 @@ async def run_thread( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.run_thread(): Exception occured - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.run_thread(): Exception occured - {e}") verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10833,7 +10825,7 @@ async def run_thread( code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) else: - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), @@ -11760,7 +11752,7 @@ async def _apply_search_filter_to_models( ) search_total_count = router_models_count + db_models_total_count except Exception as e: - verbose_proxy_logger.exception(f"Error querying database models with search: {e!s}") + verbose_proxy_logger.exception(f"Error querying database models with search: {e}") search_total_count = router_models_count else: search_total_count = router_models_count @@ -11895,7 +11887,7 @@ def _sort_models( sorted_models = sorted(all_models, key=get_sort_key, reverse=reverse) return sorted_models except Exception as e: - verbose_proxy_logger.exception(f"Error sorting models by {sort_by}: {e!s}") + verbose_proxy_logger.exception(f"Error sorting models by {sort_by}: {e}") return all_models @@ -11975,7 +11967,7 @@ async def _load_team_object_for_model_filter(team_id: str, prisma_client: Prisma return None return LiteLLM_TeamTable.model_validate(team_db_object.model_dump()) except Exception as e: - verbose_proxy_logger.exception(f"Error fetching team {team_id}: {e!s}") + verbose_proxy_logger.exception(f"Error fetching team {team_id}: {e}") return None @@ -12025,7 +12017,7 @@ async def _gather_team_accessible_model_ids( if db_model.model_id: team_accessible_model_ids.add(db_model.model_id) except Exception as e: - verbose_proxy_logger.debug(f"Error querying database models for team {team_id}: {e!s}") + verbose_proxy_logger.debug(f"Error querying database models for team {team_id}: {e}") return team_accessible_model_ids @@ -12163,7 +12155,7 @@ async def _find_model_by_id( if decrypted_models: found_model = decrypted_models[0] except Exception as e: - verbose_proxy_logger.exception(f"Error querying database for modelId {model_id}: {e!s}") + verbose_proxy_logger.exception(f"Error querying database for modelId {model_id}: {e}") # If model found, verify search filter if provided if found_model is not None: @@ -13613,7 +13605,7 @@ async def async_queue_request( ) if isinstance(e, HTTPException): raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e!s})"), + message=getattr(e, "detail", f"Authentication Error({e})"), type=ProxyErrorTypes.auth_error, param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), @@ -13779,7 +13771,7 @@ async def login_v2(request: Request): json_response.set_cookie(key="token", value=jwt_token) return json_response except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.login_v2(): Exception occurred - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.login_v2(): Exception occurred - {e}") if isinstance(e, ProxyException): raise e elif isinstance(e, HTTPException): @@ -13790,7 +13782,7 @@ async def login_v2(request: Request): code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) else: - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=error_msg, type=ProxyErrorTypes.auth_error, @@ -13856,7 +13848,7 @@ async def login_v3(request: Request): status_code=status.HTTP_200_OK, ) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.login_v3(): Exception occurred - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.login_v3(): Exception occurred - {e}") if isinstance(e, ProxyException): raise e elif isinstance(e, HTTPException): @@ -13867,7 +13859,7 @@ async def login_v3(request: Request): code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) else: - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=error_msg, type=ProxyErrorTypes.auth_error, @@ -13929,7 +13921,7 @@ async def login_v3_exchange(request: Request): except ProxyException: raise except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.login_v3_exchange(): Exception occurred - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.login_v3_exchange(): Exception occurred - {e}") raise ProxyException( message=str(e), type=ProxyErrorTypes.auth_error, @@ -14756,11 +14748,11 @@ async def update_config( return {"message": "Config updated successfully"} except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.update_config(): Exception occured - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.update_config(): Exception occured - {e}") verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e!s})"), + message=getattr(e, "detail", f"Authentication Error({e})"), type=ProxyErrorTypes.auth_error, param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), @@ -15584,7 +15576,7 @@ async def delete_callback( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.delete_callback(): Exception occurred - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.delete_callback(): Exception occurred - {e}") verbose_proxy_logger.debug(traceback.format_exc()) raise ProxyException( message="Error deleting callback: " + str(e), @@ -15708,10 +15700,10 @@ async def get_config( "available_callbacks": all_available_callbacks, } except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.get_config(): Exception occured - {e!s}") + verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.get_config(): Exception occured - {e}") if isinstance(e, HTTPException): raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e!s})"), + message=getattr(e, "detail", f"Authentication Error({e})"), type=ProxyErrorTypes.auth_error, param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), @@ -15826,8 +15818,8 @@ async def reload_model_cost_map( "timestamp": current_time.isoformat(), } except Exception as e: - verbose_proxy_logger.exception(f"Failed to reload model cost map: {e!s}") - raise HTTPException(status_code=500, detail=f"Failed to reload model cost map: {e!s}") + verbose_proxy_logger.exception(f"Failed to reload model cost map: {e}") + raise HTTPException(status_code=500, detail=f"Failed to reload model cost map: {e}") @router.post( @@ -15883,10 +15875,10 @@ async def schedule_model_cost_map_reload( "timestamp": datetime.utcnow().isoformat(), } except Exception as e: - verbose_proxy_logger.exception(f"Failed to schedule model cost map reload: {e!s}") + verbose_proxy_logger.exception(f"Failed to schedule model cost map reload: {e}") raise HTTPException( status_code=500, - detail=f"Failed to schedule model cost map reload: {e!s}", + detail=f"Failed to schedule model cost map reload: {e}", ) @@ -15928,8 +15920,8 @@ async def cancel_model_cost_map_reload( "timestamp": datetime.utcnow().isoformat(), } except Exception as e: - verbose_proxy_logger.exception(f"Failed to cancel model cost map reload: {e!s}") - raise HTTPException(status_code=500, detail=f"Failed to cancel model cost map reload: {e!s}") + verbose_proxy_logger.exception(f"Failed to cancel model cost map reload: {e}") + raise HTTPException(status_code=500, detail=f"Failed to cancel model cost map reload: {e}") @router.get( @@ -16015,10 +16007,10 @@ async def get_model_cost_map_reload_status( "next_run": next_run, } except Exception as e: - verbose_proxy_logger.exception(f"Failed to get model cost map reload status: {e!s}") + verbose_proxy_logger.exception(f"Failed to get model cost map reload status: {e}") raise HTTPException( status_code=500, - detail=f"Failed to get model cost map reload status: {e!s}", + detail=f"Failed to get model cost map reload status: {e}", ) @@ -16063,10 +16055,10 @@ async def get_model_cost_map_source( "model_count": model_count, } except Exception as e: - verbose_proxy_logger.exception(f"Failed to get model cost map source info: {e!s}") + verbose_proxy_logger.exception(f"Failed to get model cost map source info: {e}") raise HTTPException( status_code=500, - detail=f"Failed to get model cost map source info: {e!s}", + detail=f"Failed to get model cost map source info: {e}", ) @@ -16142,8 +16134,8 @@ async def reload_anthropic_beta_headers( "timestamp": current_time.isoformat(), } except Exception as e: - verbose_proxy_logger.exception(f"Failed to reload anthropic beta headers: {e!s}") - raise HTTPException(status_code=500, detail=f"Failed to reload anthropic beta headers: {e!s}") + verbose_proxy_logger.exception(f"Failed to reload anthropic beta headers: {e}") + raise HTTPException(status_code=500, detail=f"Failed to reload anthropic beta headers: {e}") @router.post( @@ -16199,10 +16191,10 @@ async def schedule_anthropic_beta_headers_reload( "timestamp": datetime.utcnow().isoformat(), } except Exception as e: - verbose_proxy_logger.exception(f"Failed to schedule anthropic beta headers reload: {e!s}") + verbose_proxy_logger.exception(f"Failed to schedule anthropic beta headers reload: {e}") raise HTTPException( status_code=500, - detail=f"Failed to schedule anthropic beta headers reload: {e!s}", + detail=f"Failed to schedule anthropic beta headers reload: {e}", ) @@ -16244,10 +16236,10 @@ async def cancel_anthropic_beta_headers_reload( "timestamp": datetime.utcnow().isoformat(), } except Exception as e: - verbose_proxy_logger.exception(f"Failed to cancel anthropic beta headers reload: {e!s}") + verbose_proxy_logger.exception(f"Failed to cancel anthropic beta headers reload: {e}") raise HTTPException( status_code=500, - detail=f"Failed to cancel anthropic beta headers reload: {e!s}", + detail=f"Failed to cancel anthropic beta headers reload: {e}", ) @@ -16336,10 +16328,10 @@ async def get_anthropic_beta_headers_reload_status( "next_run": next_run, } except Exception as e: - verbose_proxy_logger.exception(f"Failed to get anthropic beta headers reload status: {e!s}") + verbose_proxy_logger.exception(f"Failed to get anthropic beta headers reload status: {e}") raise HTTPException( status_code=500, - detail=f"Failed to get anthropic beta headers reload status: {e!s}", + detail=f"Failed to get anthropic beta headers reload status: {e}", ) diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 6b8227ee94f..5aef914d178 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -364,7 +364,7 @@ async def get_litellm_model_cost_map(): except Exception as e: raise HTTPException( status_code=500, - detail=f"Internal Server Error ({e!s})", + detail=f"Internal Server Error ({e})", ) diff --git a/litellm/proxy/rerank_endpoints/endpoints.py b/litellm/proxy/rerank_endpoints/endpoints.py index 69a5a9861d2..f1c138fa1bf 100644 --- a/litellm/proxy/rerank_endpoints/endpoints.py +++ b/litellm/proxy/rerank_endpoints/endpoints.py @@ -103,7 +103,7 @@ async def rerank( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.rerank(): Exception occured - {e!s}") + verbose_proxy_logger.error(f"litellm.proxy.proxy_server.rerank(): Exception occured - {e}") if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "message", str(e)), @@ -112,7 +112,7 @@ async def rerank( code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) else: - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index f17d546b88b..9fa634dc12e 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -250,9 +250,7 @@ async def responses_api( f"Stored background response {response.id} in managed objects table with unified_id={response.id}" ) except Exception as e: - verbose_proxy_logger.error( - f"Failed to store background response in managed objects table: {e!s}" - ) + verbose_proxy_logger.error(f"Failed to store background response in managed objects table: {e}") return response except ModifyResponseException as e: diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index 84dcc5718e7..b744396e850 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -328,7 +328,7 @@ async def background_streaming_task( ) except Exception as e: - verbose_proxy_logger.error(f"Error in background streaming task for {polling_id}: {e!s}") + verbose_proxy_logger.error(f"Error in background streaming task for {polling_id}: {e}") import traceback verbose_proxy_logger.error(traceback.format_exc()) diff --git a/litellm/proxy/search_endpoints/endpoints.py b/litellm/proxy/search_endpoints/endpoints.py index 7c3a924b3b5..0032083b09c 100644 --- a/litellm/proxy/search_endpoints/endpoints.py +++ b/litellm/proxy/search_endpoints/endpoints.py @@ -170,7 +170,7 @@ async def search( team_object=team_object, ) except Exception as e: - verbose_proxy_logger.error(f"Search tool authorization failed for {search_tool_name_value}: {e!s}") + verbose_proxy_logger.error(f"Search tool authorization failed for {search_tool_name_value}: {e}") raise if llm_router is not None and hasattr(llm_router, "search_tools"): diff --git a/litellm/proxy/search_endpoints/search_tool_registry.py b/litellm/proxy/search_endpoints/search_tool_registry.py index be4a588660c..d7e5efa6d1e 100644 --- a/litellm/proxy/search_endpoints/search_tool_registry.py +++ b/litellm/proxy/search_endpoints/search_tool_registry.py @@ -78,8 +78,8 @@ class SearchToolRegistry: return search_tool_dict except Exception as e: - verbose_proxy_logger.exception(f"Error adding search tool to DB: {e!s}") - raise Exception(f"Error adding search tool to DB: {e!s}") + verbose_proxy_logger.exception(f"Error adding search tool to DB: {e}") + raise Exception(f"Error adding search tool to DB: {e}") async def delete_search_tool_from_db(self, search_tool_id: str, prisma_client: PrismaClient): """ @@ -109,8 +109,8 @@ class SearchToolRegistry: "search_tool_name": existing_tool.search_tool_name, } except Exception as e: - verbose_proxy_logger.exception(f"Error deleting search tool from DB: {e!s}") - raise Exception(f"Error deleting search tool from DB: {e!s}") + verbose_proxy_logger.exception(f"Error deleting search tool from DB: {e}") + raise Exception(f"Error deleting search tool from DB: {e}") async def update_search_tool_in_db(self, search_tool_id: str, search_tool: SearchTool, prisma_client: PrismaClient): """ @@ -143,8 +143,8 @@ class SearchToolRegistry: # Convert to dict with ISO formatted datetimes return self._convert_prisma_to_dict(updated_search_tool) except Exception as e: - verbose_proxy_logger.exception(f"Error updating search tool in DB: {e!s}") - raise Exception(f"Error updating search tool in DB: {e!s}") + verbose_proxy_logger.exception(f"Error updating search tool in DB: {e}") + raise Exception(f"Error updating search tool in DB: {e}") @staticmethod async def get_all_search_tools_from_db( @@ -176,8 +176,8 @@ class SearchToolRegistry: return search_tools except Exception as e: - verbose_proxy_logger.exception(f"Error getting search tools from DB: {e!s}") - raise Exception(f"Error getting search tools from DB: {e!s}") + verbose_proxy_logger.exception(f"Error getting search tools from DB: {e}") + raise Exception(f"Error getting search tools from DB: {e}") async def get_search_tool_by_id_from_db( self, search_tool_id: str, prisma_client: PrismaClient @@ -204,8 +204,8 @@ class SearchToolRegistry: search_tool_dict = self._convert_prisma_to_dict(search_tool) return SearchTool(**search_tool_dict) # type: ignore except Exception as e: - verbose_proxy_logger.exception(f"Error getting search tool from DB: {e!s}") - raise Exception(f"Error getting search tool from DB: {e!s}") + verbose_proxy_logger.exception(f"Error getting search tool from DB: {e}") + raise Exception(f"Error getting search tool from DB: {e}") async def get_search_tool_by_name_from_db( self, search_tool_name: str, prisma_client: PrismaClient @@ -232,5 +232,5 @@ class SearchToolRegistry: search_tool_dict = self._convert_prisma_to_dict(search_tool) return SearchTool(**search_tool_dict) # type: ignore except Exception as e: - verbose_proxy_logger.exception(f"Error getting search tool from DB: {e!s}") - raise Exception(f"Error getting search tool from DB: {e!s}") + verbose_proxy_logger.exception(f"Error getting search tool from DB: {e}") + raise Exception(f"Error getting search tool from DB: {e}") diff --git a/litellm/proxy/spend_tracking/cloudzero_endpoints.py b/litellm/proxy/spend_tracking/cloudzero_endpoints.py index 37a53d06b0b..7b573b2fad7 100644 --- a/litellm/proxy/spend_tracking/cloudzero_endpoints.py +++ b/litellm/proxy/spend_tracking/cloudzero_endpoints.py @@ -161,10 +161,10 @@ async def get_cloudzero_settings( # Re-raise HTTPExceptions as-is raise e except Exception as e: - verbose_proxy_logger.error(f"Error retrieving CloudZero settings: {e!s}") + verbose_proxy_logger.error(f"Error retrieving CloudZero settings: {e}") raise HTTPException( status_code=500, - detail={"error": f"Failed to retrieve CloudZero settings: {e!s}"}, + detail={"error": f"Failed to retrieve CloudZero settings: {e}"}, ) @@ -238,10 +238,10 @@ async def update_cloudzero_settings( ) raise e except Exception as e: - verbose_proxy_logger.error(f"Error updating CloudZero settings: {e!s}") + verbose_proxy_logger.error(f"Error updating CloudZero settings: {e}") raise HTTPException( status_code=500, - detail={"error": f"Failed to update CloudZero settings: {e!s}"}, + detail={"error": f"Failed to update CloudZero settings: {e}"}, ) @@ -275,7 +275,7 @@ async def is_cloudzero_setup_in_db() -> bool: return cloudzero_config is not None and cloudzero_config.param_value is not None except Exception as e: - verbose_proxy_logger.error(f"Error checking CloudZero status: {e!s}") + verbose_proxy_logger.error(f"Error checking CloudZero status: {e}") return False @@ -317,7 +317,7 @@ async def is_cloudzero_setup() -> bool: return False except Exception as e: - verbose_proxy_logger.error(f"Error checking CloudZero setup: {e!s}") + verbose_proxy_logger.error(f"Error checking CloudZero setup: {e}") return False @@ -364,10 +364,10 @@ async def init_cloudzero_settings( return CloudZeroInitResponse(message="CloudZero settings initialized successfully", status="success") except Exception as e: - verbose_proxy_logger.error(f"Error initializing CloudZero settings: {e!s}") + verbose_proxy_logger.error(f"Error initializing CloudZero settings: {e}") raise HTTPException( status_code=500, - detail={"error": f"Failed to initialize CloudZero settings: {e!s}"}, + detail={"error": f"Failed to initialize CloudZero settings: {e}"}, ) @@ -422,10 +422,10 @@ async def cloudzero_dry_run_export( ) except Exception as e: - verbose_proxy_logger.error(f"Error performing CloudZero dry run export: {e!s}") + verbose_proxy_logger.error(f"Error performing CloudZero dry run export: {e}") raise HTTPException( status_code=500, - detail={"error": f"Failed to perform CloudZero dry run export: {e!s}"}, + detail={"error": f"Failed to perform CloudZero dry run export: {e}"}, ) @@ -487,10 +487,10 @@ async def cloudzero_export( ) except Exception as e: - verbose_proxy_logger.error(f"Error performing CloudZero export: {e!s}") + verbose_proxy_logger.error(f"Error performing CloudZero export: {e}") raise HTTPException( status_code=500, - detail={"error": f"Failed to perform CloudZero export: {e!s}"}, + detail={"error": f"Failed to perform CloudZero export: {e}"}, ) @@ -550,8 +550,8 @@ async def delete_cloudzero_settings( except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.error(f"Error deleting CloudZero settings: {e!s}") + verbose_proxy_logger.error(f"Error deleting CloudZero settings: {e}") raise HTTPException( status_code=500, - detail={"error": f"Failed to delete CloudZero settings: {e!s}"}, + detail={"error": f"Failed to delete CloudZero settings: {e}"}, ) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index d2c3b0d9391..0bcc2b9994b 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -440,7 +440,7 @@ async def view_spend_tags( except Exception as e: if isinstance(e, HTTPException): raise ProxyException( - message=getattr(e, "detail", f"/spend/tags Error({e!s})"), + message=getattr(e, "detail", f"/spend/tags Error({e})"), type="internal_error", param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), @@ -1492,7 +1492,7 @@ async def global_get_all_tag_names(): except Exception as e: if isinstance(e, HTTPException): raise ProxyException( - message=getattr(e, "detail", f"/spend/all_tag_names Error({e!s})"), + message=getattr(e, "detail", f"/spend/all_tag_names Error({e})"), type="internal_error", param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), @@ -1648,7 +1648,7 @@ async def _get_spend_report_for_time_range( return response, spend_per_tag except Exception as e: - verbose_proxy_logger.error(f"Exception in _get_daily_spend_reports {e!s}") + verbose_proxy_logger.error(f"Exception in _get_daily_spend_reports {e}") @router.post( @@ -1798,7 +1798,7 @@ async def calculate_spend(request: SpendCalculateRequest): param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) - error_msg = f"{e!s}" + error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), @@ -2667,7 +2667,7 @@ async def view_spend_logs( except Exception as e: if isinstance(e, HTTPException): raise ProxyException( - message=getattr(e, "detail", f"/spend/logs Error({e!s})"), + message=getattr(e, "detail", f"/spend/logs Error({e})"), type="internal_error", param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), @@ -2789,7 +2789,7 @@ async def global_spend_refresh(): } except Exception as e: - verbose_proxy_logger.exception(f"Failed to refresh materialized view - {e!s}") + verbose_proxy_logger.exception(f"Failed to refresh materialized view - {e}") return { "message": "Failed to refresh materialized view", "status": "failure", @@ -2830,7 +2830,7 @@ async def global_spend_for_internal_user( return response except Exception as e: - verbose_proxy_logger.error(f"/global/spend/logs Error: {e!s}") + verbose_proxy_logger.error(f"/global/spend/logs Error: {e}") raise e @@ -3387,7 +3387,7 @@ async def provider_budgets() -> ProviderBudgetResponse: provider_budget_response_dict[_provider] = provider_budget_response_object return ProviderBudgetResponse(providers=provider_budget_response_dict) except Exception as e: - verbose_proxy_logger.exception(f"/provider/budgets: Exception occured - {e!s}") + verbose_proxy_logger.exception(f"/provider/budgets: Exception occured - {e}") raise handle_exception_on_proxy(e) diff --git a/litellm/proxy/spend_tracking/vantage_endpoints.py b/litellm/proxy/spend_tracking/vantage_endpoints.py index 195731c3ed1..ac45594de22 100644 --- a/litellm/proxy/spend_tracking/vantage_endpoints.py +++ b/litellm/proxy/spend_tracking/vantage_endpoints.py @@ -166,10 +166,10 @@ async def get_vantage_settings( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error retrieving Vantage settings: {e!s}") + verbose_proxy_logger.error(f"Error retrieving Vantage settings: {e}") raise HTTPException( status_code=500, - detail={"error": f"Failed to retrieve Vantage settings: {e!s}"}, + detail={"error": f"Failed to retrieve Vantage settings: {e}"}, ) @@ -235,10 +235,10 @@ async def update_vantage_settings( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error updating Vantage settings: {e!s}") + verbose_proxy_logger.error(f"Error updating Vantage settings: {e}") raise HTTPException( status_code=500, - detail={"error": f"Failed to update Vantage settings: {e!s}"}, + detail={"error": f"Failed to update Vantage settings: {e}"}, ) @@ -257,7 +257,7 @@ async def is_vantage_setup_in_db() -> bool: return vantage_config is not None and vantage_config.param_value is not None except Exception as e: - verbose_proxy_logger.error(f"Error checking Vantage status: {e!s}") + verbose_proxy_logger.error(f"Error checking Vantage status: {e}") return False @@ -280,7 +280,7 @@ async def is_vantage_setup() -> bool: return True return False except Exception as e: - verbose_proxy_logger.error(f"Error checking Vantage setup: {e!s}") + verbose_proxy_logger.error(f"Error checking Vantage setup: {e}") return False @@ -324,10 +324,10 @@ async def init_vantage_settings( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error initializing Vantage settings: {e!s}") + verbose_proxy_logger.error(f"Error initializing Vantage settings: {e}") raise HTTPException( status_code=500, - detail={"error": f"Failed to initialize Vantage settings: {e!s}"}, + detail={"error": f"Failed to initialize Vantage settings: {e}"}, ) @@ -415,10 +415,10 @@ async def vantage_dry_run_export( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error performing Vantage dry run export: {e!s}") + verbose_proxy_logger.error(f"Error performing Vantage dry run export: {e}") raise HTTPException( status_code=500, - detail={"error": f"Failed to perform Vantage dry run export: {e!s}"}, + detail={"error": f"Failed to perform Vantage dry run export: {e}"}, ) @@ -488,10 +488,10 @@ async def vantage_export( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error performing Vantage export: {e!s}") + verbose_proxy_logger.error(f"Error performing Vantage export: {e}") raise HTTPException( status_code=500, - detail={"error": f"Failed to perform Vantage export: {e!s}"}, + detail={"error": f"Failed to perform Vantage export: {e}"}, ) @@ -548,8 +548,8 @@ async def delete_vantage_settings( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error deleting Vantage settings: {e!s}") + verbose_proxy_logger.error(f"Error deleting Vantage settings: {e}") raise HTTPException( status_code=500, - detail={"error": f"Failed to delete Vantage settings: {e!s}"}, + detail={"error": f"Failed to delete Vantage settings: {e}"}, ) diff --git a/litellm/proxy/types_utils/utils.py b/litellm/proxy/types_utils/utils.py index e9fb18b258e..e61fcdd859b 100644 --- a/litellm/proxy/types_utils/utils.py +++ b/litellm/proxy/types_utils/utils.py @@ -176,7 +176,7 @@ def _load_instance_from_remote_storage(remote_url: str, config_file_path: str | return instance except Exception as e: - raise ImportError(f"Failed to load custom logger from {remote_url}: {e!s}") from e + raise ImportError(f"Failed to load custom logger from {remote_url}: {e}") from e async def _download_gcs_file_wrapper(bucket_name: str, object_key: str, local_file_path: str) -> bool: @@ -190,7 +190,7 @@ async def _download_gcs_file_wrapper(bucket_name: str, object_key: str, local_fi except Exception as e: from litellm._logging import verbose_proxy_logger - verbose_proxy_logger.error(f"Error downloading from GCS: {e!s}") + verbose_proxy_logger.error(f"Error downloading from GCS: {e}") return False diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 69178cea55e..60c88c0c371 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -966,7 +966,7 @@ async def update_sso_settings( except Exception as e: raise HTTPException( status_code=500, - detail={"error": f"Error updating environment_variables: {e!s}"}, + detail={"error": f"Error updating environment_variables: {e}"}, ) return { diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 68d75384452..5f18189b6b3 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3341,7 +3341,7 @@ class PrismaClient: reason=f"prisma_get_generic_data_{table_name}_lookup_failure", ) except Exception as e: - error_msg = f"LiteLLM Prisma Client Exception get_generic_data: {e!s}" + error_msg = f"LiteLLM Prisma Client Exception get_generic_data: {e}" verbose_proxy_logger.error(error_msg) error_msg = error_msg + f"\nException Type: {type(e)}" error_traceback = error_msg + "\n" + traceback.format_exc() @@ -3956,7 +3956,7 @@ class PrismaClient: except Exception as e: import traceback - error_msg = f"LiteLLM Prisma Client Exception in insert_data: {e!s}" + error_msg = f"LiteLLM Prisma Client Exception in insert_data: {e}" print_verbose(error_msg) error_traceback = error_msg + "\n" + traceback.format_exc() end_time = time.time() @@ -4205,7 +4205,7 @@ class PrismaClient: except Exception as e: import traceback - error_msg = f"LiteLLM Prisma Client Exception - update_data: {e!s}" + error_msg = f"LiteLLM Prisma Client Exception - update_data: {e}" print_verbose(error_msg) error_traceback = error_msg + "\n" + traceback.format_exc() end_time = time.time() @@ -4271,7 +4271,7 @@ class PrismaClient: except Exception as e: import traceback - error_msg = f"LiteLLM Prisma Client Exception - delete_data: {e!s}" + error_msg = f"LiteLLM Prisma Client Exception - delete_data: {e}" print_verbose(error_msg) error_traceback = error_msg + "\n" + traceback.format_exc() end_time = time.time() @@ -4304,7 +4304,7 @@ class PrismaClient: except Exception as e: import traceback - error_msg = f"LiteLLM Prisma Client Exception connect(): {e!s}" + error_msg = f"LiteLLM Prisma Client Exception connect(): {e}" print_verbose(error_msg) error_traceback = error_msg + "\n" + traceback.format_exc() end_time = time.time() @@ -4334,7 +4334,7 @@ class PrismaClient: except Exception as e: import traceback - error_msg = f"LiteLLM Prisma Client Exception disconnect(): {e!s}" + error_msg = f"LiteLLM Prisma Client Exception disconnect(): {e}" print_verbose(error_msg) error_traceback = error_msg + "\n" + traceback.format_exc() end_time = time.time() @@ -5023,7 +5023,7 @@ class PrismaClient: except Exception as e: import traceback - error_msg = f"LiteLLM Prisma Client Exception disconnect(): {e!s}" + error_msg = f"LiteLLM Prisma Client Exception disconnect(): {e}" print_verbose(error_msg) error_traceback = error_msg + "\n" + traceback.format_exc() end_time = time.time() @@ -5833,7 +5833,7 @@ def _raise_failed_update_spend_exception(e: Exception, start_time: float, proxy_ """ import traceback - error_msg = f"[Non-Blocking]LiteLLM Prisma Client Exception - update spend logs: {e!s}" + error_msg = f"[Non-Blocking]LiteLLM Prisma Client Exception - update spend logs: {e}" error_traceback = error_msg + "\n" + traceback.format_exc() end_time = time.time() _duration = end_time - start_time @@ -6125,7 +6125,7 @@ def handle_exception_on_proxy(e: Exception) -> ProxyException: if isinstance(e, HTTPException): return ProxyException( - message=getattr(e, "detail", f"error({e!s})"), + message=getattr(e, "detail", f"error({e})"), type=ProxyErrorTypes.internal_server_error, param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index 811597f3821..6176ae03d3d 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -211,7 +211,7 @@ def _resolve_embedding_config_from_router(embedding_model: str, llm_router) -> d ) return embedding_config except Exception as e: - verbose_proxy_logger.debug(f"Error resolving embedding config from router for model {model_name}: {e!s}") + verbose_proxy_logger.debug(f"Error resolving embedding config from router for model {model_name}: {e}") continue return None @@ -299,7 +299,7 @@ async def _resolve_embedding_config_from_db(embedding_model: str, prisma_client) ) return embedding_config except Exception as e: - verbose_proxy_logger.debug(f"Error resolving embedding config for model {model_name}: {e!s}") + verbose_proxy_logger.debug(f"Error resolving embedding config for model {model_name}: {e}") continue return None @@ -542,7 +542,7 @@ async def new_vector_store( "vector_store": response_vs, } except Exception as e: - verbose_proxy_logger.exception(f"Error creating vector store: {e!s}") + verbose_proxy_logger.exception(f"Error creating vector store: {e}") raise HTTPException(status_code=500, detail=str(e)) @@ -647,7 +647,7 @@ async def list_vector_stores( return response except Exception as e: - verbose_proxy_logger.exception(f"Error listing vector stores: {e!s}") + verbose_proxy_logger.exception(f"Error listing vector stores: {e}") raise HTTPException(status_code=500, detail=str(e)) @@ -727,7 +727,7 @@ async def delete_vector_store( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error deleting vector store: {e!s}") + verbose_proxy_logger.exception(f"Error deleting vector store: {e}") raise HTTPException(status_code=500, detail=str(e)) @@ -799,7 +799,7 @@ async def get_vector_store_info( # the catch-all below would otherwise rewrite them as 500. raise except Exception as e: - verbose_proxy_logger.exception(f"Error getting vector store info: {e!s}") + verbose_proxy_logger.exception(f"Error getting vector store info: {e}") raise HTTPException(status_code=500, detail=str(e)) @@ -888,5 +888,5 @@ async def update_vector_store( # as 500 with the original status code embedded in the detail. raise except Exception as e: - verbose_proxy_logger.exception(f"Error updating vector store: {e!s}") + verbose_proxy_logger.exception(f"Error updating vector store: {e}") raise HTTPException(status_code=500, detail=str(e)) diff --git a/litellm/proxy/vertex_ai_endpoints/langfuse_endpoints.py b/litellm/proxy/vertex_ai_endpoints/langfuse_endpoints.py index 767e526804c..d46c93a2038 100644 --- a/litellm/proxy/vertex_ai_endpoints/langfuse_endpoints.py +++ b/litellm/proxy/vertex_ai_endpoints/langfuse_endpoints.py @@ -60,7 +60,7 @@ def _normalize_langfuse_base_url(base_target_url: str) -> str: except Exception as e: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail={"error": f"Invalid Langfuse host: {e!s}"}, + detail={"error": f"Invalid Langfuse host: {e}"}, ) if base_url.scheme not in ("http", "https") or not base_url.host: @@ -137,7 +137,7 @@ def _build_langfuse_proxy_target( except SSRFError as e: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail={"error": f"Invalid Langfuse host: {e!s}"}, + detail={"error": f"Invalid Langfuse host: {e}"}, ) custom_headers["Host"] = host_header return target_url, custom_headers diff --git a/litellm/rerank_api/main.py b/litellm/rerank_api/main.py index 03c13e504ac..2733fed744a 100644 --- a/litellm/rerank_api/main.py +++ b/litellm/rerank_api/main.py @@ -534,5 +534,5 @@ def rerank( # Placeholder return return response except Exception as e: - verbose_logger.error(f"Error in rerank: {e!s}") + verbose_logger.error(f"Error in rerank: {e}") raise exception_type(model=model, custom_llm_provider=custom_llm_provider, original_exception=e) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index c20b35b6bbc..0241453c15f 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -313,7 +313,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def _default_response_created_event_data(self) -> dict: # Use cached response ID if available, otherwise generate a new one if self._cached_response_id is None: - self._cached_response_id = f"resp_{uuid.uuid4()!s}" + self._cached_response_id = f"resp_{uuid.uuid4()}" response_created_event_data = { "id": self._cached_response_id, @@ -386,7 +386,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def create_output_item_added_event(self) -> OutputItemAddedEvent: if self._cached_item_id is None: - self._cached_item_id = f"msg_{uuid.uuid4()!s}" + self._cached_item_id = f"msg_{uuid.uuid4()}" self._sequence_number += 1 event = OutputItemAddedEvent( @@ -407,7 +407,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def create_content_part_added_event(self) -> ContentPartAddedEvent: if self._cached_item_id is None: - self._cached_item_id = f"msg_{uuid.uuid4()!s}" + self._cached_item_id = f"msg_{uuid.uuid4()}" self._sequence_number += 1 event = ContentPartAddedEvent( @@ -528,7 +528,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def create_output_text_done_event(self, litellm_complete_object: ModelResponse) -> OutputTextDoneEvent: if self._cached_item_id is None: - self._cached_item_id = f"msg_{uuid.uuid4()!s}" + self._cached_item_id = f"msg_{uuid.uuid4()}" return OutputTextDoneEvent( type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE, @@ -541,7 +541,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def create_output_content_part_done_event(self, litellm_complete_object: ModelResponse) -> ContentPartDoneEvent: if self._cached_item_id is None: - self._cached_item_id = f"msg_{uuid.uuid4()!s}" + self._cached_item_id = f"msg_{uuid.uuid4()}" text = getattr(litellm_complete_object.choices[0].message, "content", "") or "" # type: ignore reasoning_content = getattr(litellm_complete_object.choices[0].message, "reasoning_content", "") or "" # type: ignore @@ -577,7 +577,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def create_output_item_done_event(self, litellm_complete_object: ModelResponse) -> OutputItemDoneEvent: if self._cached_item_id is None: - self._cached_item_id = f"msg_{uuid.uuid4()!s}" + self._cached_item_id = f"msg_{uuid.uuid4()}" text = self.litellm_model_response.choices[0].message.content or "" # type: ignore annotations = getattr(self.litellm_model_response.choices[0].message, "annotations", None) # type: ignore diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 50744a7b93f..39881277a10 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -844,8 +844,8 @@ class LiteLLM_Proxy_MCP_Handler: request_data=logging_request_data, error=e, ) - verbose_logger.error(f"BlockedPiiEntityError in MCP tool call: {e!s}") - error_message = f"Tool call blocked: PII entity '{getattr(e, 'entity_type', 'unknown')}' detected by guardrail '{getattr(e, 'guardrail_name', 'unknown')}'. {e!s}" + verbose_logger.error(f"BlockedPiiEntityError in MCP tool call: {e}") + error_message = f"Tool call blocked: PII entity '{getattr(e, 'entity_type', 'unknown')}' detected by guardrail '{getattr(e, 'guardrail_name', 'unknown')}'. {e}" tool_results.append( { "tool_call_id": tool_call_id, @@ -860,9 +860,9 @@ class LiteLLM_Proxy_MCP_Handler: request_data=logging_request_data, error=e, ) - verbose_logger.error(f"GuardrailRaisedException in MCP tool call: {e!s}") + verbose_logger.error(f"GuardrailRaisedException in MCP tool call: {e}") error_message = ( - f"Tool call blocked: Guardrail '{getattr(e, 'guardrail_name', 'unknown')}' violation. {e!s}" + f"Tool call blocked: Guardrail '{getattr(e, 'guardrail_name', 'unknown')}' violation. {e}" ) tool_results.append( { @@ -878,7 +878,7 @@ class LiteLLM_Proxy_MCP_Handler: request_data=logging_request_data, error=e, ) - verbose_logger.error(f"HTTPException in MCP tool call: {e!s}") + verbose_logger.error(f"HTTPException in MCP tool call: {e}") error_message = f"Tool call failed: {str(e.detail) if hasattr(e, 'detail') else str(e)}" tool_results.append( { @@ -898,7 +898,7 @@ class LiteLLM_Proxy_MCP_Handler: tool_results.append( { "tool_call_id": tool_call_id, - "result": f"Error executing tool: {e!s}", + "result": f"Error executing tool: {e}", "name": tool_name, } ) diff --git a/litellm/router.py b/litellm/router.py index 37190bdbf38..e6613e1d302 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1720,7 +1720,7 @@ class Router: return _deployment_copy except Exception as e: - verbose_router_logger.debug(f"Error occurred while printing deployment - {e!s}") + verbose_router_logger.debug(f"Error occurred while printing deployment - {e}") raise e ### COMPLETION, EMBEDDING, IMG GENERATION FUNCTIONS @@ -1828,7 +1828,7 @@ class Router: return response except Exception as e: - verbose_router_logger.info(f"litellm.completion(model={model_name})\033[31m Exception {e!s}\033[0m") + verbose_router_logger.info(f"litellm.completion(model={model_name})\033[31m Exception {e}\033[0m") # Set per-deployment num_retries on exception for retry logic if deployment is not None: self._set_deployment_num_retries_on_exception(e, deployment) @@ -1923,7 +1923,7 @@ class Router: finally: loop.close() except Exception as e: - verbose_router_logger.error(f"Silent experiment failed for model {silent_model}: {e!s}") + verbose_router_logger.error(f"Silent experiment failed for model {silent_model}: {e}") # fmt: off @@ -2754,7 +2754,7 @@ class Router: **silent_kwargs, ) except Exception as e: - verbose_router_logger.error(f"Silent experiment failed for model {silent_model}: {e!s}") + verbose_router_logger.error(f"Silent experiment failed for model {silent_model}: {e}") async def _acompletion( self, model: str, messages: list[dict[str, str]], **kwargs @@ -2907,7 +2907,7 @@ class Router: self._set_failed_deployment_id_on_exception(e, deployment) raise e except Exception as e: - verbose_router_logger.info(f"litellm.acompletion(model={model_name})\033[31m Exception {e!s}\033[0m") + verbose_router_logger.info(f"litellm.acompletion(model={model_name})\033[31m Exception {e}\033[0m") if model_name is not None: self.fail_calls[model_name] += 1 # Set per-deployment num_retries on exception for retry logic @@ -3696,7 +3696,7 @@ class Router: verbose_router_logger.info(f"litellm.image_generation(model={model_name})\033[32m 200 OK\033[0m") return response except Exception as e: - verbose_router_logger.info(f"litellm.image_generation(model={model_name})\033[31m Exception {e!s}\033[0m") + verbose_router_logger.info(f"litellm.image_generation(model={model_name})\033[31m Exception {e}\033[0m") if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -3780,7 +3780,7 @@ class Router: verbose_router_logger.info(f"litellm.aimage_generation(model={model_name})\033[32m 200 OK\033[0m") return response except Exception as e: - verbose_router_logger.info(f"litellm.aimage_generation(model={model_name})\033[31m Exception {e!s}\033[0m") + verbose_router_logger.info(f"litellm.aimage_generation(model={model_name})\033[31m Exception {e}\033[0m") if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -3884,7 +3884,7 @@ class Router: verbose_router_logger.info(f"litellm.atranscription(model={model_name})\033[32m 200 OK\033[0m") return response except Exception as e: - verbose_router_logger.info(f"litellm.atranscription(model={model_name})\033[31m Exception {e!s}\033[0m") + verbose_router_logger.info(f"litellm.atranscription(model={model_name})\033[31m Exception {e}\033[0m") if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -3998,7 +3998,7 @@ class Router: verbose_router_logger.info(f"litellm.aspeech(model={model_name})\033[32m 200 OK\033[0m") return response except Exception as e: - verbose_router_logger.info(f"litellm.aspeech(model={model_name})\033[31m Exception {e!s}\033[0m") + verbose_router_logger.info(f"litellm.aspeech(model={model_name})\033[31m Exception {e}\033[0m") if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -4056,7 +4056,7 @@ class Router: verbose_router_logger.info(f"litellm.arerank(model={model_name})\033[32m 200 OK\033[0m") return response except Exception as e: - verbose_router_logger.info(f"litellm.arerank(model={model_name})\033[31m Exception {e!s}\033[0m") + verbose_router_logger.info(f"litellm.arerank(model={model_name})\033[31m Exception {e}\033[0m") if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -4190,7 +4190,7 @@ class Router: verbose_router_logger.info(f"litellm.atext_completion(model={model_name})\033[32m 200 OK\033[0m") return response except Exception as e: - verbose_router_logger.info(f"litellm.atext_completion(model={model})\033[31m Exception {e!s}\033[0m") + verbose_router_logger.info(f"litellm.atext_completion(model={model})\033[31m Exception {e}\033[0m") if model is not None: self.fail_calls[model] += 1 raise e @@ -4280,7 +4280,7 @@ class Router: verbose_router_logger.info(f"litellm.aadapter_completion(model={model_name})\033[32m 200 OK\033[0m") return response except Exception as e: - verbose_router_logger.info(f"litellm.aadapter_completion(model={model})\033[31m Exception {e!s}\033[0m") + verbose_router_logger.info(f"litellm.aadapter_completion(model={model})\033[31m Exception {e}\033[0m") if model is not None: self.fail_calls[model] += 1 raise e @@ -4539,9 +4539,7 @@ class Router: return response except Exception as e: - verbose_router_logger.info( - f"ageneric_api_call_with_fallbacks(model={model})\033[31m Exception {e!s}\033[0m" - ) + verbose_router_logger.info(f"ageneric_api_call_with_fallbacks(model={model})\033[31m Exception {e}\033[0m") if model is not None: self.fail_calls[model] += 1 raise e @@ -4661,7 +4659,7 @@ class Router: verbose_router_logger.info(f"{handler_name}(model={model_name})\033[32m 200 OK\033[0m") return response except Exception as e: - verbose_router_logger.info(f"{handler_name}(model={model})\033[31m Exception {e!s}\033[0m") + verbose_router_logger.info(f"{handler_name}(model={model})\033[31m Exception {e}\033[0m") if model is not None: self.fail_calls[model] += 1 raise e @@ -4726,7 +4724,7 @@ class Router: verbose_router_logger.info(f"litellm.embedding(model={model_name})\033[32m 200 OK\033[0m") return response except Exception as e: - verbose_router_logger.info(f"litellm.embedding(model={model_name})\033[31m Exception {e!s}\033[0m") + verbose_router_logger.info(f"litellm.embedding(model={model_name})\033[31m Exception {e}\033[0m") if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -4813,7 +4811,7 @@ class Router: verbose_router_logger.info(f"litellm.aembedding(model={model_name})\033[32m 200 OK\033[0m") return response except Exception as e: - verbose_router_logger.info(f"litellm.aembedding(model={model_name})\033[31m Exception {e!s}\033[0m") + verbose_router_logger.info(f"litellm.aembedding(model={model_name})\033[31m Exception {e}\033[0m") if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -4966,7 +4964,7 @@ class Router: return returned_response except Exception as e: verbose_router_logger.exception( - f"litellm.acreate_file(model={model}, {kwargs})\033[31m Exception {e!s}\033[0m" + f"litellm.acreate_file(model={model}, {kwargs})\033[31m Exception {e}\033[0m" ) if model is not None: self.fail_calls[model] += 1 @@ -5061,9 +5059,7 @@ class Router: return response except Exception as e: - verbose_router_logger.exception( - f"litellm.avector_store_create(model={model})\033[31m Exception {e!s}\033[0m" - ) + verbose_router_logger.exception(f"litellm.avector_store_create(model={model})\033[31m Exception {e}\033[0m") if model is not None: self.fail_calls[model] += 1 raise e @@ -5178,7 +5174,7 @@ class Router: return response # type: ignore except Exception as e: verbose_router_logger.exception( - f"litellm._acreate_batch(model={model}, {kwargs})\033[31m Exception {e!s}\033[0m" + f"litellm._acreate_batch(model={model}, {kwargs})\033[31m Exception {e}\033[0m" ) if model is not None: self.fail_calls[model] += 1 @@ -5400,7 +5396,7 @@ class Router: return response # type: ignore except Exception as e: verbose_router_logger.exception( - f"litellm._acancel_batch(model={model}, {kwargs})\033[31m Exception {e!s}\033[0m" + f"litellm._acancel_batch(model={model}, {kwargs})\033[31m Exception {e}\033[0m" ) if model is not None: self.fail_calls[model] += 1 @@ -6948,7 +6944,7 @@ class Router: except Exception as e: verbose_router_logger.debug( - f"litellm.router.Router::deployment_callback_on_success(): Exception occured - {e!s}" + f"litellm.router.Router::deployment_callback_on_success(): Exception occured - {e}" ) def sync_deployment_callback_on_success( @@ -9014,7 +9010,7 @@ class Router: custom_llm_provider=litellm_params.custom_llm_provider, ) except litellm.exceptions.BadRequestError as e: - verbose_router_logger.error(f"litellm.router.py::get_model_group_info() - {e!s}") + verbose_router_logger.error(f"litellm.router.py::get_model_group_info() - {e}") if model_info is None: supported_openai_params = litellm.get_supported_openai_params( @@ -10228,7 +10224,7 @@ class Router: ) except Exception as e: verbose_router_logger.error( - f"litellm.router.py::_pre_call_checks: failed to count tokens. Returning initial list of deployments. Got - {e!s}" + f"litellm.router.py::_pre_call_checks: failed to count tokens. Returning initial list of deployments. Got - {e}" ) return _returned_deployments if input_tokens > max_input_tokens: @@ -10239,7 +10235,7 @@ class Router: ) continue except Exception as e: - verbose_router_logger.exception(f"An error occurs - {e!s}") + verbose_router_logger.exception(f"An error occurs - {e}") model_id = _model_info.get("id", "") ## RPM CHECK ## @@ -11623,7 +11619,7 @@ class Router: if model_id is not None: self._update_usage(model_id, parent_otel_span) # update in-memory cache for tracking except Exception as e: - verbose_router_logger.error(f"Error in _track_deployment_metrics: {e!s}") + verbose_router_logger.error(f"Error in _track_deployment_metrics: {e}") def get_num_retries_from_retry_policy(self, exception: Exception, model_group: str | None = None): return _get_num_retries_from_retry_policy( diff --git a/litellm/router_strategy/base_routing_strategy.py b/litellm/router_strategy/base_routing_strategy.py index ff395828b2a..70e1c12665d 100644 --- a/litellm/router_strategy/base_routing_strategy.py +++ b/litellm/router_strategy/base_routing_strategy.py @@ -97,7 +97,7 @@ class BaseRoutingStrategy(ABC): default_sync_interval ) # Wait for DEFAULT_REDIS_SYNC_INTERVAL seconds before next sync except Exception as e: - verbose_router_logger.error(f"Error in periodic sync task: {e!s}") + verbose_router_logger.error(f"Error in periodic sync task: {e}") await asyncio.sleep( default_sync_interval ) # Still wait DEFAULT_REDIS_SYNC_INTERVAL seconds on error before retrying @@ -146,7 +146,7 @@ class BaseRoutingStrategy(ABC): return return_result except Exception as e: - verbose_router_logger.error(f"Error syncing in-memory cache with Redis: {e!s}") + verbose_router_logger.error(f"Error syncing in-memory cache with Redis: {e}") self.redis_increment_operation_queue = [] def add_to_in_memory_keys_to_update(self, key: str): @@ -226,4 +226,4 @@ class BaseRoutingStrategy(ABC): await self.dual_cache.in_memory_cache.async_set_cache(key=key, value=merged) except Exception as e: - verbose_router_logger.exception(f"Error syncing in-memory cache with Redis: {e!s}") + verbose_router_logger.exception(f"Error syncing in-memory cache with Redis: {e}") diff --git a/litellm/router_strategy/budget_limiter.py b/litellm/router_strategy/budget_limiter.py index 619f1fc4629..3b8a75f4e49 100644 --- a/litellm/router_strategy/budget_limiter.py +++ b/litellm/router_strategy/budget_limiter.py @@ -514,7 +514,7 @@ class RouterBudgetLimiting(CustomLogger): DEFAULT_REDIS_SYNC_INTERVAL ) # Wait for DEFAULT_REDIS_SYNC_INTERVAL seconds before next sync except Exception as e: - verbose_router_logger.error(f"Error in periodic sync task: {e!s}") + verbose_router_logger.error(f"Error in periodic sync task: {e}") await asyncio.sleep( DEFAULT_REDIS_SYNC_INTERVAL ) # Still wait DEFAULT_REDIS_SYNC_INTERVAL seconds on error before retrying @@ -545,7 +545,7 @@ class RouterBudgetLimiting(CustomLogger): self.redis_increment_operation_queue = [] except Exception as e: - verbose_router_logger.error(f"Error syncing in-memory cache with Redis: {e!s}") + verbose_router_logger.error(f"Error syncing in-memory cache with Redis: {e}") async def _sync_in_memory_spend_with_redis(self): """ @@ -600,7 +600,7 @@ class RouterBudgetLimiting(CustomLogger): verbose_router_logger.debug(f"Updated in-memory cache for {key}: {value}") except Exception as e: - verbose_router_logger.error(f"Error syncing in-memory cache with Redis: {e!s}") + verbose_router_logger.error(f"Error syncing in-memory cache with Redis: {e}") def _get_budget_config_for_deployment( self, diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index 12820ae1237..ba7d32c42ad 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -91,7 +91,7 @@ class LowestCostLoggingHandler(CustomLogger): self.logged_success += 1 except Exception as e: verbose_logger.exception( - f"litellm.router_strategy.lowest_cost.py::log_success_event(): Exception occured - {e!s}" + f"litellm.router_strategy.lowest_cost.py::log_success_event(): Exception occured - {e}" ) async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -170,7 +170,7 @@ class LowestCostLoggingHandler(CustomLogger): self.logged_success += 1 except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {e!s}" + f"litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {e}" ) async def async_get_available_deployments( diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index 2f73450b8d2..0adcdebcbf2 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -160,7 +160,7 @@ class LowestLatencyLoggingHandler(CustomLogger): self.logged_success += 1 except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {e!s}" + f"litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {e}" ) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): @@ -217,7 +217,7 @@ class LowestLatencyLoggingHandler(CustomLogger): return except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {e!s}" + f"litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {e}" ) async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -350,7 +350,7 @@ class LowestLatencyLoggingHandler(CustomLogger): self.logged_success += 1 except Exception as e: verbose_logger.exception( - f"litellm.router_strategy.lowest_latency.py::async_log_success_event(): Exception occured - {e!s}" + f"litellm.router_strategy.lowest_latency.py::async_log_success_event(): Exception occured - {e}" ) def _get_available_deployments( diff --git a/litellm/router_strategy/lowest_tpm_rpm.py b/litellm/router_strategy/lowest_tpm_rpm.py index 4a4352fe19d..f8e7e93eb54 100644 --- a/litellm/router_strategy/lowest_tpm_rpm.py +++ b/litellm/router_strategy/lowest_tpm_rpm.py @@ -73,7 +73,7 @@ class LowestTPMLoggingHandler(CustomLogger): self.logged_success += 1 except Exception as e: verbose_router_logger.error( - f"litellm.router_strategy.lowest_tpm_rpm.py::async_log_success_event(): Exception occured - {e!s}" + f"litellm.router_strategy.lowest_tpm_rpm.py::async_log_success_event(): Exception occured - {e}" ) verbose_router_logger.debug(traceback.format_exc()) @@ -135,7 +135,7 @@ class LowestTPMLoggingHandler(CustomLogger): self.logged_success += 1 except Exception as e: verbose_router_logger.exception( - f"litellm.router_strategy.lowest_tpm_rpm.py::async_log_success_event(): Exception occured - {e!s}" + f"litellm.router_strategy.lowest_tpm_rpm.py::async_log_success_event(): Exception occured - {e}" ) verbose_router_logger.debug(traceback.format_exc()) diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py index 03793c5577c..a81428fd5fa 100644 --- a/litellm/router_strategy/lowest_tpm_rpm_v2.py +++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py @@ -245,7 +245,7 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): self.logged_success += 1 except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.lowest_tpm_rpm_v2.py::log_success_event(): Exception occured - {e!s}" + f"litellm.proxy.hooks.lowest_tpm_rpm_v2.py::log_success_event(): Exception occured - {e}" ) async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -289,7 +289,7 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): self.logged_success += 1 except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.lowest_tpm_rpm_v2.py::async_log_success_event(): Exception occured - {e!s}" + f"litellm.proxy.hooks.lowest_tpm_rpm_v2.py::async_log_success_event(): Exception occured - {e}" ) def _return_potential_deployments( diff --git a/litellm/router_utils/cooldown_cache.py b/litellm/router_utils/cooldown_cache.py index ef62a5d8c6c..4e9a11a4bfd 100644 --- a/litellm/router_utils/cooldown_cache.py +++ b/litellm/router_utils/cooldown_cache.py @@ -58,7 +58,7 @@ class CooldownCache: return cooldown_key, cooldown_data except Exception as e: - verbose_logger.error(f"CooldownCache::_common_add_cooldown_logic - Exception occurred - {e!s}") + verbose_logger.error(f"CooldownCache::_common_add_cooldown_logic - Exception occurred - {e}") raise e def add_deployment_to_cooldown( @@ -92,7 +92,7 @@ class CooldownCache: ttl=_cooldown_time, ) except Exception as e: - verbose_logger.error(f"CooldownCache::add_deployment_to_cooldown - Exception occurred - {e!s}") + verbose_logger.error(f"CooldownCache::add_deployment_to_cooldown - Exception occurred - {e}") raise e @staticmethod diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 0c92a6fa2ab..3fad860fa7d 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -190,7 +190,7 @@ async def log_success_fallback_event(original_model_group: str, kwargs: dict, or original_exception=original_exception, ) except Exception as e: - verbose_router_logger.error(f"Error in log_success_fallback_event: {e!s}") + verbose_router_logger.error(f"Error in log_success_fallback_event: {e}") async def log_failure_fallback_event(original_model_group: str, kwargs: dict, original_exception: Exception): @@ -218,7 +218,7 @@ async def log_failure_fallback_event(original_model_group: str, kwargs: dict, or original_exception=original_exception, ) except Exception as e: - verbose_router_logger.error(f"Error in log_failure_fallback_event: {e!s}") + verbose_router_logger.error(f"Error in log_failure_fallback_event: {e}") def _check_non_standard_fallback_format(fallbacks: list[Any] | None) -> bool: diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index 004f7b53869..42704cea826 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -150,7 +150,7 @@ class PatternMatchRouter: matched_pattern=pattern_match, deployments=llm_deployments ) except Exception as e: - verbose_router_logger.debug(f"Error in PatternMatchRouter.route: {e!s}") + verbose_router_logger.debug(f"Error in PatternMatchRouter.route: {e}") return None # No matching pattern found diff --git a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py index d67f2a2bf47..da8b452fa8a 100644 --- a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py @@ -212,7 +212,7 @@ class ModelRateLimitingCheck(CustomLogger): self._refund_io_token_reservation_if_any() raise except Exception as e: - verbose_router_logger.debug(f"Error in ModelRateLimitingCheck.pre_call_check: {e!s}") + verbose_router_logger.debug(f"Error in ModelRateLimitingCheck.pre_call_check: {e}") # Don't fail the request if rate limit check fails return deployment @@ -300,7 +300,7 @@ class ModelRateLimitingCheck(CustomLogger): await self._async_refund_io_token_reservation_if_any(parent_otel_span=parent_otel_span) raise except Exception as e: - verbose_router_logger.debug(f"Error in ModelRateLimitingCheck.async_pre_call_check: {e!s}") + verbose_router_logger.debug(f"Error in ModelRateLimitingCheck.async_pre_call_check: {e}") # Don't fail the request if rate limit check fails return deployment @@ -360,7 +360,7 @@ class ModelRateLimitingCheck(CustomLogger): ) except Exception as e: - verbose_router_logger.debug(f"Error in ModelRateLimitingCheck.async_log_success_event: {e!s}") + verbose_router_logger.debug(f"Error in ModelRateLimitingCheck.async_log_success_event: {e}") async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): from litellm.litellm_core_utils.core_helpers import ( @@ -418,7 +418,7 @@ class ModelRateLimitingCheck(CustomLogger): ) except Exception as e: - verbose_router_logger.debug(f"Error in ModelRateLimitingCheck.log_success_event: {e!s}") + verbose_router_logger.debug(f"Error in ModelRateLimitingCheck.log_success_event: {e}") def log_failure_event(self, kwargs, response_obj, start_time, end_time): with contextlib.suppress(Exception): diff --git a/litellm/router_utils/search_api_router.py b/litellm/router_utils/search_api_router.py index 531b2b577b1..0ce0d4229c1 100644 --- a/litellm/router_utils/search_api_router.py +++ b/litellm/router_utils/search_api_router.py @@ -77,7 +77,7 @@ class SearchAPIRouter: verbose_router_logger.info(f"Successfully updated router with {len(router_search_tools)} search tool(s)") except Exception as e: - verbose_router_logger.exception(f"Error updating router with search tools: {e!s}") + verbose_router_logger.exception(f"Error updating router with search tools: {e}") raise e @staticmethod @@ -226,6 +226,6 @@ class SearchAPIRouter: except Exception as e: verbose_router_logger.error( - f"Error in SearchAPIRouter.async_search_with_fallbacks_helper for {search_tool_name}: {e!s}" + f"Error in SearchAPIRouter.async_search_with_fallbacks_helper for {search_tool_name}: {e}" ) raise e diff --git a/litellm/secret_managers/main.py b/litellm/secret_managers/main.py index a05ea367b19..2982d30274b 100644 --- a/litellm/secret_managers/main.py +++ b/litellm/secret_managers/main.py @@ -282,7 +282,7 @@ def get_secret( raise ValueError("Azure OIDC provider returned None token") return oidc_token except Exception as e: - error_msg = f"Azure OIDC provider failed: {e!s}" + error_msg = f"Azure OIDC provider failed: {e}" verbose_logger.error(error_msg) raise ValueError(error_msg) with open(azure_federated_token_file, "r") as f: @@ -335,7 +335,7 @@ def get_secret( ) except Exception as e: # check if it's in os.environ verbose_logger.error( - f"Defaulting to os.environ value for key={secret_name}. An exception occurred - {e!s}.\n\n{traceback.format_exc()}" + f"Defaulting to os.environ value for key={secret_name}. An exception occurred - {e}.\n\n{traceback.format_exc()}" ) secret = os.getenv(secret_name) try: diff --git a/litellm/secret_managers/secret_manager_handler.py b/litellm/secret_managers/secret_manager_handler.py index 2acb154dd59..64a00f0df58 100644 --- a/litellm/secret_managers/secret_manager_handler.py +++ b/litellm/secret_managers/secret_manager_handler.py @@ -119,7 +119,7 @@ def get_secret_from_manager( if secret is None: raise ValueError(f"No secret found in Google Secret Manager for {secret_name}") except Exception as e: - print_verbose(f"An error occurred - {e!s}") + print_verbose(f"An error occurred - {e}") raise e elif key_manager == KeyManagementSystem.HASHICORP_VAULT.value: @@ -128,7 +128,7 @@ def get_secret_from_manager( if secret is None: raise ValueError(f"No secret found in Hashicorp Secret Manager for {secret_name}") except Exception as e: - print_verbose(f"An error occurred - {e!s}") + print_verbose(f"An error occurred - {e}") raise e elif key_manager == KeyManagementSystem.CYBERARK.value: @@ -137,7 +137,7 @@ def get_secret_from_manager( if secret is None: raise ValueError(f"No secret found in CyberArk Secret Manager for {secret_name}") except Exception as e: - print_verbose(f"An error occurred - {e!s}") + print_verbose(f"An error occurred - {e}") raise e elif key_manager == KeyManagementSystem.CUSTOM.value: diff --git a/litellm/utils.py b/litellm/utils.py index 6ef3871a3c1..eb3e578b7e8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -947,7 +947,7 @@ def function_setup( except Exception as e: # Log the error but don't fail the request - verbose_logger.warning(f"Error removing thought signatures from tool call IDs: {e!s}") + verbose_logger.warning(f"Error removing thought signatures from tool call IDs: {e}") elif call_type == CallTypes.embedding.value or call_type == CallTypes.aembedding.value: messages = args[1] if len(args) > 1 else kwargs.get("input", None) elif call_type == CallTypes.image_generation.value or call_type == CallTypes.aimage_generation.value: @@ -1004,7 +1004,7 @@ def function_setup( else: messages = "default-message-value" except Exception as e: - verbose_logger.debug(f"Error extracting messages from Google contents: {e!s}") + verbose_logger.debug(f"Error extracting messages from Google contents: {e}") messages = "default-message-value" else: messages = "default-message-value" @@ -1410,7 +1410,7 @@ def client(original_function): ) kwargs["max_tokens"] = modified_max_tokens except Exception as e: - print_verbose(f"Error while checking max token limit: {e!s}") + print_verbose(f"Error while checking max token limit: {e}") # MODEL CALL result = original_function(*args, **kwargs) end_time = datetime.datetime.now() @@ -1675,7 +1675,7 @@ def client(original_function): ) kwargs["max_tokens"] = modified_max_tokens except Exception as e: - print_verbose(f"Error while checking max token limit: {e!s}") + print_verbose(f"Error while checking max token limit: {e}") # MODEL CALL result = await original_function(*args, **kwargs) @@ -2224,7 +2224,7 @@ def supports_native_streaming(model: str, custom_llm_provider: str | None) -> bo return supports_native_streaming except Exception as e: verbose_logger.debug( - f"Model not found or error in checking supports_native_streaming support. You passed model={model}, custom_llm_provider={custom_llm_provider}. Error: {e!s}" + f"Model not found or error in checking supports_native_streaming support. You passed model={model}, custom_llm_provider={custom_llm_provider}. Error: {e}" ) return False @@ -2248,7 +2248,7 @@ def supports_response_schema(model: str, custom_llm_provider: str | None = None) model, custom_llm_provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) except Exception as e: verbose_logger.debug( - f"Model not found or error in checking response schema support. You passed model={model}, custom_llm_provider={custom_llm_provider}. Error: {e!s}" + f"Model not found or error in checking response schema support. You passed model={model}, custom_llm_provider={custom_llm_provider}. Error: {e}" ) return False @@ -2362,7 +2362,7 @@ def _supports_factory(model: str, custom_llm_provider: str | None, key: str) -> return False except Exception as e: verbose_logger.debug( - f"Model not found or error in checking {key} support. You passed model={model}, custom_llm_provider={custom_llm_provider}. Error: {e!s}" + f"Model not found or error in checking {key} support. You passed model={model}, custom_llm_provider={custom_llm_provider}. Error: {e}" ) supported_by_provider = _supports_provider_info_factory(model, custom_llm_provider, key) @@ -2404,7 +2404,7 @@ def _is_explicitly_disabled_factory(model: str, custom_llm_provider: str | None, verbose_logger.debug( f"Model not found or error in checking {key} disabled state. " f"You passed model={model}, custom_llm_provider={custom_llm_provider}. " - f"Error: {e!s}" + f"Error: {e}" ) return False @@ -2537,7 +2537,7 @@ def get_supported_regions(model: str, custom_llm_provider: str | None = None) -> return None except Exception as e: verbose_logger.debug( - f"Model not found or error in checking supported_regions support. You passed model={model}, custom_llm_provider={custom_llm_provider}. Error: {e!s}" + f"Model not found or error in checking supported_regions support. You passed model={model}, custom_llm_provider={custom_llm_provider}. Error: {e}" ) return None @@ -6542,7 +6542,7 @@ class TextCompletionStreamWrapper: return response except Exception as e: - raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e!s}") + raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}") def __next__(self): # model_response = ModelResponse(stream=True, model=self.model) @@ -6868,7 +6868,7 @@ def trim_messages( return final_messages, response_tokens return final_messages except Exception as e: # [NON-Blocking, if error occurs just return final_messages - verbose_logger.exception(f"Got exception while token trimming - {e!s}") + verbose_logger.exception(f"Got exception while token trimming - {e}") return original_messages diff --git a/litellm/vector_stores/vector_store_registry.py b/litellm/vector_stores/vector_store_registry.py index 4abd587bce5..1350e2b187e 100644 --- a/litellm/vector_stores/vector_store_registry.py +++ b/litellm/vector_stores/vector_store_registry.py @@ -235,7 +235,7 @@ class VectorStoreRegistry: self.add_vector_store_to_registry(vector_store=db_vector_store) return db_vector_store except Exception as e: - verbose_logger.debug(f"Error fetching vector store from database: {e!s}") + verbose_logger.debug(f"Error fetching vector store from database: {e}") return None @@ -346,7 +346,7 @@ class VectorStoreRegistry: self.delete_vector_store_from_registry(vector_store_id=vector_store_id) vector_store = None except Exception as e: - verbose_logger.debug(f"Error verifying vector store {vector_store_id} in database: {e!s}") + verbose_logger.debug(f"Error verifying vector store {vector_store_id} in database: {e}") # Fall back to database if not found in memory (or was deleted) if vector_store is None and prisma_client is not None: @@ -355,7 +355,7 @@ class VectorStoreRegistry: vector_store_id=vector_store_id, prisma_client=prisma_client ) except Exception as e: - verbose_logger.debug(f"Error fetching vector store {vector_store_id} from database: {e!s}") + verbose_logger.debug(f"Error fetching vector store {vector_store_id} from database: {e}") if vector_store is not None: # Create a copy to avoid modifying the registry From eea9bb1497d69ed16fcc022a51bc81d6048da735 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 1 Aug 2026 17:13:15 -0700 Subject: [PATCH 034/124] chore(lint): use a PEP 604 union for the flat tool_choice helper (UP007) --- .../litellm_responses_transformation/transformation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 1cc1e88f1fa..79a4620dc80 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -141,7 +141,7 @@ def _tool_call_dict_from_output_item(item: Mapping[str, Any], index: int) -> _Ch return tool_call_dict -def _flat_responses_tool_choice(choice_type: str, name: str) -> Union[ToolChoiceFunctionParam, ToolChoiceCustomParam]: +def _flat_responses_tool_choice(choice_type: str, name: str) -> ToolChoiceFunctionParam | ToolChoiceCustomParam: if choice_type == "custom": return ToolChoiceCustomParam(type="custom", name=name) return ToolChoiceFunctionParam(type="function", name=name) From f25f1d292164487941290f3038d4489b8d80078c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 1 Aug 2026 17:42:03 -0700 Subject: [PATCH 035/124] feat(proxy): resolve Cursor thinking/fast model-name suffixes on /cursor/chat/completions Cursor appends -thinking- and -fast to custom model names when the user picks a thinking level or fast mode, so a model configured as claude-opus-5 arrives as claude-opus-5-thinking-xhigh-fast and fails routing with no healthy deployments. When the raw name is not servable by the router but the suffix-stripped base name is, rewrite the body to the base model and carry the thinking level into reasoning_effort (chat bodies) or reasoning.effort (Responses bodies), never clobbering an effort the client already sent. Explicitly configured aliases keep winning because the raw-name servability check runs first. --- .../proxy/response_api_endpoints/endpoints.py | 64 ++++- .../response_api_endpoints/test_endpoints.py | 228 ++++++++++++++++++ 2 files changed, 288 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 7dcb01d3e59..6b0b4db1b18 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -3,7 +3,7 @@ import json import time from collections.abc import AsyncIterator, Mapping from types import MappingProxyType -from typing import Any, cast +from typing import TYPE_CHECKING, Any, NamedTuple, cast, get_args from uuid import uuid4 import fastapi @@ -19,9 +19,12 @@ from litellm.proxy.auth.user_api_key_auth import ( user_api_key_auth_websocket, ) from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing -from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse +from litellm.types.llms.openai import REASONING_EFFORT, ResponseAPIUsage, ResponsesAPIResponse from litellm.types.responses.main import DeleteResponseResult +if TYPE_CHECKING: + from litellm.router import Router + router = APIRouter() _user_api_key_auth_dep = Depends(user_api_key_auth) @@ -93,6 +96,58 @@ def _is_chat_completions_body(data: Mapping[str, Any]) -> bool: return "messages" in data and "input" not in data +_CURSOR_THINKING_SEPARATOR = "-thinking-" +_CURSOR_FAST_SUFFIX = "-fast" +_CURSOR_THINKING_LEVELS: frozenset[str] = frozenset(get_args(REASONING_EFFORT)) + + +class _CursorModelVariant(NamedTuple): + base_model: str + reasoning_effort: str | None + + +def _parse_cursor_model_variant(model: str) -> _CursorModelVariant: + stripped = model.removesuffix(_CURSOR_FAST_SUFFIX) + base, separator, level = stripped.rpartition(_CURSOR_THINKING_SEPARATOR) + if separator and base and level in _CURSOR_THINKING_LEVELS: + return _CursorModelVariant(base, level) + return _CursorModelVariant(stripped, None) + + +def _router_can_serve(model: str, llm_router: "Router | None") -> bool: + if llm_router is None: + return False + if model in llm_router.model_names or model in llm_router.model_group_alias: + return True + if model in llm_router.team_public_model_names: + return True + return bool(llm_router.pattern_router.get_pattern(model)) + + +def _resolve_cursor_model_variant( + data: dict, llm_router: "Router | None" +) -> dict: # mutable-ok: the parsed request body contract is a plain dict + model = data.get("model") + if not isinstance(model, str) or _router_can_serve(model, llm_router): + return data + variant = _parse_cursor_model_variant(model) + if variant.base_model == model or not _router_can_serve(variant.base_model, llm_router): + return data + resolved = {**data, "model": variant.base_model} # mutable-ok: plain body dict + if variant.reasoning_effort is None: + return resolved + if _is_chat_completions_body(data): + if "reasoning_effort" in data: + return resolved + return {**resolved, "reasoning_effort": variant.reasoning_effort} # mutable-ok: plain body dict + reasoning = data.get("reasoning") + if isinstance(reasoning, dict): + if reasoning.get("effort"): + return resolved + return {**resolved, "reasoning": {**reasoning, "effort": variant.reasoning_effort}} # mutable-ok: same + return {**resolved, "reasoning": {"effort": variant.reasoning_effort}} # mutable-ok: plain body dict + + @router.post( "/v1/responses", dependencies=[Depends(user_api_key_auth)], @@ -440,7 +495,8 @@ async def cursor_chat_completions( from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ModelResponse - data = await _read_request_body(request=request) + raw_body = await _read_request_body(request=request) + data = _resolve_cursor_model_variant(raw_body, llm_router) if _is_chat_completions_body(data): # Genuine chat completions body (Cursor sends these for models whose BYOK it @@ -448,7 +504,7 @@ async def cursor_chat_completions( # Keyed on messages CONTENT, not key presence: Cursor can send a null or # empty messages stub alongside a real agent-mode input array normalized = _normalize_tool_dialect(data, to_chat=True) - if normalized is not data: + if normalized is not raw_body: _safe_set_request_parsed_body(request=request, parsed_body=normalized) return await chat_completion( request=request, diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index 00ac8ca386a..60168e7f912 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -1340,3 +1340,231 @@ class TestChatCompletionsBodyDetection: assert response.status_code == 200 assert mock_router.aresponses.call_args is not None assert mock_router.aresponses.call_args.kwargs["input"] == [{"role": "user", "content": "hello"}] + + +class TestParseCursorModelVariant: + @pytest.mark.parametrize( + "model,expected_base,expected_effort", + [ + ("claude-opus-5-thinking-high", "claude-opus-5", "high"), + ("claude-opus-5-thinking-xhigh-fast", "claude-opus-5", "xhigh"), + ("gemini-3.0-pro-thinking-low", "gemini-3.0-pro", "low"), + ("claude-opus-5-fast", "claude-opus-5", None), + ("gpt-5.6-sol", "gpt-5.6-sol", None), + ("foo-thinking-ultra-fast", "foo-thinking-ultra", None), + ("-thinking-high", "-thinking-high", None), + ], + ) + def test_parse_matrix(self, model, expected_base, expected_effort): + from litellm.proxy.response_api_endpoints.endpoints import _parse_cursor_model_variant + + variant = _parse_cursor_model_variant(model) + assert variant.base_model == expected_base + assert variant.reasoning_effort == expected_effort + + +class TestResolveCursorModelVariant: + @pytest.fixture(scope="class") + def wildcard_router(self): + from litellm import Router + + return Router( + model_list=[ + {"model_name": "anthropic/*", "litellm_params": {"model": "anthropic/*", "api_key": "fake"}}, + {"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": "fake"}}, + { + "model_name": "explicit-alias-thinking-high", + "litellm_params": {"model": "anthropic/claude-opus-5", "api_key": "fake"}, + }, + ] + ) + + def test_chat_body_suffix_stripped_into_reasoning_effort(self, wildcard_router): + from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant + + body = { + "model": "claude-opus-5-thinking-xhigh-fast", + "messages": [{"role": "user", "content": "hi"}], + } + resolved = _resolve_cursor_model_variant(body, wildcard_router) + assert resolved["model"] == "claude-opus-5" + assert resolved["reasoning_effort"] == "xhigh" + assert resolved["messages"] == body["messages"] + assert body["model"] == "claude-opus-5-thinking-xhigh-fast" + + def test_responses_body_suffix_stripped_into_reasoning_dict(self, wildcard_router): + from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant + + body = {"model": "claude-opus-5-thinking-high", "input": [{"role": "user", "content": "hi"}]} + resolved = _resolve_cursor_model_variant(body, wildcard_router) + assert resolved["model"] == "claude-opus-5" + assert resolved["reasoning"] == {"effort": "high"} + + def test_responses_body_merges_effort_into_existing_reasoning(self, wildcard_router): + from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant + + body = { + "model": "claude-opus-5-thinking-high", + "input": [{"role": "user", "content": "hi"}], + "reasoning": {"summary": "auto"}, + } + resolved = _resolve_cursor_model_variant(body, wildcard_router) + assert resolved["model"] == "claude-opus-5" + assert resolved["reasoning"] == {"summary": "auto", "effort": "high"} + + def test_existing_reasoning_effort_wins_but_model_still_rewritten(self, wildcard_router): + from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant + + chat_body = { + "model": "claude-opus-5-thinking-high", + "messages": [{"role": "user", "content": "hi"}], + "reasoning_effort": "low", + } + resolved_chat = _resolve_cursor_model_variant(chat_body, wildcard_router) + assert resolved_chat["model"] == "claude-opus-5" + assert resolved_chat["reasoning_effort"] == "low" + + responses_body = { + "model": "claude-opus-5-thinking-high", + "input": [{"role": "user", "content": "hi"}], + "reasoning": {"effort": "low"}, + } + resolved_responses = _resolve_cursor_model_variant(responses_body, wildcard_router) + assert resolved_responses["model"] == "claude-opus-5" + assert resolved_responses["reasoning"] == {"effort": "low"} + + def test_fast_only_suffix_strips_without_reasoning(self, wildcard_router): + from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant + + body = {"model": "claude-opus-5-fast", "messages": [{"role": "user", "content": "hi"}]} + resolved = _resolve_cursor_model_variant(body, wildcard_router) + assert resolved["model"] == "claude-opus-5" + assert "reasoning_effort" not in resolved + + def test_explicitly_configured_suffixed_name_untouched(self, wildcard_router): + from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant + + body = {"model": "explicit-alias-thinking-high", "messages": [{"role": "user", "content": "hi"}]} + assert _resolve_cursor_model_variant(body, wildcard_router) is body + + def test_provider_inferable_bare_name_untouched(self, wildcard_router): + from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant + + body = {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]} + assert _resolve_cursor_model_variant(body, wildcard_router) is body + + def test_unservable_base_untouched(self, wildcard_router): + from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant + + body = {"model": "totally-unknown-thinking-high", "messages": [{"role": "user", "content": "hi"}]} + assert _resolve_cursor_model_variant(body, wildcard_router) is body + + def test_no_router_untouched(self): + from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant + + body = {"model": "claude-opus-5-thinking-high", "messages": [{"role": "user", "content": "hi"}]} + assert _resolve_cursor_model_variant(body, None) is body + + def test_missing_or_non_string_model_untouched(self, wildcard_router): + from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant + + no_model = {"messages": [{"role": "user", "content": "hi"}]} + assert _resolve_cursor_model_variant(no_model, wildcard_router) is no_model + null_model = {"model": None, "messages": [{"role": "user", "content": "hi"}]} + assert _resolve_cursor_model_variant(null_model, wildcard_router) is null_model + + +def _router_serving_only(base_model: str) -> MagicMock: + mock_router = MagicMock() + mock_router.model_names = set() + mock_router.model_group_alias = {} + mock_router.team_public_model_names = frozenset() + mock_router.pattern_router.get_pattern.side_effect = ( + lambda model: [{"model_name": "anthropic/*"}] if model == base_model else None + ) + return mock_router + + +class TestCursorModelSuffixResolutionEndToEnd: + @pytest.mark.asyncio + async def test_chat_arm_rewrites_suffixed_model_before_delegation(self): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + seen = {} + + async def fake_chat_completion(request, fastapi_response, model, user_api_key_dict): + from litellm.proxy.common_utils.http_parsing_utils import _read_request_body + + seen["body"] = await _read_request_body(request=request) + return {"id": "chatcmpl-fake", "object": "chat.completion", "choices": []} + + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-1234") + try: + with ( + patch("litellm.proxy.proxy_server.llm_router", new=_router_serving_only("claude-opus-5")), + patch("litellm.proxy.proxy_server.chat_completion", new=fake_chat_completion), + ): + client = TestClient(app) + response = client.post( + "/cursor/chat/completions", + json={ + "model": "claude-opus-5-thinking-xhigh-fast", + "messages": [{"role": "user", "content": "hi"}], + }, + headers={"Authorization": "Bearer sk-1234"}, + ) + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 200 + assert seen["body"]["model"] == "claude-opus-5" + assert seen["body"]["reasoning_effort"] == "xhigh" + assert seen["body"]["messages"] == [{"role": "user", "content": "hi"}] + + @pytest.mark.asyncio + async def test_responses_arm_rewrites_suffixed_model_before_routing(self): + from openai.types.responses import ResponseOutputMessage, ResponseOutputText + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.types.llms.openai import ResponsesAPIResponse + + mock_response = ResponsesAPIResponse( + id="resp_suffix1", + created_at=1234567890, + model="claude-opus-5", + object="response", + output=[ + ResponseOutputMessage( + id="msg_suffix1", + type="message", + role="assistant", + status="completed", + content=[ResponseOutputText(type="output_text", text="ok", annotations=[])], + ) + ], + ) + + mock_router = _router_serving_only("claude-opus-5") + mock_router.aresponses = AsyncMock(return_value=mock_response) + + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-1234") + try: + with patch("litellm.proxy.proxy_server.llm_router", new=mock_router): + client = TestClient(app) + response = client.post( + "/cursor/chat/completions", + json={ + "model": "claude-opus-5-thinking-high", + "input": [{"role": "user", "content": "hello"}], + }, + headers={"Authorization": "Bearer sk-1234"}, + ) + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 200 + assert mock_router.aresponses.call_args is not None + assert mock_router.aresponses.call_args.kwargs["model"] == "claude-opus-5" + assert mock_router.aresponses.call_args.kwargs["reasoning"] == {"effort": "high"} From deaade232d352379e4cca16f93417524b37efd42 Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 2 Aug 2026 00:57:44 +0000 Subject: [PATCH 036/124] feat(gemini): add gemini-robotics-er-2-preview and gemini-robotics-er-1.6-preview pricing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 93 +++++++++++++++++++ model_prices_and_context_window.json | 93 +++++++++++++++++++ 2 files changed, 186 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 07f04136bd0..158f7e3b8ba 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -18883,6 +18883,99 @@ "search_context_size_high": 0.035 } }, + "gemini/gemini-robotics-er-2-preview": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_audio_token": 2e-06, + "input_cost_per_token": 2e-06, + "litellm_provider": "gemini", + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 1e-05, + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-robotics-er-2", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" + }, + "gemini/gemini-robotics-er-1.6-preview": { + "input_cost_per_audio_token": 2e-06, + "input_cost_per_token": 1e-06, + "litellm_provider": "gemini", + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 5e-06, + "output_cost_per_token": 5e-06, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-robotics-er", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" + }, "gemini-2.5-computer-use-preview-10-2025": { "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 346f613ea3e..5e5e741705a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -18961,6 +18961,99 @@ "search_context_size_high": 0.035 } }, + "gemini/gemini-robotics-er-2-preview": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_audio_token": 2e-06, + "input_cost_per_token": 2e-06, + "litellm_provider": "gemini", + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 1e-05, + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-robotics-er-2", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" + }, + "gemini/gemini-robotics-er-1.6-preview": { + "input_cost_per_audio_token": 2e-06, + "input_cost_per_token": 1e-06, + "litellm_provider": "gemini", + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 5e-06, + "output_cost_per_token": 5e-06, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-robotics-er", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" + }, "gemini-2.5-computer-use-preview-10-2025": { "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, From 7c8364c991b5533821a54b8d01c5e8af5965f44d Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 3 Aug 2026 11:02:51 -0700 Subject: [PATCH 037/124] fix(team-callbacks): actually stop logging when disable_logging is called (#35520) disable_team_logging cleared only metadata["callback_settings"], but callbacks registered through POST /team/{team_id}/callback and the Admin UI live in metadata["logging"], and request-time resolution stops at that slot without ever reading callback_settings. The endpoint reported success while the team kept sending request and response data to its third-party destination. Empty the logging slot alongside the existing callback_settings reset, and refresh the cached team object so the change applies to keys that are already in flight rather than at the next cache expiry. The same refresh is added to add_team_callbacks, which has the symmetric problem of a newly registered callback staying dormant until the entry expires. Resolves LIT-5101 --- .../team_callback_endpoints.py | 46 +++- .../test_team_callback_endpoints.py | 224 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 3 + 3 files changed, 270 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 8fab485cc7a..bdd8244427a 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -37,7 +37,10 @@ from litellm.proxy.litellm_pre_call_utils import ( _get_validated_callback_metadata, convert_key_logging_metadata_to_callback, ) -from litellm.proxy.management_endpoints.team_endpoints import _verify_team_access +from litellm.proxy.management_endpoints.team_endpoints import ( + _refresh_cached_team, + _verify_team_access, +) from litellm.proxy.management_helpers.utils import management_endpoint_wrapper from litellm.repositories.team_repository import TeamRepository @@ -262,7 +265,11 @@ async def add_team_callbacks( """ try: from litellm.proxy._types import CommonProxyErrors - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) if prisma_client is None: raise HTTPException( @@ -316,6 +323,17 @@ async def add_team_callbacks( new_team_row = await TeamRepository(prisma_client).table.update( where={"team_id": team_id}, data={"metadata": team_metadata_json}, # type: ignore + # `object_permission` is included so `_refresh_cached_team` doesn't + # write a cached team with the relation nulled out — see + # team_model_add for the full rationale. + include={"object_permission": True}, # mutable-ok: prisma include takes a dict literal + ) + + # Without this a newly registered callback stays dormant for existing keys. + await _refresh_cached_team( + team_row=new_team_row, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, ) await _emit_team_callback_audit_log( @@ -363,6 +381,9 @@ async def disable_team_logging( """ Disable all logging callbacks for a team + Callbacks registered through POST /team/{team_id}/callback and the Admin UI are cleared, so + re-enabling logging means registering them again with their callback_vars + Parameters: - team_id (str, required): The unique identifier for the team @@ -375,7 +396,11 @@ async def disable_team_logging( """ try: - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) if prisma_client is None: raise HTTPException(status_code=500, detail={"error": "No db connected"}) @@ -408,6 +433,9 @@ async def disable_team_logging( # Update metadata team_metadata["callback_settings"] = team_callback_settings_obj.model_dump() + # _get_dynamic_logging_metadata stops at metadata["logging"], where the API + # and Admin UI register callbacks, without ever reading callback_settings. + team_metadata["logging"] = [] # mutable-ok: the disabled state is persisted as an empty JSON array team_metadata = encrypt_callback_vars(team_metadata) team_metadata_json = json.dumps(team_metadata) @@ -415,6 +443,10 @@ async def disable_team_logging( updated_team = await TeamRepository(prisma_client).table.update( where={"team_id": team_id}, data={"metadata": team_metadata_json}, # type: ignore + # `object_permission` is included so `_refresh_cached_team` doesn't + # write a cached team with the relation nulled out — see + # team_model_add for the full rationale. + include={"object_permission": True}, # mutable-ok: prisma include takes a dict literal ) if updated_team is None: @@ -423,6 +455,14 @@ async def disable_team_logging( detail={"error": f"Team id = {team_id} does not exist. Error updating team logging"}, ) + # Request-time callback resolution reads the cached team, so without this + # the DB says logging is off while live keys keep sending until it expires. + await _refresh_cached_team( + team_row=updated_team, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + # Disabling a team's logging callbacks is itself a logging-control # action — emit an audit-log row so the action remains traceable # even though the team's own observability is now off. diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py index 3b2b1ccb793..21e25d30b82 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py @@ -66,6 +66,23 @@ def _admin_auth() -> UserAPIKeyAuth: ) +@pytest.fixture(autouse=True) +def stub_team_cache_refresh(): + """Keep the cached-team refresh out of the way of the mocked prisma rows. + + The endpoints under test now refresh the auth cache after their DB write. + That helper validates a real Prisma row into LiteLLM_TeamTableCachedObj, + which the MagicMock rows these tests use cannot satisfy. The refresh being + called at all is asserted explicitly in + test_disable_team_logging_refreshes_cached_team. + """ + with patch( + "litellm.proxy.management_endpoints.team_callback_endpoints._refresh_cached_team", + new_callable=AsyncMock, + ) as refresh: + yield refresh + + @pytest.fixture def unauthorized_caller(): return UserAPIKeyAuth( @@ -238,6 +255,9 @@ async def test_disable_team_logging_emits_audit_log_when_enabled(monkeypatch): assert before["metadata"]["callback_settings"]["success_callback"] == ["langfuse"] assert after["metadata"]["callback_settings"]["success_callback"] == [] assert after["metadata"]["callback_settings"]["failure_callback"] == [] + # The audit row has to show the slot the callbacks actually live in, so a + # disable of a logging-configured team does not record an empty diff. + assert after["metadata"]["logging"] == [] @pytest.mark.asyncio @@ -718,3 +738,207 @@ async def test_get_team_callbacks_reports_empty_for_team_without_callbacks(): "failure_callbacks": [], "callback_vars": {}, } + + +@pytest.mark.asyncio +async def test_disable_team_logging_stops_callbacks_registered_via_api(): + """Disabling logging must stop the callbacks that are actually running. + + Callbacks registered through the API or the Admin UI live in + metadata["logging"], and request-time resolution stops at that slot without + reading callback_settings. Clearing only callback_settings therefore reports + success while the team keeps sending to its logging destination. This drives + the endpoint and then asks the real request-time resolver what the written + row would do. + """ + from litellm.proxy.litellm_pre_call_utils import _get_dynamic_logging_metadata + + metadata = { + "logging": [ + { + "callback_name": "langsmith", + "callback_type": "success", + "callback_vars": {"langsmith_project": "tenant-project"}, + } + ] + } + mock_prisma = _patch_prisma(_team_row(team_id="team-1", metadata=metadata)) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.master_key", None), + ): + response = await disable_team_logging( + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + + assert response["status"] == "success" + written = json.loads(mock_prisma.db.litellm_teamtable.update.await_args.kwargs["data"]["metadata"]) + assert written["logging"] == [] + + resolved = _get_dynamic_logging_metadata( + UserAPIKeyAuth(api_key="hashed", team_id="team-1", team_metadata=written), + proxy_config=MagicMock(**{"load_team_config.return_value": {}}), + ) + assert not (resolved.success_callback if resolved else None) + assert not (resolved.failure_callback if resolved else None) + + +@pytest.mark.asyncio +async def test_disable_team_logging_refreshes_cached_team(stub_team_cache_refresh): + """The DB write alone does not stop delivery. + + Auth serves a cached team object and request-time callback resolution reads + the metadata off it, so without this refresh a key that is already in flight + keeps sending to the destination until the cache entry expires. + """ + metadata = { + "logging": [ + { + "callback_name": "langsmith", + "callback_type": "success", + "callback_vars": {"langsmith_project": "tenant-project"}, + } + ] + } + mock_prisma = _patch_prisma(_team_row(team_id="team-1", metadata=metadata)) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.master_key", None), + ): + await disable_team_logging( + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + + stub_team_cache_refresh.assert_awaited_once() + refreshed = stub_team_cache_refresh.await_args.kwargs["team_row"] + assert refreshed is mock_prisma.db.litellm_teamtable.update.return_value + # The row fed to the cache has to carry object_permission, or the refresh + # publishes a team whose tool allowlists look empty, which reads as + # unrestricted on the search-tool and MCP-tool checks. + update_kwargs = mock_prisma.db.litellm_teamtable.update.await_args.kwargs + assert update_kwargs["include"]["object_permission"] is True + + +@pytest.mark.asyncio +async def test_add_team_callbacks_refreshes_cached_team(stub_team_cache_refresh): + """Registering a callback must take effect for keys that are already live.""" + mock_prisma = _patch_prisma(_team_row(team_id="team-1", metadata={"logging": []})) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.master_key", None), + ): + await add_team_callbacks( + data=AddTeamCallback( + callback_name="langsmith", + callback_type="success", + callback_vars={"langsmith_project": "tenant-project"}, + ), + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + + stub_team_cache_refresh.assert_awaited_once() + refreshed = stub_team_cache_refresh.await_args.kwargs["team_row"] + assert refreshed is mock_prisma.db.litellm_teamtable.update.return_value + update_kwargs = mock_prisma.db.litellm_teamtable.update.await_args.kwargs + assert update_kwargs["include"]["object_permission"] is True + + +@pytest.mark.asyncio +async def test_disable_team_logging_clears_both_metadata_shapes(): + """A team carrying both shapes ends up with neither active.""" + from litellm.proxy.litellm_pre_call_utils import _get_dynamic_logging_metadata + + metadata = { + "logging": [ + { + "callback_name": "langsmith", + "callback_type": "success_and_failure", + "callback_vars": {"langsmith_project": "tenant-project"}, + } + ], + "callback_settings": { + "success_callback": ["gcs_bucket"], + "failure_callback": ["langfuse"], + "callback_vars": {"gcs_bucket_name": "legacy-bucket"}, + }, + } + mock_prisma = _patch_prisma(_team_row(team_id="team-1", metadata=metadata)) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.master_key", None), + ): + await disable_team_logging( + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + + written = json.loads(mock_prisma.db.litellm_teamtable.update.await_args.kwargs["data"]["metadata"]) + assert written["logging"] == [] + assert written["callback_settings"]["success_callback"] == [] + assert written["callback_settings"]["failure_callback"] == [] + + resolved = _get_dynamic_logging_metadata( + UserAPIKeyAuth(api_key="hashed", team_id="team-1", team_metadata=written), + proxy_config=MagicMock(**{"load_team_config.return_value": {}}), + ) + assert not (resolved.success_callback if resolved else None) + assert not (resolved.failure_callback if resolved else None) + + +@pytest.mark.asyncio +async def test_disable_team_logging_leaves_team_re_enablable(): + """The emptied slot must still accept a fresh registration afterwards.""" + metadata = { + "logging": [ + { + "callback_name": "langsmith", + "callback_type": "success", + "callback_vars": {"langsmith_project": "tenant-project"}, + } + ] + } + row = _team_row(team_id="team-1", metadata=metadata) + mock_prisma = _patch_prisma(row) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.master_key", None), + ): + await disable_team_logging( + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + row.metadata = json.loads(mock_prisma.db.litellm_teamtable.update.await_args.kwargs["data"]["metadata"]) + row.model_dump.return_value["metadata"] = row.metadata + + await add_team_callbacks( + data=AddTeamCallback( + callback_name="langfuse", + callback_type="success", + callback_vars={"langfuse_public_key": "pk-lf-new"}, + ), + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + + written = json.loads(mock_prisma.db.litellm_teamtable.update.await_args.kwargs["data"]["metadata"]) + assert [entry["callback_name"] for entry in written["logging"]] == ["langfuse"] diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d24937fd672..58855f80b42 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -14092,6 +14092,9 @@ export interface paths { * Disable Team Logging * @description Disable all logging callbacks for a team * + * Callbacks registered through POST /team/{team_id}/callback and the Admin UI are cleared, so + * re-enabling logging means registering them again with their callback_vars + * * Parameters: * - team_id (str, required): The unique identifier for the team * From 5b6194f427356ae7c6ca1ec6ea84bc4afc92552c Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:55:10 -0700 Subject: [PATCH 038/124] fix(proxy): backfill null user_email on existing users during JWT auth (#34588) * fix(proxy): backfill null user_email on existing users during JWT auth Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): guard mapped-key email backfill and make null update atomic Resolve Greptile review on the JWT user_email backfill: - only backfill when the mapped virtual-key owner is the JWT principal, so a mismatched admin-created mapping cannot write one user's email onto another - make the best-effort mapped-key enrichment non-fatal so a database outage on a cached-key request no longer fails otherwise-valid authentication - persist the backfill with an atomic null-guarded update_many so concurrent writers cannot overwrite an already-populated email Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep cache coherent when a concurrent backfill wins the null-email update * fix(proxy): cache DB-persisted email after JWT backfill, not the proposed value Resolve the Greptile finding that a successful null-guarded backfill could cache this request's proposed email even if a concurrent ordinary user update wrote a different email first. The helper now always re-reads the row after the atomic update and refreshes the cache from the value the database holds, so cache-hit auth and attribution stay consistent with the persisted record. Annotate the Prisma and model_copy dict literals to keep the LIT002 budget within its ceiling. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: shivam Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: ryan-crabbe-berri --- litellm/proxy/auth/auth_checks.py | 41 +++- litellm/proxy/auth/user_api_key_auth.py | 20 ++ litellm/repositories/user_repository.py | 11 + .../proxy/auth/test_auth_checks.py | 223 +++++++++++++++++ .../proxy/auth/test_user_api_key_auth.py | 225 ++++++++++++++++++ 5 files changed, 519 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 52943737eed..6b1d845ec95 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -1620,6 +1620,34 @@ async def _get_fuzzy_user_object( return response +async def _backfill_null_user_email( + prisma_client: PrismaClient | None, + user_api_key_cache: UserApiKeyCache, + user_row: LiteLLM_UserTable, + user_email: str | None, +) -> LiteLLM_UserTable: + if user_email is None or user_row.user_email is not None or prisma_client is None: + return user_row + + user_repo = UserRepository(prisma_client) + await user_repo.backfill_null_user_email( + user_id=user_row.user_id, + user_email=user_email, + ) + db_row = await user_repo.find_by_id(user_row.user_id) + if db_row is None: + return user_row + email_update = {"user_email": db_row.user_email} # mutable-ok: model_copy update payload is dict-shaped + updated_row = user_row.model_copy(update=email_update) + await user_api_key_cache.async_set_cache( + key=user_row.user_id, + value=updated_row, + model_type=LiteLLM_UserTable, + ttl=get_management_object_ttl(user_api_key_cache), + ) + return updated_row + + @log_db_metrics async def get_user_object( user_id: str | None, @@ -1648,7 +1676,12 @@ async def get_user_object( model_type=LiteLLM_UserTable, ) if cached_user_obj is not None: - return cached_user_obj + return await _backfill_null_user_email( + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + user_row=cached_user_obj, + user_email=user_email, + ) # else, check db if prisma_client is None: raise Exception("No db connected") @@ -1732,6 +1765,12 @@ async def get_user_object( response.organization_memberships = _dumped_memberships _response = LiteLLM_UserTable.model_validate(dict(response)) + _response = await _backfill_null_user_email( + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + user_row=_response, + user_email=user_email, + ) response_dict = _response.model_dump() # save the user object to cache diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 2905eb86c0f..286837c8909 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -1232,6 +1232,26 @@ async def _user_api_key_auth_builder( valid_token.jwt_claims = jwt_claims do_standard_jwt_auth = False # Fall through to virtual key checks + if valid_token.user_id is not None and valid_token.user_email is None: + mapped_claims = jwt_claims or {} # mutable-ok: empty-dict fallback for the None-claims case + mapped_user_email = jwt_handler.get_user_email(token=mapped_claims, default_value=None) + mapped_jwt_user_id = jwt_handler.get_user_id(token=mapped_claims, default_value=None) + if mapped_user_email is not None and mapped_jwt_user_id == valid_token.user_id: + try: + mapped_user_obj = await get_user_object( + user_id=valid_token.user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + user_id_upsert=False, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + user_email=mapped_user_email, + ) + except Exception as e: + verbose_proxy_logger.debug(f"JWT mapped-key user_email backfill skipped: {e}") + else: + if mapped_user_obj is not None: + valid_token.user_email = mapped_user_obj.user_email elif isinstance(resolve_result, _PendingAutoRegister): # Run full JWT policy (RBAC, scope, custom_validate, # email-domain) via auth_builder, then create the key diff --git a/litellm/repositories/user_repository.py b/litellm/repositories/user_repository.py index 5eb326bda18..2b567e8b52a 100644 --- a/litellm/repositories/user_repository.py +++ b/litellm/repositories/user_repository.py @@ -195,6 +195,17 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): return await self.update(user_id, data, id_field="user_id") + async def backfill_null_user_email(self, user_id: str, user_email: str) -> int: + """Set user_email only when the stored value is null, atomically at the database. + + Returns the number of rows updated: 0 means another writer already set an email. + """ + updated_count: int = await self.table.update_many( + where={"user_id": user_id, "user_email": None}, # mutable-ok: Prisma query filters are dict-shaped + data={"user_email": user_email}, # mutable-ok: Prisma update payloads are dict-shaped + ) + return updated_count + async def delete_user(self, user_id: str) -> LiteLLM_UserTable | None: """Delete a user.""" return await self.delete(user_id, id_field="user_id") diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 5f3b0f36b95..f5aa695cb78 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -883,6 +883,229 @@ async def test_get_user_object_upsert_includes_user_email(): assert creation_args["user_id"] == "new_test_user" +@pytest.mark.asyncio +async def test_get_user_object_backfills_null_email_from_cache_hit(): + """ + Regression (LIT-4710): an existing user row with a null user_email must be + backfilled from the JWT-provided email even when served from cache, so the + JWT-to-virtual-key path (which resolves straight to the cached user) stops + logging user_api_key_user_email=null forever. Before the fix the cached row + was returned unchanged and the DB was never updated. + """ + cache = UserApiKeyCache() + existing = LiteLLM_UserTable( + user_id="jwt-user-1", user_email=None, user_role="internal_user" + ) + await cache.async_set_cache( + key="jwt-user-1", value=existing, model_type=LiteLLM_UserTable + ) + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.update_many = AsyncMock(return_value=1) + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock( + return_value=LiteLLM_UserTable( + user_id="jwt-user-1", + user_email="jwt-user-1@example.com", + user_role="internal_user", + ) + ) + + result = await get_user_object( + user_id="jwt-user-1", + prisma_client=mock_prisma_client, + user_api_key_cache=cache, + user_id_upsert=False, + proxy_logging_obj=None, + user_email="jwt-user-1@example.com", + ) + + assert result is not None + assert result.user_email == "jwt-user-1@example.com" + + mock_prisma_client.db.litellm_usertable.update_many.assert_called_once() + update_kwargs = mock_prisma_client.db.litellm_usertable.update_many.call_args.kwargs + assert update_kwargs["where"] == {"user_id": "jwt-user-1", "user_email": None} + assert update_kwargs["data"]["user_email"] == "jwt-user-1@example.com" + + refreshed = await cache.async_get_cache( + key="jwt-user-1", model_type=LiteLLM_UserTable + ) + assert refreshed is not None + assert refreshed.user_email == "jwt-user-1@example.com" + + +@pytest.mark.asyncio +async def test_get_user_object_backfills_null_email_from_db_read(): + """ + Regression (LIT-4710): a user row read from the DB with a null user_email is + backfilled from the JWT-provided email before it is cached and returned. + """ + cache = UserApiKeyCache() + db_row = LiteLLM_UserTable( + user_id="jwt-user-3", user_email=None, user_role="internal_user" + ) + backfilled_row = LiteLLM_UserTable( + user_id="jwt-user-3", + user_email="jwt-user-3@example.com", + user_role="internal_user", + ) + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock( + side_effect=[db_row, backfilled_row] + ) + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=None) + mock_prisma_client.db.litellm_usertable.update_many = AsyncMock(return_value=1) + + with patch( + "litellm.proxy.auth.auth_checks._should_check_db", return_value=True + ): + result = await get_user_object( + user_id="jwt-user-3", + prisma_client=mock_prisma_client, + user_api_key_cache=cache, + user_id_upsert=False, + proxy_logging_obj=None, + user_email="jwt-user-3@example.com", + ) + + assert result is not None + assert result.user_email == "jwt-user-3@example.com" + mock_prisma_client.db.litellm_usertable.update_many.assert_called_once() + + refreshed = await cache.async_get_cache( + key="jwt-user-3", model_type=LiteLLM_UserTable + ) + assert refreshed is not None + assert refreshed.user_email == "jwt-user-3@example.com" + + +@pytest.mark.asyncio +async def test_get_user_object_does_not_overwrite_existing_email(): + """ + LIT-4710 guardrail: backfill is scoped to null-to-value. An existing non-null + user_email (e.g. one an operator set intentionally) must never be overwritten + by the JWT-provided email. + """ + cache = UserApiKeyCache() + existing = LiteLLM_UserTable( + user_id="jwt-user-2", + user_email="operator-set@example.com", + user_role="internal_user", + ) + await cache.async_set_cache( + key="jwt-user-2", value=existing, model_type=LiteLLM_UserTable + ) + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.update_many = AsyncMock(return_value=0) + + result = await get_user_object( + user_id="jwt-user-2", + prisma_client=mock_prisma_client, + user_api_key_cache=cache, + user_id_upsert=False, + proxy_logging_obj=None, + user_email="different@example.com", + ) + + assert result is not None + assert result.user_email == "operator-set@example.com" + mock_prisma_client.db.litellm_usertable.update_many.assert_not_called() + + +@pytest.mark.asyncio +async def test_get_user_object_backfill_race_prefers_db_email(): + """ + LIT-4710 race guard: when the null-guarded update matches 0 rows because a + concurrent writer already backfilled an email, the cache must be refreshed + with the value the DB accepted, not this request's proposed email. + """ + cache = UserApiKeyCache() + existing = LiteLLM_UserTable( + user_id="jwt-user-4", user_email=None, user_role="internal_user" + ) + await cache.async_set_cache( + key="jwt-user-4", value=existing, model_type=LiteLLM_UserTable + ) + + winner_row = LiteLLM_UserTable( + user_id="jwt-user-4", + user_email="winner@example.com", + user_role="internal_user", + ) + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.update_many = AsyncMock(return_value=0) + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock( + return_value=winner_row + ) + + result = await get_user_object( + user_id="jwt-user-4", + prisma_client=mock_prisma_client, + user_api_key_cache=cache, + user_id_upsert=False, + proxy_logging_obj=None, + user_email="loser@example.com", + ) + + assert result is not None + assert result.user_email == "winner@example.com" + + refreshed = await cache.async_get_cache( + key="jwt-user-4", model_type=LiteLLM_UserTable + ) + assert refreshed is not None + assert refreshed.user_email == "winner@example.com" + + +@pytest.mark.asyncio +async def test_get_user_object_backfill_caches_persisted_email_not_proposed(): + """ + LIT-4710 cache-coherence: even when the null-guarded update succeeds, the + cache must be refreshed from the row the DB actually holds, not this + request's proposed email. A concurrent ordinary user update (not null + guarded) can change the email in the window before the cache write, so + optimistically caching the proposed email would serve a stale value. + """ + cache = UserApiKeyCache() + existing = LiteLLM_UserTable( + user_id="jwt-user-5", user_email=None, user_role="internal_user" + ) + await cache.async_set_cache( + key="jwt-user-5", value=existing, model_type=LiteLLM_UserTable + ) + + persisted_row = LiteLLM_UserTable( + user_id="jwt-user-5", + user_email="admin-edited@example.com", + user_role="internal_user", + ) + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.update_many = AsyncMock(return_value=1) + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock( + return_value=persisted_row + ) + + result = await get_user_object( + user_id="jwt-user-5", + prisma_client=mock_prisma_client, + user_api_key_cache=cache, + user_id_upsert=False, + proxy_logging_obj=None, + user_email="jwt-user-5@example.com", + ) + + assert result is not None + assert result.user_email == "admin-edited@example.com" + + refreshed = await cache.async_get_cache( + key="jwt-user-5", model_type=LiteLLM_UserTable + ) + assert refreshed is not None + assert refreshed.user_email == "admin-edited@example.com" + + @pytest.mark.asyncio async def test_get_user_object_upsert_routes_default_team_to_membership(monkeypatch): """Regression for LIT-4324: a configured default team (list of NewUserRequestTeam diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index affaaa3fbf4..3177fc5ba44 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -1989,6 +1989,231 @@ class TestJWTOAuth2Coexistence: assert result.org_id == "validated-org" assert result.user_email == "validated@example.com" + @pytest.mark.asyncio + async def test_mapped_virtual_key_backfills_and_sets_user_email(self): + """ + Regression (LIT-4710): when a JWT resolves straight to an existing + virtual-key mapping (skipping auth_builder), the token's user_email must + still backfill the resolved user and be set on the returned + UserAPIKeyAuth. Before the fix the mapped path never passed the email + through, so user_api_key_user_email stayed null on every request. + """ + jwt_token = "eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ1c2VyMSJ9.signature" + general_settings = {"enable_jwt_auth": True} + user_api_key_cache = DualCache() + prisma_client = MagicMock() + jwt_handler = MagicMock() + jwt_handler.is_jwt.return_value = True + jwt_handler.auth_jwt = AsyncMock(return_value={"sub": "mapped-user"}) + jwt_handler.get_user_email = MagicMock(return_value="mapped@example.com") + jwt_handler.get_user_id = MagicMock(return_value="mapped-user") + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( + virtual_key_claim_field="sub", + user_email_jwt_field="sub", + virtual_key_mapping_cache_ttl=300, + ) + + mapped_key = UserAPIKeyAuth( + token="hashed-mapped-key", + api_key="hashed-mapped-key", + user_id="mapped-user", + user_email=None, + ) + backfilled_user = LiteLLM_UserTable( + user_id="mapped-user", + user_email="mapped@example.com", + user_role="internal_user", + ) + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.method = "POST" + mock_request.headers = {"authorization": f"Bearer {jwt_token}"} + mock_request.query_params = {} + mock_request.state = SimpleNamespace() + + with ( + patch("litellm.proxy.proxy_server.general_settings", general_settings), + patch("litellm.proxy.proxy_server.premium_user", True), + patch("litellm.proxy.proxy_server.master_key", "sk-master"), + patch("litellm.proxy.proxy_server.prisma_client", prisma_client), + patch("litellm.proxy.proxy_server.user_api_key_cache", user_api_key_cache), + patch("litellm.proxy.proxy_server.proxy_logging_obj", MagicMock()), + patch("litellm.proxy.proxy_server.jwt_handler", jwt_handler), + patch( + "litellm.proxy.auth.user_api_key_auth._resolve_jwt_to_virtual_key", + new_callable=AsyncMock, + return_value=mapped_key, + ), + patch( + "litellm.proxy.auth.user_api_key_auth.get_user_object", + new_callable=AsyncMock, + return_value=backfilled_user, + ) as mock_get_user_object, + ): + result = await _user_api_key_auth_builder( + request=mock_request, + api_key=jwt_token, + azure_api_key_header="", + anthropic_api_key_header=None, + google_ai_studio_api_key_header=None, + azure_apim_header=None, + request_data={"model": "gpt-4o-mini"}, + ) + + assert result.user_id == "mapped-user" + assert result.user_email == "mapped@example.com" + assert ( + mock_get_user_object.call_args_list[0].kwargs["user_email"] + == "mapped@example.com" + ) + + @pytest.mark.asyncio + async def test_mapped_virtual_key_does_not_backfill_mismatched_owner(self): + """ + LIT-4710 security guard: when an admin-created mapping points a JWT at a + virtual key owned by a different user, the JWT principal's email must not + be written onto the mapped key owner's record. Backfill only runs when the + mapped key owner is the JWT principal. + """ + jwt_token = "eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ1c2VyMSJ9.signature" + general_settings = {"enable_jwt_auth": True} + user_api_key_cache = DualCache() + prisma_client = MagicMock() + jwt_handler = MagicMock() + jwt_handler.is_jwt.return_value = True + jwt_handler.auth_jwt = AsyncMock(return_value={"sub": "jwt-principal"}) + jwt_handler.get_user_email = MagicMock(return_value="principal@example.com") + jwt_handler.get_user_id = MagicMock(return_value="jwt-principal") + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( + virtual_key_claim_field="sub", + user_email_jwt_field="sub", + virtual_key_mapping_cache_ttl=300, + ) + + mapped_key = UserAPIKeyAuth( + token="hashed-mapped-key", + api_key="hashed-mapped-key", + user_id="other-owner", + user_email=None, + ) + other_owner = LiteLLM_UserTable( + user_id="other-owner", + user_email=None, + user_role="internal_user", + ) + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.method = "POST" + mock_request.headers = {"authorization": f"Bearer {jwt_token}"} + mock_request.query_params = {} + mock_request.state = SimpleNamespace() + + with ( + patch("litellm.proxy.proxy_server.general_settings", general_settings), + patch("litellm.proxy.proxy_server.premium_user", True), + patch("litellm.proxy.proxy_server.master_key", "sk-master"), + patch("litellm.proxy.proxy_server.prisma_client", prisma_client), + patch("litellm.proxy.proxy_server.user_api_key_cache", user_api_key_cache), + patch("litellm.proxy.proxy_server.proxy_logging_obj", MagicMock()), + patch("litellm.proxy.proxy_server.jwt_handler", jwt_handler), + patch( + "litellm.proxy.auth.user_api_key_auth._resolve_jwt_to_virtual_key", + new_callable=AsyncMock, + return_value=mapped_key, + ), + patch( + "litellm.proxy.auth.user_api_key_auth.get_user_object", + new_callable=AsyncMock, + return_value=other_owner, + ) as mock_get_user_object, + ): + result = await _user_api_key_auth_builder( + request=mock_request, + api_key=jwt_token, + azure_api_key_header="", + anthropic_api_key_header=None, + google_ai_studio_api_key_header=None, + azure_apim_header=None, + request_data={"model": "gpt-4o-mini"}, + ) + + assert result.user_id == "other-owner" + assert result.user_email is None + assert all( + call.kwargs.get("user_email") != "principal@example.com" + for call in mock_get_user_object.call_args_list + ) + + @pytest.mark.asyncio + async def test_mapped_virtual_key_backfill_failure_does_not_break_auth(self): + """ + LIT-4710 resilience: a mapped-key request served from a valid cached key + must still authenticate when the best-effort email backfill cannot reach + the database, retaining null email rather than failing the request. + """ + jwt_token = "eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ1c2VyMSJ9.signature" + general_settings = {"enable_jwt_auth": True} + user_api_key_cache = DualCache() + prisma_client = MagicMock() + jwt_handler = MagicMock() + jwt_handler.is_jwt.return_value = True + jwt_handler.auth_jwt = AsyncMock(return_value={"sub": "mapped-user"}) + jwt_handler.get_user_email = MagicMock(return_value="mapped@example.com") + jwt_handler.get_user_id = MagicMock(return_value="mapped-user") + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( + virtual_key_claim_field="sub", + user_email_jwt_field="sub", + virtual_key_mapping_cache_ttl=300, + ) + + mapped_key = UserAPIKeyAuth( + token="hashed-mapped-key", + api_key="hashed-mapped-key", + user_id="mapped-user", + user_email=None, + ) + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.method = "POST" + mock_request.headers = {"authorization": f"Bearer {jwt_token}"} + mock_request.query_params = {} + mock_request.state = SimpleNamespace() + + with ( + patch("litellm.proxy.proxy_server.general_settings", general_settings), + patch("litellm.proxy.proxy_server.premium_user", True), + patch("litellm.proxy.proxy_server.master_key", "sk-master"), + patch("litellm.proxy.proxy_server.prisma_client", prisma_client), + patch("litellm.proxy.proxy_server.user_api_key_cache", user_api_key_cache), + patch("litellm.proxy.proxy_server.proxy_logging_obj", MagicMock()), + patch("litellm.proxy.proxy_server.jwt_handler", jwt_handler), + patch( + "litellm.proxy.auth.user_api_key_auth._resolve_jwt_to_virtual_key", + new_callable=AsyncMock, + return_value=mapped_key, + ), + patch( + "litellm.proxy.auth.user_api_key_auth.get_user_object", + new_callable=AsyncMock, + side_effect=Exception("can't reach database server"), + ), + ): + result = await _user_api_key_auth_builder( + request=mock_request, + api_key=jwt_token, + azure_api_key_header="", + anthropic_api_key_header=None, + google_ai_studio_api_key_header=None, + azure_apim_header=None, + request_data={"model": "gpt-4o-mini"}, + ) + + assert result.user_id == "mapped-user" + assert result.user_email is None + @pytest.mark.asyncio async def test_routing_override_routes_matching_jwt_to_oauth2(self): """ From 41e44089061d3e03cdc969cd9772d4f36681dd4c Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 3 Aug 2026 12:55:39 -0700 Subject: [PATCH 039/124] feat(playground): add non-streaming response toggle (#35560) Adds a Stream responses checkbox (default on) to the playground Model Settings popover. When unchecked, chat completions and responses API requests are sent with stream: false and the full reply renders at once. The non-streamed result is replayed through the existing streaming handlers as synthesized chunks/events so MCP events, vector store results, usage and response ids behave identically in both modes. TTFT is suppressed when not streaming; total latency now also reported for the responses API. The toggle is scoped to the chat and responses endpoints, persists via sessionStorage, and is isolated from the simplified Agent Builder chat. Resolves LIT-3251 --- .../chat_ui/AdditionalModelSettings.test.tsx | 38 ++++ .../chat_ui/AdditionalModelSettings.tsx | 122 ++++++++----- .../components/chat_ui/ChatUI.test.tsx | 167 ++++++++++++++++++ .../playground/components/chat_ui/ChatUI.tsx | 17 +- .../llm_calls/chat_completion.test.tsx | 137 ++++++++++++++ .../components/llm_calls/chat_completion.tsx | 60 ++++--- .../llm_calls/responses_api.test.tsx | 152 ++++++++++++++++ .../components/llm_calls/responses_api.tsx | 80 +++++++-- 8 files changed, 685 insertions(+), 88 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.test.tsx index d6f29b469b8..1b443e98495 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.test.tsx @@ -48,6 +48,44 @@ describe("AdditionalModelSettings", () => { expect(maxTokensSlider).not.toBeDisabled(); }); + it("should not show Stream responses when onStreamingChange is not provided", () => { + render(); + expect(screen.queryByText(/Stream responses/i)).not.toBeInTheDocument(); + }); + + it("should render Stream responses checked by default and report unchecking it", async () => { + const user = userEvent.setup(); + const onStreamingChange = vi.fn(); + + render(); + + const streamingCheckbox = screen.getByRole("checkbox", { name: /Stream responses/i }); + expect(streamingCheckbox).toBeChecked(); + + await act(async () => { + await user.click(streamingCheckbox); + }); + + await waitFor(() => { + expect(onStreamingChange).toHaveBeenCalledWith(false); + }); + }); + + it("should keep the streaming toggle but drop advanced params when showAdvancedParams is false", () => { + render(); + + expect(screen.getByRole("checkbox", { name: /Stream responses/i })).toBeInTheDocument(); + expect(screen.queryByText("Use Advanced Parameters")).not.toBeInTheDocument(); + expect(screen.queryByText("Temperature")).not.toBeInTheDocument(); + expect(screen.queryByText("Max Tokens")).not.toBeInTheDocument(); + }); + + it("should reflect a disabled streaming setting from props", () => { + render(); + + expect(screen.getByRole("checkbox", { name: /Stream responses/i })).not.toBeChecked(); + }); + it("should not show Simulate failure to test fallbacks when onMockTestFallbacksChange is not provided", () => { render(); expect(screen.queryByText(/Simulate failure to test fallbacks/i)).not.toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.tsx index 078c1b66afb..d4320110c4c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.tsx @@ -12,6 +12,9 @@ interface AdditionalModelSettingsProps { onUseAdvancedParamsChange?: (value: boolean) => void; mockTestFallbacks?: boolean; onMockTestFallbacksChange?: (value: boolean) => void; + streamingEnabled?: boolean; + onStreamingChange?: (value: boolean) => void; + showAdvancedParams?: boolean; } const AdditionalModelSettings: React.FC = ({ @@ -23,6 +26,9 @@ const AdditionalModelSettings: React.FC = ({ onUseAdvancedParamsChange, mockTestFallbacks, onMockTestFallbacksChange, + streamingEnabled = true, + onStreamingChange, + showAdvancedParams = true, }) => { const [internalUseAdvancedParams, setInternalUseAdvancedParams] = useState(false); const useAdvancedParams = @@ -64,9 +70,25 @@ const AdditionalModelSettings: React.FC = ({ return (
- handleUseAdvancedParamsChange(e.target.checked)}> - Use Advanced Parameters - + {onStreamingChange && ( +
+ onStreamingChange(e.target.checked)}> + Stream responses + + + + +
+ )} + + {showAdvancedParams && ( + handleUseAdvancedParamsChange(e.target.checked)}> + Use Advanced Parameters + + )} {onMockTestFallbacksChange && (
@@ -104,72 +126,74 @@ const AdditionalModelSettings: React.FC = ({
)} -
-
-
-
- Temperature - - - + {showAdvancedParams && ( +
+
+
+
+ Temperature + + + +
+
-
- -
-
-
-
- Max Tokens - - - +
+
+
+ Max Tokens + + + +
+
-
-
-
+ )}
); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx index 9da3e3a4a08..b5f2bf7b10c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx @@ -2,12 +2,17 @@ import { act, fireEvent, render, screen, waitFor } from "@testing-library/react" import { beforeEach, describe, expect, it, vi } from "vitest"; import ChatUI from "./ChatUI"; import * as fetchModelsModule from "@/components/llm_calls/fetch_models"; +import { makeOpenAIChatCompletionRequest } from "@/components/llm_calls/chat_completion"; // Mock the fetchAvailableModels function vi.mock("@/components/llm_calls/fetch_models", () => ({ fetchAvailableModels: vi.fn(), })); +vi.mock("@/components/llm_calls/chat_completion", () => ({ + makeOpenAIChatCompletionRequest: vi.fn().mockResolvedValue(undefined), +})); + // Mock other networking functions that cause errors vi.mock("@/components/networking", () => ({ tagListCall: vi.fn().mockResolvedValue({ data: [] }), @@ -21,6 +26,9 @@ beforeEach(() => { Element.prototype.scrollIntoView = () => {}; }); +const CHAT_REQUEST_ARG_COUNT = 26; +const STREAMING_ENABLED_ARG_INDEX = 25; + describe("ChatUI", () => { beforeEach(() => { // Reset mocks before each test @@ -334,6 +342,165 @@ describe("ChatUI", () => { }); }); + it("should send the chat request non-streaming after Stream responses is unchecked", async () => { + render( + , + ); + + await waitFor(() => { + expect(screen.getByText("Test Key")).toBeInTheDocument(); + }); + + const selectModelLabel = screen.getByText("Select Model"); + const modelSelect = selectModelLabel.closest("div")?.querySelector(".ant-select-selector"); + await act(async () => { + fireEvent.mouseDown(modelSelect!); + }); + + await waitFor(() => { + expect(screen.getAllByText("Model 1").length).toBeGreaterThan(0); + }); + + const model1Options = screen.getAllByText("Model 1"); + await act(async () => { + fireEvent.click(model1Options[model1Options.length - 1]); + }); + + await waitFor(() => { + expect(screen.getByTestId("model-settings-button")).toBeInTheDocument(); + }); + + await act(async () => { + fireEvent.click(screen.getByTestId("model-settings-button")); + }); + + const streamingCheckbox = await screen.findByRole("checkbox", { name: /Stream responses/i }); + expect(streamingCheckbox).toBeChecked(); + + await act(async () => { + fireEvent.click(streamingCheckbox); + }); + + await waitFor(() => { + expect(screen.getByRole("checkbox", { name: /Stream responses/i })).not.toBeChecked(); + }); + + const messageInput = screen.getByPlaceholderText("Type your message... (Shift+Enter for new line)"); + await act(async () => { + fireEvent.change(messageInput, { target: { value: "hello" } }); + }); + await act(async () => { + fireEvent.keyDown(messageInput, { key: "Enter", code: "Enter" }); + }); + + await waitFor(() => { + expect(makeOpenAIChatCompletionRequest).toHaveBeenCalledTimes(1); + }); + + const requestArgs = vi.mocked(makeOpenAIChatCompletionRequest).mock.calls[0]; + expect(requestArgs).toHaveLength(CHAT_REQUEST_ARG_COUNT); + expect(requestArgs[STREAMING_ENABLED_ARG_INDEX]).toBe(false); + }); + + it("should force streaming in simplified mode even when the playground setting is off", async () => { + sessionStorage.setItem("streamingEnabled", "false"); + + render( + , + ); + + await waitFor(() => { + expect(screen.getByText("Chat")).toBeInTheDocument(); + }); + + const messageInput = screen.getByPlaceholderText("Type your message... (Shift+Enter for new line)"); + await act(async () => { + fireEvent.change(messageInput, { target: { value: "hello" } }); + }); + await act(async () => { + fireEvent.keyDown(messageInput, { key: "Enter", code: "Enter" }); + }); + + await waitFor(() => { + expect(makeOpenAIChatCompletionRequest).toHaveBeenCalledTimes(1); + }); + + const requestArgs = vi.mocked(makeOpenAIChatCompletionRequest).mock.calls[0]; + expect(requestArgs).toHaveLength(CHAT_REQUEST_ARG_COUNT); + expect(requestArgs[STREAMING_ENABLED_ARG_INDEX]).toBe(true); + expect(sessionStorage.getItem("streamingEnabled")).toBe("false"); + }); + + it("should offer the streaming toggle for a responses-only model without advanced params", async () => { + (fetchModelsModule.fetchAvailableModels as any).mockResolvedValue([ + { model_group: "ResponsesModel", mode: "responses" }, + ]); + + render( + , + ); + + await waitFor(() => { + expect(screen.getByText("Test Key")).toBeInTheDocument(); + }); + + const endpointTypeText = screen.getByText("Endpoint Type"); + const endpointSelect = endpointTypeText.parentElement?.querySelector(".ant-select-selector"); + await act(async () => { + fireEvent.mouseDown(endpointSelect!); + }); + await act(async () => { + fireEvent.click(screen.getByText("/v1/responses")); + }); + + const selectModelLabel = screen.getByText("Select Model"); + const modelSelect = selectModelLabel.closest("div")?.querySelector(".ant-select-selector"); + await act(async () => { + fireEvent.mouseDown(modelSelect!); + }); + + await waitFor(() => { + expect(screen.getAllByText("ResponsesModel").length).toBeGreaterThan(0); + }); + + const modelOptions = screen.getAllByText("ResponsesModel"); + await act(async () => { + fireEvent.click(modelOptions[modelOptions.length - 1]); + }); + + await waitFor(() => { + expect(screen.getByTestId("model-settings-button")).toBeInTheDocument(); + }); + + await act(async () => { + fireEvent.click(screen.getByTestId("model-settings-button")); + }); + + expect(await screen.findByRole("checkbox", { name: /Stream responses/i })).toBeChecked(); + expect(screen.queryByText("Temperature")).not.toBeInTheDocument(); + expect(screen.queryByText("Use Advanced Parameters")).not.toBeInTheDocument(); + }); + it("should show Fill button and populate customProxyBaseUrl when proxySettings.LITELLM_UI_API_DOC_BASE_URL is provided", async () => { const testProxyUrl = "http://localhost:5000"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx index d2cf27e0c8b..684814bfe5b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx @@ -261,6 +261,11 @@ const ChatUI: React.FC = ({ const [maxTokens, setMaxTokens] = useState(2048); const [useAdvancedParams, setUseAdvancedParams] = useState(false); const [mockTestFallbacks, setMockTestFallbacks] = useState(false); + const [streamingEnabled, setStreamingEnabled] = useState(() => { + if (simplified) return true; + const saved = sessionStorage.getItem("streamingEnabled"); + return saved === null ? true : saved === "true"; + }); // Code Interpreter state (using custom hook) const codeInterpreter = useCodeInterpreter(); @@ -372,6 +377,7 @@ const ChatUI: React.FC = ({ sessionStorage.removeItem("selectedMCPTools"); // Clean up old key if (!simplified) { + sessionStorage.setItem("streamingEnabled", JSON.stringify(streamingEnabled)); if (selectedModel) { sessionStorage.setItem("selectedModel", selectedModel); } else { @@ -392,6 +398,7 @@ const ChatUI: React.FC = ({ selectedMCPServers, mcpServerToolRestrictions, selectedVoice, + streamingEnabled, ]); useEffect(() => { @@ -771,6 +778,7 @@ const ChatUI: React.FC = ({ handleMCPEvent, mockTestFallbacks, mcpToolsets, + streamingEnabled, ); } else if (endpointType === EndpointType.IMAGE) { // For image generation @@ -852,6 +860,8 @@ const ChatUI: React.FC = ({ mcpServers, mcpServerToolRestrictions, mcpToolsets, + streamingEnabled, + updateTotalLatency, ); } else if (endpointType === EndpointType.ANTHROPIC_MESSAGES) { const apiChatHistory = [ @@ -1035,6 +1045,8 @@ const ChatUI: React.FC = ({ return !model.mode || model.mode === "chat"; }; + const supportsStreamingToggle = endpointType === EndpointType.CHAT || endpointType === EndpointType.RESPONSES; + const antIcon = ; return ( @@ -1184,10 +1196,11 @@ const ChatUI: React.FC = ({ Select Model - {isChatModel() ? ( + {isChatModel() || supportsStreamingToggle ? ( = ({ onUseAdvancedParamsChange={setUseAdvancedParams} mockTestFallbacks={mockTestFallbacks} onMockTestFallbacksChange={setMockTestFallbacks} + streamingEnabled={streamingEnabled} + onStreamingChange={supportsStreamingToggle ? setStreamingEnabled : undefined} /> } title="Model Settings" diff --git a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx index 8649834b318..10252adecd0 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx @@ -224,6 +224,143 @@ describe("chat_completion", () => { expect(callArgs.mock_testing_fallbacks).toBe(true); }); + it("should send a non-streaming request and render the whole message at once when streaming is disabled", async () => { + mockCreate.mockResolvedValueOnce({ + id: "chatcmpl-1", + object: "chat.completion", + created: 1, + model: "gpt-4", + choices: [ + { + index: 0, + finish_reason: "stop", + message: { role: "assistant", content: "Hello there" }, + }, + ], + usage: { + completion_tokens: 2, + prompt_tokens: 5, + total_tokens: 7, + cost: 0.25, + }, + }); + + const onTimingData = vi.fn(); + const onUsageData = vi.fn(); + const onTotalLatency = vi.fn(); + + await makeOpenAIChatCompletionRequest( + mockChatHistory, + mockUpdateUI, + "gpt-4", + "test-token", + undefined, // tags + undefined, // signal + undefined, // onReasoningContent + onTimingData, + onUsageData, + undefined, // traceId + undefined, // vector_store_ids + undefined, // guardrails + undefined, // policies + undefined, // selectedMCPServers + undefined, // onImageGenerated + undefined, // onSearchResults + undefined, // temperature + undefined, // max_tokens + onTotalLatency, + undefined, // customBaseUrl + undefined, // mcpServers + undefined, // mcpServerToolRestrictions + undefined, // onMCPEvent + undefined, // mockTestFallbacks + undefined, // mcpToolsets + false, // streamingEnabled + ); + + expect(mockCreate).toHaveBeenCalledTimes(1); + const callArgs = mockCreate.mock.calls[0][0]; + expect(callArgs.stream).toBe(false); + expect(callArgs).not.toHaveProperty("stream_options"); + + expect(mockUpdateUI).toHaveBeenCalledTimes(1); + expect(mockUpdateUI).toHaveBeenCalledWith("Hello there", "gpt-4"); + + expect(onUsageData).toHaveBeenCalledWith({ + completionTokens: 2, + promptTokens: 5, + totalTokens: 7, + cost: 0.25, + }); + expect(onTimingData).not.toHaveBeenCalled(); + expect(onTotalLatency).toHaveBeenCalledWith(expect.any(Number)); + }); + + it("should surface reasoning content and MCP metadata from a non-streaming response", async () => { + mockCreate.mockResolvedValueOnce({ + model: "gpt-4", + choices: [ + { + index: 0, + finish_reason: "stop", + message: { + role: "assistant", + content: "done", + reasoning_content: "thinking", + provider_specific_fields: { + mcp_tool_calls: [{ id: "call_1", function: { name: "search_docs", arguments: "{}" } }], + mcp_call_results: [{ tool_call_id: "call_1", result: "found it" }], + }, + }, + }, + ], + }); + + const onReasoningContent = vi.fn(); + const onMCPEvent = vi.fn(); + + await makeOpenAIChatCompletionRequest( + mockChatHistory, + mockUpdateUI, + "gpt-4", + "test-token", + undefined, // tags + undefined, // signal + onReasoningContent, + undefined, // onTimingData + undefined, // onUsageData + undefined, // traceId + undefined, // vector_store_ids + undefined, // guardrails + undefined, // policies + undefined, // selectedMCPServers + undefined, // onImageGenerated + undefined, // onSearchResults + undefined, // temperature + undefined, // max_tokens + undefined, // onTotalLatency + undefined, // customBaseUrl + undefined, // mcpServers + undefined, // mcpServerToolRestrictions + onMCPEvent, + undefined, // mockTestFallbacks + undefined, // mcpToolsets + false, // streamingEnabled + ); + + expect(onReasoningContent).toHaveBeenCalledWith("thinking"); + expect(onMCPEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: "response.output_item.done", + item: expect.objectContaining({ + type: "mcp_call", + name: "search_docs", + output: "found it", + }), + }), + ); + }); + it("should not include mock_testing_fallbacks in request body when mockTestFallbacks is false or undefined", async () => { await makeOpenAIChatCompletionRequest( mockChatHistory, diff --git a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx index 66be2fc7893..c20d758fe91 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx @@ -1,10 +1,26 @@ import openai from "openai"; -import { ChatCompletionMessageParam } from "openai/resources/chat/completions"; +import { ChatCompletion, ChatCompletionChunk, ChatCompletionMessageParam } from "openai/resources/chat/completions"; import { TokenUsage } from "../chat_ui/ResponseMetrics"; import { VectorStoreSearchResponse } from "../chat_ui/types"; import { getProxyBaseUrl } from "@/components/networking"; import { MCPServer, MCPToolset, type MCPEvent } from "@/components/mcp_tools/types"; +const completionAsSingleChunk = (completion: ChatCompletion): ChatCompletionChunk => + ({ + id: completion.id, + object: "chat.completion.chunk", + created: completion.created, + model: completion.model, + usage: completion.usage, + choices: [ + { + index: 0, + finish_reason: completion.choices[0]?.finish_reason ?? null, + delta: completion.choices[0]?.message ?? {}, + }, + ], + }) as unknown as ChatCompletionChunk; + export async function makeOpenAIChatCompletionRequest( chatHistory: { role: string; content: string | any[] }[], updateUI: (chunk: string, model?: string) => void, @@ -31,6 +47,7 @@ export async function makeOpenAIChatCompletionRequest( onMCPEvent?: (event: MCPEvent) => void, mockTestFallbacks?: boolean, mcpToolsets?: MCPToolset[], + streamingEnabled: boolean = true, ) { // base url should be the current base_url const isLocal = process.env.NODE_ENV === "development"; @@ -111,26 +128,25 @@ export async function makeOpenAIChatCompletionRequest( } } - // @ts-ignore - const response = await client.chat.completions.create( - { - model: selectedModel, - stream: true, - stream_options: { - include_usage: true, - }, - litellm_trace_id: traceId, - messages: chatHistory as ChatCompletionMessageParam[], - ...(vector_store_ids ? { vector_store_ids } : {}), - ...(guardrails ? { guardrails } : {}), - ...(policies ? { policies } : {}), - ...(tools.length > 0 ? { tools, tool_choice: "auto" } : {}), - ...(temperature !== undefined ? { temperature } : {}), - ...(max_tokens !== undefined ? { max_tokens } : {}), - ...(mockTestFallbacks ? { mock_testing_fallbacks: true } : {}), - }, - { signal }, - ); + const requestBody = { + model: selectedModel, + litellm_trace_id: traceId, + messages: chatHistory as ChatCompletionMessageParam[], + ...(vector_store_ids ? { vector_store_ids } : {}), + ...(guardrails ? { guardrails } : {}), + ...(policies ? { policies } : {}), + ...(tools.length > 0 ? { tools, tool_choice: "auto" as const } : {}), + ...(temperature !== undefined ? { temperature } : {}), + ...(max_tokens !== undefined ? { max_tokens } : {}), + ...(mockTestFallbacks ? { mock_testing_fallbacks: true } : {}), + }; + + const response: AsyncIterable | ChatCompletionChunk[] = streamingEnabled + ? await client.chat.completions.create( + { ...requestBody, stream: true, stream_options: { include_usage: true } }, + { signal }, + ) + : [completionAsSingleChunk(await client.chat.completions.create({ ...requestBody, stream: false }, { signal }))]; for await (const chunk of response) { // Process content and measure time to first token @@ -142,7 +158,7 @@ export async function makeOpenAIChatCompletionRequest( if (!firstTokenReceived && (chunk.choices[0]?.delta?.content || (delta && delta.reasoning_content))) { firstTokenReceived = true; timeToFirstToken = Date.now() - startTime; - if (onTimingData) { + if (onTimingData && streamingEnabled) { onTimingData(timeToFirstToken); } } diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx index 77ff5fd00bb..a897e6fc4cc 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx @@ -69,6 +69,158 @@ describe("responses_api", () => { expect(mockUpdateTextUI).toHaveBeenCalledWith("assistant", "Hi", "gpt-4"); }); + it("should send a non-streaming request and render the whole output at once when streaming is disabled", async () => { + mockResponsesCreate.mockResolvedValueOnce({ + id: "resp_456", + output: [ + { + type: "message", + content: [ + { type: "output_text", text: "Full " }, + { type: "output_text", text: "answer" }, + ], + }, + ], + usage: { output_tokens: 3, input_tokens: 4, total_tokens: 7 }, + }); + + const onTimingData = vi.fn(); + const onUsageData = vi.fn(); + const onResponseId = vi.fn(); + + await makeOpenAIResponsesRequest( + messages, + mockUpdateTextUI, + "gpt-4", + "test-token", + undefined, // tags + undefined, // signal + undefined, // onReasoningContent + onTimingData, + onUsageData, + undefined, // traceId + undefined, // vector_store_ids + undefined, // guardrails + undefined, // policies + undefined, // selectedMCPServers + undefined, // previousResponseId + onResponseId, + undefined, // onMCPEvent + undefined, // codeInterpreterEnabled + undefined, // onCodeInterpreterResult + undefined, // customBaseUrl + undefined, // mcpServers + undefined, // mcpServerToolRestrictions + undefined, // mcpToolsets + false, // streamingEnabled + ); + + expect(mockResponsesCreate).toHaveBeenCalledTimes(1); + expect(mockResponsesCreate.mock.calls[0][0].stream).toBe(false); + + expect(mockUpdateTextUI).toHaveBeenCalledTimes(1); + expect(mockUpdateTextUI).toHaveBeenCalledWith("assistant", "Full answer", "gpt-4"); + + expect(onUsageData).toHaveBeenCalledWith({ completionTokens: 3, promptTokens: 4, totalTokens: 7 }, ""); + expect(onResponseId).toHaveBeenCalledWith("resp_456"); + expect(onTimingData).not.toHaveBeenCalled(); + }); + + it("should report total latency in both streaming and non-streaming modes", async () => { + const onTotalLatency = vi.fn(); + const callWithStreaming = (streamingEnabled: boolean) => + makeOpenAIResponsesRequest( + messages, + mockUpdateTextUI, + "gpt-4", + "test-token", + undefined, // tags + undefined, // signal + undefined, // onReasoningContent + undefined, // onTimingData + undefined, // onUsageData + undefined, // traceId + undefined, // vector_store_ids + undefined, // guardrails + undefined, // policies + undefined, // selectedMCPServers + undefined, // previousResponseId + undefined, // onResponseId + undefined, // onMCPEvent + undefined, // codeInterpreterEnabled + undefined, // onCodeInterpreterResult + undefined, // customBaseUrl + undefined, // mcpServers + undefined, // mcpServerToolRestrictions + undefined, // mcpToolsets + streamingEnabled, + onTotalLatency, + ); + + await callWithStreaming(true); + expect(onTotalLatency).toHaveBeenCalledTimes(1); + expect(onTotalLatency).toHaveBeenLastCalledWith(expect.any(Number)); + + mockResponsesCreate.mockResolvedValueOnce({ + id: "resp_latency", + output: [{ type: "message", content: [{ type: "output_text", text: "Answer" }] }], + }); + + await callWithStreaming(false); + expect(onTotalLatency).toHaveBeenCalledTimes(2); + expect(onTotalLatency).toHaveBeenLastCalledWith(expect.any(Number)); + }); + + it("should replay MCP output items as events for a non-streaming response", async () => { + mockResponsesCreate.mockResolvedValueOnce({ + id: "resp_789", + output: [ + { type: "mcp_call", id: "mcp_1", name: "search_docs", arguments: "{}", output: "found it" }, + { type: "message", content: [{ type: "output_text", text: "Answer" }] }, + ], + usage: { output_tokens: 1, input_tokens: 1, total_tokens: 2 }, + }); + + const onMCPEvent = vi.fn(); + const onUsageData = vi.fn(); + + await makeOpenAIResponsesRequest( + messages, + mockUpdateTextUI, + "gpt-4", + "test-token", + undefined, // tags + undefined, // signal + undefined, // onReasoningContent + undefined, // onTimingData + onUsageData, + undefined, // traceId + undefined, // vector_store_ids + undefined, // guardrails + undefined, // policies + undefined, // selectedMCPServers + undefined, // previousResponseId + undefined, // onResponseId + onMCPEvent, + undefined, // codeInterpreterEnabled + undefined, // onCodeInterpreterResult + undefined, // customBaseUrl + undefined, // mcpServers + undefined, // mcpServerToolRestrictions + undefined, // mcpToolsets + false, // streamingEnabled + ); + + expect(onMCPEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: "response.output_item.done", + item_id: "mcp_1", + item: expect.objectContaining({ type: "mcp_call", name: "search_docs", output: "found it" }), + }), + ); + expect(onUsageData).toHaveBeenCalledWith(expect.anything(), "search_docs"); + }); + it("should configure MCP tools per server with restrictions", async () => { const selectedMCPServers = ["server-1", "server-2"]; const mcpServers = [ diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx index ef510d86b94..f356b2cb2c3 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx @@ -14,6 +14,49 @@ import { export type { CodeInterpreterResult } from "./code_interpreter_handler"; +interface ResponseOutputPart { + type?: string; + text?: string; +} + +interface ResponseOutputItem { + type?: string; + content?: ResponseOutputPart[]; + summary?: ResponseOutputPart[]; +} + +interface NonStreamedResponse { + output?: ResponseOutputItem[]; +} + +type SynthesizedResponseEvent = + | { type: "response.output_item.done"; item: ResponseOutputItem } + | { type: "response.reasoning.delta"; delta: string } + | { type: "response.output_text.delta"; delta: string } + | { type: "response.completed"; response: NonStreamedResponse }; + +const responseAsEvents = (response: NonStreamedResponse): SynthesizedResponseEvent[] => { + const outputItems = response.output ?? []; + const outputText = outputItems + .filter((item) => item.type === "message") + .flatMap((item) => item.content ?? []) + .filter((part) => part.type === "output_text") + .map((part) => part.text ?? "") + .join(""); + const reasoningText = outputItems + .filter((item) => item.type === "reasoning") + .flatMap((item) => item.summary ?? []) + .map((part) => part.text ?? "") + .join(""); + + return [ + ...outputItems.map((item) => ({ type: "response.output_item.done" as const, item })), + ...(reasoningText ? [{ type: "response.reasoning.delta" as const, delta: reasoningText }] : []), + ...(outputText ? [{ type: "response.output_text.delta" as const, delta: outputText }] : []), + { type: "response.completed" as const, response }, + ]; +}; + export async function makeOpenAIResponsesRequest( messages: MessageType[], updateTextUI: (role: string, delta: string, model?: string) => void, @@ -38,6 +81,8 @@ export async function makeOpenAIResponsesRequest( mcpServers?: MCPServer[], mcpServerToolRestrictions?: Record, mcpToolsets?: MCPToolset[], + streamingEnabled: boolean = true, + onTotalLatency?: (latency: number) => void, ) { if (!accessToken) { throw new Error("Virtual Key is required"); @@ -143,27 +188,26 @@ export async function makeOpenAIResponsesRequest( }); } + const requestBody = { + model: selectedModel, + input: formattedInput, + litellm_trace_id: traceId, + ...(previousResponseId ? { previous_response_id: previousResponseId } : {}), + ...(vector_store_ids ? { vector_store_ids } : {}), + ...(guardrails ? { guardrails } : {}), + ...(policies ? { policies } : {}), + ...(tools.length > 0 ? { tools, tool_choice: "auto" } : {}), + }; + // Create request to OpenAI responses API // Use 'any' type to avoid TypeScript issues with the experimental API - const response = await (client as any).responses.create( - { - model: selectedModel, - input: formattedInput, - stream: true, - litellm_trace_id: traceId, - ...(previousResponseId ? { previous_response_id: previousResponseId } : {}), - ...(vector_store_ids ? { vector_store_ids } : {}), - ...(guardrails ? { guardrails } : {}), - ...(policies ? { policies } : {}), - ...(tools.length > 0 ? { tools, tool_choice: "auto" } : {}), - }, - { signal }, - ); + const response = await (client as any).responses.create({ ...requestBody, stream: streamingEnabled }, { signal }); + const events = streamingEnabled ? response : responseAsEvents(response); let mcpToolUsed = ""; let codeInterpreterState: CodeInterpreterState = { code: "", containerId: "" }; - for await (const event of response) { + for await (const event of events) { // Use a type-safe approach to handle events if (typeof event === "object" && event !== null) { // Handle MCP events first @@ -215,7 +259,7 @@ export async function makeOpenAIResponsesRequest( firstTokenReceived = true; const timeToFirstToken = Date.now() - startTime; - if (onTimingData) { + if (onTimingData && streamingEnabled) { onTimingData(timeToFirstToken); } } @@ -259,6 +303,10 @@ export async function makeOpenAIResponsesRequest( } } + if (onTotalLatency) { + onTotalLatency(Date.now() - startTime); + } + return response; } catch (error) { if (signal?.aborted) { From 46b6eae799b8ee6fb7b63d6007876aaa77971827 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 3 Aug 2026 12:57:12 -0700 Subject: [PATCH 040/124] feat(teams): apply default organization to new teams from default team settings (#35540) * feat(teams): apply default organization to new teams from default team settings Adds organization_id to DefaultTeamSSOParams so proxy admins can pick a default organization in Default Team Settings. new_team applies it before org validation whenever a team is created without an explicit organization_id, so API, Admin UI, SCIM, SSO, and team upsert creations all inherit it and go through the same existence and org-limit checks. Explicit organization selections win and existing teams are untouched. The default is validated at save time (PATCH /update/default_team_settings returns 400 for an unknown org) and at create time, where a missing org now surfaces as a clean 400 instead of a 500 by routing OrganizationNotFoundError into the previously dead org_table None guard. The Admin UI Default Team Settings tab gets a Default Organization row backed by the shared OrganizationDropdown. * fix(teams): validate org limits against final team state including defaults Applies default_team_params and the legacy max_budget fallback before the organization validation block, so _check_org_team_limits sees the values the team will actually be persisted with. Also loads the org's budget table in the lookup; without include_budget_table every budget comparison in _check_org_team_limits was skipped because litellm_budget_table was None. * test(proxy_behavior): pin org team limits as enforced on /team/new The dead-code pins existed to turn red when include_budget_table went live; that happened, so the scenarios now assert the 400 rejections plus within-cap acceptance, and the unknown-org pin asserts the handler's 400 instead of the surfaced 500. --- .../management_endpoints/team_endpoints.py | 46 ++-- .../proxy_setting_endpoints.py | 34 +++ .../proxy/management_endpoints/ui_sso.py | 4 + .../management/test_team_budget_limits.py | 79 +++--- .../management/test_team_new.py | 29 +- .../test_team_default_params.py | 259 +++++++++++++----- .../proxy/management_endpoints/test_ui_sso.py | 49 ++++ .../test_proxy_setting_endpoints.py | 88 ++++++ .../src/components/TeamSSOSettings.test.tsx | 172 +++++++++++- .../src/components/TeamSSOSettings.tsx | 35 ++- .../OrganizationDropdown.tsx | 4 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 + 12 files changed, 656 insertions(+), 148 deletions(-) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 54ef697d16e..4dd86e5769d 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -76,6 +76,7 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.auth_checks import ( + OrganizationNotFoundError, _cache_team_object, allowed_route_check_inside_route, can_org_access_model, @@ -1210,24 +1211,10 @@ async def new_team( detail={"error": f"Team id = {data.team_id} already exists. Please use a different team id."}, ) - # check org key limits - done here to handle inheriting org id from team - if data.organization_id is not None and prisma_client is not None: - org_table = await get_org_object( - org_id=data.organization_id, - user_api_key_cache=user_api_key_cache, - prisma_client=prisma_client, - ) - if org_table is None: - raise HTTPException( - status_code=400, - detail=f"Organization not found for organization_id={data.organization_id}", - ) - - await _check_org_team_limits( - org_table=org_table, - data=data, - prisma_client=prisma_client, - ) + if data.organization_id is None: + default_organization_id = _get_default_team_param("organization_id") + if isinstance(default_organization_id, str): + data.organization_id = default_organization_id # Apply defaults from litellm.default_team_params for any fields # not explicitly provided in the request. @@ -1255,6 +1242,29 @@ async def new_team( if default_budget is not None: data.max_budget = default_budget + # check org key limits - done here to handle inheriting org id from team + if data.organization_id is not None and prisma_client is not None: + try: + org_table = await get_org_object( + org_id=data.organization_id, + user_api_key_cache=user_api_key_cache, + prisma_client=prisma_client, + include_budget_table=True, + ) + except OrganizationNotFoundError: + org_table = None + if org_table is None: + raise HTTPException( + status_code=400, + detail=f"Organization not found for organization_id={data.organization_id}", + ) + + await _check_org_team_limits( + org_table=org_table, + data=data, + prisma_client=prisma_client, + ) + if ( user_api_key_dict.user_role is None or user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN ): # don't restrict proxy admin diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 60c88c0c371..8ed848ac1bf 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -23,6 +23,7 @@ from litellm.proxy.config_resolvers.sso import ( ) from litellm.proxy.utils import invalidate_config_param from litellm.repositories.config_repository import ConfigRepository +from litellm.repositories.organization_repository import OrganizationRepository from litellm.repositories.table_repositories import ( SSOConfigRepository, UISettingsRepository, @@ -636,6 +637,36 @@ async def _validate_default_teams_exist(teams: list[str] | list[NewUserRequestTe ) +async def _validate_default_organization_exists(organization_id: str) -> None: + """Reject a default organization that cannot be assigned. + + Teams are created from these settings long after they are saved, and an unknown + organization id would fail every future team creation instead of here, where the + admin who typed it can still fix it. + """ + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={ # mutable-ok: HTTPException detail must be a plain dict for FastAPI JSON serialization + "error": "Database not connected. Please connect a database." + }, + ) + + organization_exists = await OrganizationRepository(prisma_client).exists( + organization_id, id_field="organization_id" + ) + if not organization_exists: + raise HTTPException( + status_code=400, + detail={ # mutable-ok: HTTPException detail must be a plain dict for FastAPI JSON serialization + "error": f"Organization not found: {organization_id}. " + "An organization must exist before it can be set as the default organization for new teams." + }, + ) + + async def update_default_team_member_budget(teams: list[NewUserRequestTeam], user_api_key_dict: UserAPIKeyAuth): """ 1. Update the max member budget for the team @@ -774,6 +805,9 @@ async def update_default_team_settings( Update the default team parameters for SSO users. These settings will be applied to new teams created from SSO. """ + if settings.organization_id is not None: + await _validate_default_organization_exists(settings.organization_id) + return await _update_litellm_setting( settings=settings, settings_key="default_team_params", diff --git a/litellm/types/proxy/management_endpoints/ui_sso.py b/litellm/types/proxy/management_endpoints/ui_sso.py index d4b1d98f957..f68d818d991 100644 --- a/litellm/types/proxy/management_endpoints/ui_sso.py +++ b/litellm/types/proxy/management_endpoints/ui_sso.py @@ -229,3 +229,7 @@ class DefaultTeamSSOParams(LiteLLMPydanticObjectBase): default=None, description="Default permissions granted to members of newly created teams (e.g. /key/generate, /key/update, /key/delete). /key/info and /key/health are always included.", ) + organization_id: str | None = Field( + default=None, + description="Default organization for new teams created without an explicit organization", + ) diff --git a/tests/proxy_behavior/management/test_team_budget_limits.py b/tests/proxy_behavior/management/test_team_budget_limits.py index dad775370ad..96a6fe7234a 100644 --- a/tests/proxy_behavior/management/test_team_budget_limits.py +++ b/tests/proxy_behavior/management/test_team_budget_limits.py @@ -10,14 +10,14 @@ Pins the five helpers Driven through /team/new + /team/update. -Structural finding pinned here, identical in shape to F1's org aggregate: -both call sites (lines 985 + 1751) load the org via `get_org_object` -WITHOUT `include_budget_table=True`, so `org_table.litellm_budget_table` -is `None` and the org max_budget / org tpm / org rpm guards inside -`_check_org_team_limits` (lines 641–694, 670–694) silently no-op. The -`models` subset guard (lines 654–667) IS reachable because it reads -`org_table.models` directly. The `_check_user_team_limits` guards reach -all branches through `user_api_key_dict`, no relation include needed. +Structural finding, updated: /team/new loads the org via `get_org_object` +WITH `include_budget_table=True`, so the org max_budget / org tpm / org rpm +guards inside `_check_org_team_limits` are live there and are pinned as +enforced below. /team/update still loads the org without the budget +relation, so its budget guards remain no-ops. The `models` subset guard IS +reachable on both because it reads `org_table.models` directly. The +`_check_user_team_limits` guards reach all branches through +`user_api_key_dict`, no relation include needed. """ import uuid @@ -132,48 +132,67 @@ async def test_check_org_team_limits_models_subset( headers={"Authorization": f"Bearer {seeder}"}, json=body, ) - assert ( - resp.status_code == expected_status - ), f"{body!r} → {resp.status_code}: {resp.text}" + assert resp.status_code == expected_status, f"{body!r} → {resp.status_code}: {resp.text}" rows = await prisma.db.litellm_teamtable.find_many(where={"team_id": team_id}) assert len(rows) == (1 if expected_status == 200 else 0) # --------------------------------------------------------------------------- -# _check_org_team_limits — budget / tpm / rpm structurally unreachable -# (org_table.litellm_budget_table is None at guard time). Pin the -# no-op behavior so a future change that flips include_budget_table=True -# turns these into reds. +# _check_org_team_limits — budget / tpm / rpm live on /team/new since its +# get_org_object call passes include_budget_table=True. (/team/update still +# loads the org without the budget relation, so its guards remain no-ops.) # --------------------------------------------------------------------------- -_ORG_BUDGET_DEAD_SCENARIOS = [ +_ORG_BUDGET_ENFORCED_SCENARIOS = [ ( - "org_budget/over_max_budget_unenforced", + "org_budget/over_max_budget_rejected", {"max_budget": 100, "tpm_limit": None, "rpm_limit": None}, {"max_budget": 999_999}, + 400, ), ( - "org_tpm/over_unenforced", + "org_budget/within_max_budget_accepted", + {"max_budget": 100, "tpm_limit": None, "rpm_limit": None}, + {"max_budget": 50}, + 200, + ), + ( + "org_tpm/over_rejected", {"max_budget": None, "tpm_limit": 100, "rpm_limit": None}, {"tpm_limit": 999_999}, + 400, ), ( - "org_rpm/over_unenforced", + "org_tpm/within_accepted", + {"max_budget": None, "tpm_limit": 100, "rpm_limit": None}, + {"tpm_limit": 50}, + 200, + ), + ( + "org_rpm/over_rejected", {"max_budget": None, "tpm_limit": None, "rpm_limit": 100}, {"rpm_limit": 999_999}, + 400, + ), + ( + "org_rpm/within_accepted", + {"max_budget": None, "tpm_limit": None, "rpm_limit": 100}, + {"rpm_limit": 50}, + 200, ), ] @pytest.mark.parametrize( - "org_budget,body_extras", - [(b, c) for (_id, b, c) in _ORG_BUDGET_DEAD_SCENARIOS], - ids=[s[0] for s in _ORG_BUDGET_DEAD_SCENARIOS], + "org_budget,body_extras,expected_status", + [(b, c, d) for (_id, b, c, d) in _ORG_BUDGET_ENFORCED_SCENARIOS], + ids=[s[0] for s in _ORG_BUDGET_ENFORCED_SCENARIOS], ) -async def test_check_org_team_limits_budget_dead_code_pin( +async def test_check_org_team_limits_budget_enforced( org_budget, body_extras: Dict[str, Any], + expected_status: int, proxy_client, prisma, scratch, @@ -192,9 +211,9 @@ async def test_check_org_team_limits_budget_dead_code_pin( **body_extras, }, ) - assert resp.status_code == 200, resp.text + assert resp.status_code == expected_status, f"{body_extras!r} → {resp.status_code}: {resp.text}" rows = await prisma.db.litellm_teamtable.find_many(where={"team_id": team_id}) - assert len(rows) == 1 + assert len(rows) == (1 if expected_status == 200 else 0) # --------------------------------------------------------------------------- @@ -279,9 +298,9 @@ async def test_check_user_team_limits( **body_extras, }, ) - assert ( - resp.status_code == expected_status - ), f"caps={actor_caps} body={body_extras} → {resp.status_code}: {resp.text}" + assert resp.status_code == expected_status, ( + f"caps={actor_caps} body={body_extras} → {resp.status_code}: {resp.text}" + ) rows = await prisma.db.litellm_teamtable.find_many(where={"team_id": team_id}) assert len(rows) == (1 if expected_status == 200 else 0) @@ -376,9 +395,7 @@ async def test_proxy_admin_raise_budget_allowed(proxy_client, prisma, scratch): async def test_team_admin_remove_budget_cap_blocked(proxy_client, prisma, scratch): """A team admin cannot strip the team's cap (max_budget=null); removing the ceiling is the strongest possible raise -> proxy-admin only.""" - caller_cleartext = await _seed_scratch_actor_with_caps( - prisma, scratch.prefix, max_budget=100000.0 - ) + caller_cleartext = await _seed_scratch_actor_with_caps(prisma, scratch.prefix, max_budget=100000.0) team_id = await create_scratch_team( prisma, team_id=scratch.tag("team"), diff --git a/tests/proxy_behavior/management/test_team_new.py b/tests/proxy_behavior/management/test_team_new.py index 7b07f259641..9846566d0b9 100644 --- a/tests/proxy_behavior/management/test_team_new.py +++ b/tests/proxy_behavior/management/test_team_new.py @@ -72,13 +72,9 @@ async def test_team_new_authz_matrix( headers={"Authorization": f"Bearer {caller.cleartext}"}, json=body, ) - assert ( - resp.status_code == expected_status - ), f"{actor.value} org={org_target}: {resp.status_code} {resp.text}" + assert resp.status_code == expected_status, f"{actor.value} org={org_target}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} - ) + row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) if expected_status == 200: assert row is not None assert row.organization_id == org_id @@ -94,9 +90,7 @@ async def test_team_new_rejects_negative_budget(proxy_client, prisma, scratch, w json={"team_id": scratch.prefix, "max_budget": -1}, ) assert resp.status_code == 400, resp.text - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} - ) + row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) assert row is None @@ -118,12 +112,10 @@ async def test_team_new_rejects_duplicate_team_id(proxy_client, prisma, scratch, assert second.status_code == 400, second.text -async def test_team_new_unknown_organization_is_500( - proxy_client, prisma, scratch, world -): - """SURFACED, NOT ENDORSED: a /team/new with an organization_id that does - not exist currently fails 500 (the role-resolution layer raises before - the handler's own 400 'Organization not found' check is reached).""" +async def test_team_new_unknown_organization_is_400(proxy_client, prisma, scratch, world): + """A /team/new with an organization_id that does not exist fails 400: + OrganizationNotFoundError is routed into the handler's own + 'Organization not found' guard instead of escaping as a 500.""" resp = await proxy_client.post( "/team/new", headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, @@ -132,8 +124,7 @@ async def test_team_new_unknown_organization_is_500( "organization_id": scratch.tag("no-such-org"), }, ) - assert resp.status_code == 500, resp.text - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} - ) + assert resp.status_code == 400, resp.text + assert "Organization not found" in resp.text + row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) assert row is None diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py b/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py index e0b90332ca0..a485d95db06 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py @@ -10,13 +10,14 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException -sys.path.insert( - 0, os.path.abspath("../../../") -) # Adds the parent directory to the system path +sys.path.insert(0, os.path.abspath("../../../")) # Adds the parent directory to the system path import litellm from litellm.proxy._types import ( + LiteLLM_BudgetTable, + LiteLLM_OrganizationTable, NewTeamRequest, + ProxyException, UserAPIKeyAuth, LitellmUserRoles, ) @@ -76,9 +77,7 @@ class TestConfigFieldsDefaultTeamParams: db_param_value=db_settings, ) - assert result["litellm_settings"]["default_team_params"] == { - "max_budget": 100.0 - } + assert result["litellm_settings"]["default_team_params"] == {"max_budget": 100.0} # Existing keys preserved assert result["litellm_settings"]["cache"] is False @@ -172,6 +171,22 @@ class TestNewTeamDefaultParamsApplied: user_role=LitellmUserRoles.PROXY_ADMIN, ) + def _make_org(self, organization_id: str, max_budget: float | None = None) -> LiteLLM_OrganizationTable: + return LiteLLM_OrganizationTable( + organization_id=organization_id, + budget_id="budget-id", + created_by="admin-user", + updated_by="admin-user", + litellm_budget_table=None if max_budget is None else LiteLLM_BudgetTable(max_budget=max_budget), + ) + + def _patch_org_lookup(self, monkeypatch, **mock_kwargs) -> AsyncMock: + from litellm.proxy.management_endpoints import team_endpoints + + lookup = AsyncMock(**mock_kwargs) + monkeypatch.setattr(team_endpoints, "get_org_object", lookup) + return lookup + @pytest.mark.asyncio async def test_all_defaults_applied_when_not_provided(self, monkeypatch): """When no budget/rate/permission fields are in the request, all defaults apply.""" @@ -312,6 +327,7 @@ class TestNewTeamDefaultParamsApplied: assert data.tpm_limit is None assert data.rpm_limit is None assert data.team_member_permissions is None + assert data.organization_id is None @pytest.mark.asyncio async def test_legacy_default_team_settings_fallback(self, monkeypatch): @@ -370,6 +386,144 @@ class TestNewTeamDefaultParamsApplied: # default_team_params wins (100.0), legacy fallback (999.0) not used assert data.max_budget == 100.0 + @pytest.mark.asyncio + async def test_default_organization_applied_and_validated(self, monkeypatch): + """The default org must land before the org-validation block, so a defaulted + org goes through the same existence + org-limit checks as an explicit one.""" + from litellm.proxy.management_endpoints.team_endpoints import new_team + + monkeypatch.setattr(litellm, "default_team_params", {"organization_id": "default-org"}) + org_lookup = self._patch_org_lookup(monkeypatch, return_value=self._make_org("default-org")) + + data = NewTeamRequest(team_alias="my-team") + + try: + await new_team( + data=data, + user_api_key_dict=self._make_admin_auth(), + http_request=MagicMock(), + ) + except Exception: + pass + + assert data.organization_id == "default-org" + org_lookup.assert_awaited_once() + assert org_lookup.await_args.kwargs["org_id"] == "default-org" + + @pytest.mark.asyncio + async def test_explicit_organization_wins_over_default(self, monkeypatch): + """An organization_id in the request must not be replaced by the default.""" + from litellm.proxy.management_endpoints.team_endpoints import new_team + + monkeypatch.setattr(litellm, "default_team_params", {"organization_id": "default-org"}) + org_lookup = self._patch_org_lookup(monkeypatch, return_value=self._make_org("explicit-org")) + + data = NewTeamRequest(team_alias="my-team", organization_id="explicit-org") + + try: + await new_team( + data=data, + user_api_key_dict=self._make_admin_auth(), + http_request=MagicMock(), + ) + except Exception: + pass + + assert data.organization_id == "explicit-org" + assert org_lookup.await_args.kwargs["org_id"] == "explicit-org" + + @pytest.mark.asyncio + async def test_nonexistent_default_organization_returns_400(self, monkeypatch): + """get_org_object raises instead of returning None, so an org that no longer + exists surfaced as a 500; team creation must report a 400 instead.""" + from litellm.proxy.auth.auth_checks import OrganizationNotFoundError + from litellm.proxy.management_endpoints.team_endpoints import new_team + + monkeypatch.setattr(litellm, "default_team_params", {"organization_id": "deleted-org"}) + self._patch_org_lookup( + monkeypatch, + side_effect=OrganizationNotFoundError("Organization doesn't exist in db. Organization=deleted-org"), + ) + + with pytest.raises(ProxyException) as exc_info: + await new_team( + data=NewTeamRequest(team_alias="my-team"), + user_api_key_dict=self._make_admin_auth(), + http_request=MagicMock(), + ) + + assert exc_info.value.code == "400" + assert "deleted-org" in exc_info.value.message + + @pytest.mark.asyncio + async def test_defaulted_max_budget_validated_against_org_budget(self, monkeypatch): + """Defaults must be applied BEFORE _check_org_team_limits runs, or a default + max_budget above the org's cap is persisted unchecked.""" + from litellm.proxy.management_endpoints.team_endpoints import new_team + + monkeypatch.setattr( + litellm, + "default_team_params", + {"organization_id": "capped-org", "max_budget": 500.0}, + ) + self._patch_org_lookup(monkeypatch, return_value=self._make_org("capped-org", max_budget=100.0)) + + with pytest.raises(ProxyException) as exc_info: + await new_team( + data=NewTeamRequest(team_alias="my-team"), + user_api_key_dict=self._make_admin_auth(), + http_request=MagicMock(), + ) + + assert exc_info.value.code == "400" + assert "exceeds organization's max_budget" in exc_info.value.message + + @pytest.mark.asyncio + async def test_explicit_budget_validated_against_default_org_budget(self, monkeypatch): + """The org lookup must load the budget table (include_budget_table=True); + without it litellm_budget_table is None and every budget comparison is skipped.""" + from litellm.proxy.management_endpoints.team_endpoints import new_team + + monkeypatch.setattr(litellm, "default_team_params", {"organization_id": "capped-org"}) + org_lookup = self._patch_org_lookup(monkeypatch, return_value=self._make_org("capped-org", max_budget=100.0)) + + with pytest.raises(ProxyException) as exc_info: + await new_team( + data=NewTeamRequest(team_alias="my-team", max_budget=500.0), + user_api_key_dict=self._make_admin_auth(), + http_request=MagicMock(), + ) + + assert exc_info.value.code == "400" + assert "exceeds organization's max_budget" in exc_info.value.message + assert org_lookup.await_args.kwargs["include_budget_table"] is True + + @pytest.mark.asyncio + async def test_defaults_within_org_budget_still_created(self, monkeypatch): + """A default budget under the org cap must not be rejected by the reordered check.""" + from litellm.proxy.management_endpoints.team_endpoints import new_team + + monkeypatch.setattr( + litellm, + "default_team_params", + {"organization_id": "capped-org", "max_budget": 50.0}, + ) + self._patch_org_lookup(monkeypatch, return_value=self._make_org("capped-org", max_budget=100.0)) + + data = NewTeamRequest(team_alias="my-team") + + try: + await new_team( + data=data, + user_api_key_dict=self._make_admin_auth(), + http_request=MagicMock(), + ) + except Exception: + pass + + assert data.organization_id == "capped-org" + assert data.max_budget == 50.0 + # --------------------------------------------------------------------------- # _update_litellm_setting: setattr ordering @@ -536,18 +690,12 @@ class TestBulkUpdateTeamMemberPermissions: mock_batcher.commit = AsyncMock(return_value=None) mock_prisma = MagicMock() - mock_prisma.db.litellm_teamtable.find_many = AsyncMock( - return_value=[team_a, team_b] - ) + mock_prisma.db.litellm_teamtable.find_many = AsyncMock(return_value=[team_a, team_b]) mock_prisma.db.batch_ = MagicMock(return_value=mock_batcher) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) - data = BulkUpdateTeamMemberPermissionsRequest( - permissions=["/team/daily/activity"], apply_to_all_teams=True - ) - result = await bulk_update_team_member_permissions( - data=data, user_api_key_dict=self._admin_key_dict() - ) + data = BulkUpdateTeamMemberPermissionsRequest(permissions=["/team/daily/activity"], apply_to_all_teams=True) + result = await bulk_update_team_member_permissions(data=data, user_api_key_dict=self._admin_key_dict()) assert result["teams_updated"] == 2 calls = mock_batcher.litellm_teamtable.update.call_args_list @@ -555,19 +703,14 @@ class TestBulkUpdateTeamMemberPermissions: team_a_call = [c for c in calls if c.kwargs["where"]["team_id"] == "team-a"][0] assert "/key/generate" in team_a_call.kwargs["data"]["team_member_permissions"] - assert ( - "/team/daily/activity" - in team_a_call.kwargs["data"]["team_member_permissions"] - ) + assert "/team/daily/activity" in team_a_call.kwargs["data"]["team_member_permissions"] team_b_call = [c for c in calls if c.kwargs["where"]["team_id"] == "team-b"][0] assert "/key/delete" in team_b_call.kwargs["data"]["team_member_permissions"] assert "/key/update" in team_b_call.kwargs["data"]["team_member_permissions"] @pytest.mark.asyncio - async def test_all_teams_skips_teams_that_already_have_permission( - self, monkeypatch - ): + async def test_all_teams_skips_teams_that_already_have_permission(self, monkeypatch): """apply_to_all_teams: teams that already have the permission are skipped.""" from litellm.proxy.management_endpoints.team_endpoints import ( bulk_update_team_member_permissions, @@ -583,18 +726,12 @@ class TestBulkUpdateTeamMemberPermissions: mock_batcher.commit = AsyncMock(return_value=None) mock_prisma = MagicMock() - mock_prisma.db.litellm_teamtable.find_many = AsyncMock( - return_value=[team_has, team_missing] - ) + mock_prisma.db.litellm_teamtable.find_many = AsyncMock(return_value=[team_has, team_missing]) mock_prisma.db.batch_ = MagicMock(return_value=mock_batcher) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) - data = BulkUpdateTeamMemberPermissionsRequest( - permissions=["/team/daily/activity"], apply_to_all_teams=True - ) - result = await bulk_update_team_member_permissions( - data=data, user_api_key_dict=self._admin_key_dict() - ) + data = BulkUpdateTeamMemberPermissionsRequest(permissions=["/team/daily/activity"], apply_to_all_teams=True) + result = await bulk_update_team_member_permissions(data=data, user_api_key_dict=self._admin_key_dict()) assert result["teams_updated"] == 1 calls = mock_batcher.litellm_teamtable.update.call_args_list @@ -618,18 +755,12 @@ class TestBulkUpdateTeamMemberPermissions: mock_batcher.commit = AsyncMock(return_value=None) mock_prisma = MagicMock() - mock_prisma.db.litellm_teamtable.find_many = AsyncMock( - side_effect=[page1, page2] - ) + mock_prisma.db.litellm_teamtable.find_many = AsyncMock(side_effect=[page1, page2]) mock_prisma.db.batch_ = MagicMock(return_value=mock_batcher) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) - data = BulkUpdateTeamMemberPermissionsRequest( - permissions=["/team/daily/activity"], apply_to_all_teams=True - ) - result = await bulk_update_team_member_permissions( - data=data, user_api_key_dict=self._admin_key_dict() - ) + data = BulkUpdateTeamMemberPermissionsRequest(permissions=["/team/daily/activity"], apply_to_all_teams=True) + result = await bulk_update_team_member_permissions(data=data, user_api_key_dict=self._admin_key_dict()) assert result["teams_updated"] == 502 find_calls = mock_prisma.db.litellm_teamtable.find_many.call_args_list @@ -656,18 +787,14 @@ class TestBulkUpdateTeamMemberPermissions: mock_batcher.commit = AsyncMock(return_value=None) mock_prisma = MagicMock() - mock_prisma.db.litellm_teamtable.find_many = AsyncMock( - return_value=[team_a, team_b] - ) + mock_prisma.db.litellm_teamtable.find_many = AsyncMock(return_value=[team_a, team_b]) mock_prisma.db.batch_ = MagicMock(return_value=mock_batcher) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) data = BulkUpdateTeamMemberPermissionsRequest( permissions=["/team/daily/activity"], team_ids=["team-a", "team-b"] ) - result = await bulk_update_team_member_permissions( - data=data, user_api_key_dict=self._admin_key_dict() - ) + result = await bulk_update_team_member_permissions(data=data, user_api_key_dict=self._admin_key_dict()) assert result["teams_updated"] == 2 @@ -692,18 +819,14 @@ class TestBulkUpdateTeamMemberPermissions: mock_batcher.commit = AsyncMock(return_value=None) mock_prisma = MagicMock() - mock_prisma.db.litellm_teamtable.find_many = AsyncMock( - return_value=[team_has, team_missing] - ) + mock_prisma.db.litellm_teamtable.find_many = AsyncMock(return_value=[team_has, team_missing]) mock_prisma.db.batch_ = MagicMock(return_value=mock_batcher) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) data = BulkUpdateTeamMemberPermissionsRequest( permissions=["/team/daily/activity"], team_ids=["team-has", "team-missing"] ) - result = await bulk_update_team_member_permissions( - data=data, user_api_key_dict=self._admin_key_dict() - ) + result = await bulk_update_team_member_permissions(data=data, user_api_key_dict=self._admin_key_dict()) assert result["teams_updated"] == 1 calls = mock_batcher.litellm_teamtable.update.call_args_list @@ -731,9 +854,7 @@ class TestBulkUpdateTeamMemberPermissions: ) with pytest.raises(HTTPException) as exc_info: - await bulk_update_team_member_permissions( - data=data, user_api_key_dict=self._admin_key_dict() - ) + await bulk_update_team_member_permissions(data=data, user_api_key_dict=self._admin_key_dict()) assert exc_info.value.status_code == 404 assert "team-b" in str(exc_info.value.detail) @@ -753,14 +874,10 @@ class TestBulkUpdateTeamMemberPermissions: mock_prisma = MagicMock() monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) - data = BulkUpdateTeamMemberPermissionsRequest( - permissions=["/team/daily/activity"] - ) + data = BulkUpdateTeamMemberPermissionsRequest(permissions=["/team/daily/activity"]) with pytest.raises(HTTPException) as exc_info: - await bulk_update_team_member_permissions( - data=data, user_api_key_dict=self._admin_key_dict() - ) + await bulk_update_team_member_permissions(data=data, user_api_key_dict=self._admin_key_dict()) assert exc_info.value.status_code == 400 @@ -784,9 +901,7 @@ class TestBulkUpdateTeamMemberPermissions: ) with pytest.raises(HTTPException) as exc_info: - await bulk_update_team_member_permissions( - data=data, user_api_key_dict=self._admin_key_dict() - ) + await bulk_update_team_member_permissions(data=data, user_api_key_dict=self._admin_key_dict()) assert exc_info.value.status_code == 400 @@ -804,9 +919,7 @@ class TestBulkUpdateTeamMemberPermissions: monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) data = BulkUpdateTeamMemberPermissionsRequest(permissions=[]) - result = await bulk_update_team_member_permissions( - data=data, user_api_key_dict=self._admin_key_dict() - ) + result = await bulk_update_team_member_permissions(data=data, user_api_key_dict=self._admin_key_dict()) assert result["teams_updated"] == 0 mock_prisma.db.litellm_teamtable.find_many.assert_not_called() @@ -824,14 +937,10 @@ class TestBulkUpdateTeamMemberPermissions: mock_prisma = MagicMock() monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) - data = BulkUpdateTeamMemberPermissionsRequest( - permissions=["/team/daily/activity"], apply_to_all_teams=True - ) + data = BulkUpdateTeamMemberPermissionsRequest(permissions=["/team/daily/activity"], apply_to_all_teams=True) with pytest.raises(HTTPException) as exc_info: - await bulk_update_team_member_permissions( - data=data, user_api_key_dict=self._non_admin_key_dict() - ) + await bulk_update_team_member_permissions(data=data, user_api_key_dict=self._non_admin_key_dict()) assert exc_info.value.status_code == 403 @@ -844,6 +953,4 @@ class TestBulkUpdateTeamMemberPermissions: ) with pytest.raises(ValidationError): - BulkUpdateTeamMemberPermissionsRequest( - permissions=["/not/a/real/permission"] - ) + BulkUpdateTeamMemberPermissionsRequest(permissions=["/not/a/real/permission"]) diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index 795b7cd5a9e..979eb09d7db 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -606,6 +606,55 @@ async def test_default_team_params(team_params): assert create_call_args["models"] == ["special-gpt-5"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "team_params", + [ + DefaultTeamSSOParams(max_budget=10, budget_duration="1d", organization_id="default-org"), + {"max_budget": 10, "budget_duration": "1d", "organization_id": "default-org"}, + ], +) +async def test_default_team_params_organization_id_reaches_sso_created_team(team_params): + """The SSO auto-team path builds NewTeamRequest straight from default_team_params, + so a default organization_id must land on the created team row and be validated.""" + from litellm.proxy._types import LiteLLM_OrganizationTable + + litellm.default_team_params = team_params + + mock_prisma = MagicMock() + mock_prisma.db.litellm_teamtable.find_first = AsyncMock(return_value=None) + mock_prisma.db.litellm_teamtable.create = AsyncMock() + mock_prisma.db.litellm_teamtable.count = AsyncMock(return_value=0) + mock_prisma.get_data = AsyncMock(return_value=None) + mock_prisma.jsonify_team_object = MagicMock(side_effect=lambda db_data: db_data) + + mock_org = LiteLLM_OrganizationTable( + organization_id="default-org", + budget_id="budget-id", + created_by="admin", + updated_by="admin", + ) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), patch( + "litellm.proxy.management_endpoints.team_endpoints.get_org_object", + AsyncMock(return_value=mock_org), + ) as mock_get_org: + team_id = str(uuid.uuid4()) + await MicrosoftSSOHandler.create_litellm_teams_from_service_principal_team_ids( + service_principal_teams=[ + MicrosoftServicePrincipalTeam( + principalId=team_id, + principalDisplayName="Test Team", + ) + ] + ) + + mock_prisma.db.litellm_teamtable.create.assert_called_once() + create_call_args = mock_prisma.db.litellm_teamtable.create.call_args.kwargs["data"] + assert create_call_args["organization_id"] == "default-org" + assert mock_get_org.call_args.kwargs["org_id"] == "default-org" + + @pytest.mark.asyncio async def test_create_team_without_default_params(): """ diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index d4fd5bc2dce..1075bffbeb2 100644 --- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -2816,6 +2816,94 @@ def test_update_internal_user_settings_without_teams_skips_team_lookup(mock_prox assert mock_proxy_config["save_call_count"]() == 1 +@pytest.fixture +def mock_organization_lookup(monkeypatch): + """Back /update/default_team_settings with a fake organization table. + + Yields the set of organization ids that exist; the test mutates it before the call. + """ + from unittest.mock import AsyncMock, MagicMock + + import litellm + import litellm.proxy.proxy_server as proxy_server_module + + existing_organization_ids: set = set() + + async def _find_unique(where): + organization_id = where["organization_id"] + if organization_id not in existing_organization_ids: + return None + return {"organization_id": organization_id} + + find_unique = AsyncMock(side_effect=_find_unique) + fake_prisma = MagicMock() + fake_prisma.db.litellm_organizationtable.find_unique = find_unique + + monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + monkeypatch.setattr(litellm, "default_team_params", {}) + + return { + "existing_organization_ids": existing_organization_ids, + "find_unique": find_unique, + } + + +def test_update_default_team_settings_rejects_unknown_organization( + mock_proxy_config, mock_auth, mock_organization_lookup +): + """Regression: an unknown default org saved fine here and then failed every + future team creation, far from the admin who typed it.""" + mock_organization_lookup["existing_organization_ids"].add("real-org") + + resp = client.patch( + "/update/default_team_settings", + json={"max_budget": 10.0, "organization_id": "ghost-org"}, + ) + + assert resp.status_code == 400, resp.text + assert "ghost-org" in resp.json()["detail"]["error"] + assert mock_proxy_config["save_call_count"]() == 0 + + import litellm + + assert litellm.default_team_params == {} + + +def test_update_default_team_settings_saves_when_organization_exists( + mock_proxy_config, mock_auth, mock_organization_lookup +): + """A real organization id still saves and reaches the in-memory settings.""" + mock_organization_lookup["existing_organization_ids"].add("real-org") + + resp = client.patch( + "/update/default_team_settings", + json={"max_budget": 10.0, "organization_id": "real-org"}, + ) + + assert resp.status_code == 200, resp.text + assert resp.json()["settings"]["organization_id"] == "real-org" + assert mock_proxy_config["save_call_count"]() == 1 + + import litellm + + assert litellm.default_team_params["organization_id"] == "real-org" + + +def test_update_default_team_settings_without_organization_skips_lookup( + mock_proxy_config, mock_auth, mock_organization_lookup +): + """Settings changes that don't set an organization must not pay for a DB round trip.""" + resp = client.patch( + "/update/default_team_settings", + json={"max_budget": 10.0}, + ) + + assert resp.status_code == 200, resp.text + mock_organization_lookup["find_unique"].assert_not_awaited() + assert mock_proxy_config["save_call_count"]() == 1 + + def test_update_mcp_semantic_filter_settings_requires_proxy_admin(monkeypatch): """Non-admin callers must not mutate global MCP semantic filter settings.""" from litellm.proxy._types import UserAPIKeyAuth diff --git a/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx b/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx index dd2dc42fe88..431931eb575 100644 --- a/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx +++ b/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx @@ -1,8 +1,8 @@ import React from "react"; -import { screen, waitFor } from "@testing-library/react"; +import { screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import { renderWithProviders } from "../../tests/test-utils"; +import { renderWithProviders, testQueryClient } from "../../tests/test-utils"; import TeamSSOSettings from "./TeamSSOSettings"; import * as networking from "./networking"; import NotificationsManager from "./molecules/notifications_manager"; @@ -37,6 +37,46 @@ vi.mock("./key_team_helpers/fetch_available_models_team_key", () => ({ getModelDisplayName: vi.fn((model: string) => model), })); +vi.mock("./common_components/OrganizationDropdown", () => ({ + default: ({ + organizations, + value, + onChange, + placeholder, + loading, + }: { + organizations?: { organization_id: string; organization_alias: string }[] | null; + value?: string; + onChange?: (value: string) => void; + placeholder?: string; + loading?: boolean; + }) => ( +
+ + +
+ ), +})); + vi.mock("./ModelSelect/ModelSelect", () => { const ModelSelect = ({ value, onChange }: { value: string[]; onChange: (value: string[]) => void }) => ( Date: Mon, 3 Aug 2026 13:03:46 -0700 Subject: [PATCH 041/124] fix(ui): block Playground page for viewer roles on direct URL access (#35676) --- .../playground/components/chat_ui/ChatUI.tsx | 12 +--- .../app/(dashboard)/playground/page.test.tsx | 62 +++++++++++++++++++ .../src/app/(dashboard)/playground/page.tsx | 12 ++++ ui/litellm-dashboard/src/utils/roles.ts | 2 + 4 files changed, 77 insertions(+), 11 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/page.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx index 684814bfe5b..e7261db6260 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx @@ -22,7 +22,7 @@ import { UserOutlined, } from "@ant-design/icons"; import { Card, Text, TextInput, Title, Button as TremorButton } from "@tremor/react"; -import { Button, Input, Modal, Popover, Select, Spin, Tooltip, Typography, Upload } from "antd"; +import { Button, Input, Modal, Popover, Select, Spin, Tooltip, Upload } from "antd"; import React, { useEffect, useRef, useState } from "react"; import ReactMarkdown from "react-markdown"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; @@ -1016,16 +1016,6 @@ const ChatUI: React.FC = ({ NotificationsManager.success("Chat history cleared."); }; - if (userRole && userRole === "Admin Viewer") { - const { Title, Paragraph } = Typography; - return ( -
- Access Denied - Ask your proxy admin for access to test models -
- ); - } - const onModelChange = (value: string) => { setSelectedModel(value); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/page.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/page.test.tsx new file mode 100644 index 00000000000..54e99d9db29 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/page.test.tsx @@ -0,0 +1,62 @@ +import { render, screen } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import PlaygroundPage from "./page"; + +const authState = { userRole: "Admin" }; + +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => ({ + token: "token-1", + accessToken: "sk-test", + userId: "user-1", + userRole: authState.userRole, + disabledPersonalKeyCreation: false, + }), +})); + +vi.mock("@/utils/proxyUtils", () => ({ + fetchProxySettings: vi.fn().mockResolvedValue(null), +})); + +vi.mock("@/app/(dashboard)/playground/components/chat_ui/ChatUI", () => ({ + default: () =>
, +})); + +vi.mock("@/app/(dashboard)/playground/components/compareUI/CompareUI", () => ({ + default: () =>
, +})); + +vi.mock("@/app/(dashboard)/playground/components/complianceUI/ComplianceUI", () => ({ + default: () =>
, +})); + +vi.mock("@/app/(dashboard)/playground/components/chat_ui/AgentBuilderView", () => ({ + default: () =>
, +})); + +describe("PlaygroundPage role guard", () => { + beforeEach(() => { + authState.userRole = "Admin"; + }); + + it.each(["Internal Viewer", "Admin Viewer"])("blocks the entire playground for %s", (role) => { + authState.userRole = role; + render(); + + expect(screen.getByText("Access Denied")).toBeInTheDocument(); + expect(screen.queryByRole("tab")).not.toBeInTheDocument(); + expect(screen.queryByTestId("chat-ui")).not.toBeInTheDocument(); + expect(screen.queryByTestId("compare-ui")).not.toBeInTheDocument(); + expect(screen.queryByTestId("compliance-ui")).not.toBeInTheDocument(); + expect(screen.queryByTestId("agent-builder")).not.toBeInTheDocument(); + }); + + it.each(["Admin", "Internal User", "Org Admin"])("renders the playground for %s", (role) => { + authState.userRole = role; + render(); + + expect(screen.queryByText("Access Denied")).not.toBeInTheDocument(); + expect(screen.getByRole("tab", { name: "Chat" })).toBeInTheDocument(); + expect(screen.getByTestId("chat-ui")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/page.tsx index bd3c0e31456..8986084b1a7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/page.tsx @@ -9,6 +9,7 @@ import { TabGroup, TabList, Tab, TabPanels, TabPanel } from "@tremor/react"; import { DeprecationBanner } from "@/components/DeprecationBanner"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { fetchProxySettings } from "@/utils/proxyUtils"; +import { isViewOnlyRole } from "@/utils/roles"; interface ProxySettings { PROXY_BASE_URL?: string; @@ -35,6 +36,17 @@ export default function PlaygroundPage() { initializeProxySettings(); }, [accessToken]); + if (isViewOnlyRole(userRole)) { + return ( +
+

Access Denied

+

+ Your role does not have access to the Playground. Ask your proxy admin for access to test models. +

+
+ ); + } + return (
diff --git a/ui/litellm-dashboard/src/utils/roles.ts b/ui/litellm-dashboard/src/utils/roles.ts index 38f8496c2ae..90c77a61b2d 100644 --- a/ui/litellm-dashboard/src/utils/roles.ts +++ b/ui/litellm-dashboard/src/utils/roles.ts @@ -13,6 +13,8 @@ export const rolesWithWriteAccess = ["Internal User", "Admin", "proxy_admin"]; // Per the Admin Viewer principle: read parity with Proxy Admin, no writes, // no cost-incurring actions (Playground stays gated by `rolesWithWriteAccess`). export const rolesAllowedToViewWriteScopedPages = [...rolesWithWriteAccess, "Admin Viewer", "proxy_admin_viewer"]; +export const viewOnlyRoles = ["Admin Viewer", "Internal Viewer"]; +export const isViewOnlyRole = (role: string): boolean => viewOnlyRoles.includes(role); // Helper function to check if a role is in all_admin_roles export const isAdminRole = (role: string): boolean => { From 3bc4989ce4f24a1e6a7de92138759a4860f8e333 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 3 Aug 2026 13:15:32 -0700 Subject: [PATCH 042/124] chore(ui): update brace-expansion and postcss to current patch releases The dashboard pins both packages exactly in `overrides`, so the lockfile stays on whatever those pins say. Move brace-expansion from 5.0.8 to 5.0.9 and postcss from 8.5.22 to 8.5.23, both upstream patch releases, and regenerate the lockfile. `npm ci`, `next build`, and the 5888-test vitest suite all pass on the updated lockfile. --- ui/litellm-dashboard/package-lock.json | 14 +++++++------- ui/litellm-dashboard/package.json | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index c9953cce2ab..a1bd63151b4 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -67,7 +67,7 @@ "jsdom": "27.4.0", "knip": "5.83.1", "openapi-typescript": "7.13.0", - "postcss": "8.5.22", + "postcss": "8.5.23", "prettier": "3.2.5", "tailwindcss": "4.3.2", "tw-animate-css": "1.4.0", @@ -5529,9 +5529,9 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz", - "integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "dev": true, "license": "MIT", "dependencies": { @@ -11064,9 +11064,9 @@ } }, "node_modules/postcss": { - "version": "8.5.22", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.22.tgz", - "integrity": "sha512-KBDEIpLrvpv16pp3K0Fw+UCoZfopFjjgeB+0tA/aaThfEE74kKDLrgg603YvOWJyg3+WYtyq3xYsQWsIyZlPqQ==", + "version": "8.5.23", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.23.tgz", + "integrity": "sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==", "funding": [ { "type": "opencollective", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 32d93729dbe..4760b622f9b 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -79,7 +79,7 @@ "jsdom": "27.4.0", "knip": "5.83.1", "openapi-typescript": "7.13.0", - "postcss": "8.5.22", + "postcss": "8.5.23", "prettier": "3.2.5", "tailwindcss": "4.3.2", "tw-animate-css": "1.4.0", @@ -90,13 +90,13 @@ "overrides": { "prismjs": "1.30.0", "js-yaml": "4.3.0", - "brace-expansion": "5.0.8", + "brace-expansion": "5.0.9", "glob": "13.0.0", "minimatch": "10.2.4", "ws": "8.21.0", "braces": "3.0.3", "axios": "1.13.6", - "postcss": "8.5.22", + "postcss": "8.5.23", "esbuild": "0.28.1", "date-fns": "^4.4.0", "sharp": "^0.35.0" From cad319a862e744f5598b99b3e0c32bf33c185623 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 3 Aug 2026 13:24:44 -0700 Subject: [PATCH 043/124] chore(deps): update gitpython to 3.1.57 gitpython arrives transitively through mlflow-skinny, which accepts >=3.1.9,<4, so this is a lock-only move with no pyproject change. Relocked with `uv lock --upgrade-package gitpython`; gitpython is the only package whose version changed. `uv sync --all-groups --all-extras` and tests/test_litellm/integrations/test_mlflow.py pass on the result. --- uv.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uv.lock b/uv.lock index 0bfe9208872..15e65c9dffd 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-07-29T22:09:54.255381Z" +exclude-newer = "2026-07-31T20:23:04.658774Z" exclude-newer-span = "P3D" [manifest] @@ -2378,14 +2378,14 @@ wheels = [ [[package]] name = "gitpython" -version = "3.1.55" +version = "3.1.57" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "gitdb" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b2/ab/ba0d29f2fa2277ed6256b2ac09003494045355f3a10bf32f351761287870/gitpython-3.1.55.tar.gz", hash = "sha256:781e3b1624dad81b24e9524bf0297b69786a0706db2cbceec1e2b05c38e5152f", size = 225071, upload-time = "2026-07-23T02:52:43.246Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/0d/132ed135c871b6bf91adf16a0e43797cd535b81d4973b5d09291c54fc5ee/gitpython-3.1.57.tar.gz", hash = "sha256:c493ec57c0ef6b19743798b6a5af859c71814b524e7e6f97baa2f8e658961488", size = 225898, upload-time = "2026-07-26T07:33:26.351Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/20/6a/d3b8208d2f8aac66abe8ccc1c23fa2c89464ec42cc71a601e95d05902428/gitpython-3.1.55-py3-none-any.whl", hash = "sha256:7c9ec1e69c158c081632ab35c41471e302c96db2ae42165036a5d2403378812e", size = 216590, upload-time = "2026-07-23T02:52:41.932Z" }, + { url = "https://files.pythonhosted.org/packages/41/6e/2139de986d9c7c3ac86f1f8be43858ce90bdfe2f7175e6c80c650ba15242/gitpython-3.1.57-py3-none-any.whl", hash = "sha256:4ccf7d73c10f5c9e76043fbb2675ac5a1b3ff5b41e648f56bcbed5f63792ecaf", size = 217151, upload-time = "2026-07-26T07:33:24.838Z" }, ] [[package]] From 66bc70365f69ce77288689d681557d5cf539a450 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Mon, 3 Aug 2026 13:28:38 -0700 Subject: [PATCH 044/124] fix(caching): close evicted LLM clients so their connections are reclaimed (#35492) An evicted client was left for the garbage collector, but every OpenAI/Azure SDK client is a reference cycle, so nothing freed the client or its pooled TCP connections until a generational sweep ran. Driving 2000 azure calls through the official image with no forced collection, live clients and open sockets climbed from 202 to 1361 while the cache stayed at its 200-entry bound, and RSS grew 279 MB to 456 MB against a TLS upstream. Closing on eviction is what caused the earlier 'Cannot send a request, as the client has been closed' regression, so an evicted client litellm created is now closed only once a grace window has passed, by which point any request that was already holding it has finished. A client the caller supplied is never closed, since litellm does not own its lifecycle. Resolves LIT-4883 --- litellm/caching/evicted_client_closer.py | 276 ++++++++++++ litellm/caching/llm_caching_handler.py | 45 +- litellm/constants.py | 10 + litellm/llms/azure/common_utils.py | 2 + litellm/llms/custom_httpx/http_handler.py | 2 + litellm/llms/openai/common_utils.py | 23 +- litellm/llms/openai/openai.py | 10 +- .../caching/test_evicted_client_closer.py | 409 ++++++++++++++++++ .../caching/test_llm_caching_handler.py | 66 +++ .../llms/azure/test_azure_common_utils.py | 71 +++ .../llms/openai/test_openai_common_utils.py | 72 +++ 11 files changed, 975 insertions(+), 11 deletions(-) create mode 100644 litellm/caching/evicted_client_closer.py create mode 100644 tests/test_litellm/caching/test_evicted_client_closer.py diff --git a/litellm/caching/evicted_client_closer.py b/litellm/caching/evicted_client_closer.py new file mode 100644 index 00000000000..bca9656b252 --- /dev/null +++ b/litellm/caching/evicted_client_closer.py @@ -0,0 +1,276 @@ +""" +Deferred close of HTTP/SDK clients that the LLM client cache has evicted. + +Eviction only drops the cache's reference to a client. Every OpenAI/Azure SDK +client is a reference cycle (each resource namespace holds the client back), so +an evicted client and its pooled TCP connections survive until a generational +collection runs, which under load is thousands of requests later. + +Closing at eviction time is not an option: a request that was handed the client +just before it was evicted is still using it, and closing it underneath that +request raises ``RuntimeError: Cannot send a request, as the client has been +closed.`` + +So an evicted client is closed once two conditions hold. A grace window must +have passed since its eviction, which covers a request that holds the client +but is momentarily not on the wire, and the client must report no connection in +flight. The second condition is what keeps the first honest: a request may run +for ``litellm.request_timeout`` seconds, 6000 by default, and a streaming +response is bounded only by how long the upstream keeps sending, so no deadline +on its own can promise that a request has finished. + +Only clients litellm itself created are closed; a client the caller supplied is +left alone because litellm does not own its lifecycle. + +A client that closes synchronously is closed from wherever the cache is next +used. One whose close is a coroutine needs the event loop it was evicted on, so +it waits for a call from that loop rather than having work scheduled onto a loop +it does not belong to. Queued clients are therefore bucketed by what it takes to +close them, and each bucket is ordered by deadline, so a reap walks the entries +that are due rather than the whole queue. + +The queue holds its clients weakly, so waiting out a grace window never keeps +alive anything the collector would have reclaimed first. +""" + +import asyncio +import inspect +import threading +import time +import weakref +from collections import deque +from collections.abc import Awaitable, Callable, Iterator +from dataclasses import dataclass, replace + +from litellm.constants import ( + EVICTED_LLM_CLIENT_CLOSE_GRACE_SECONDS, + EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING, +) + +_CLOSABLE_ANYWHERE = "closable-anywhere" +_CLOSABLE_ON_ANY_LOOP = "closable-on-any-loop" + +_BucketKey = str | int + + +@dataclass(frozen=True, slots=True) +class _PendingClose: + """A queued close. + + The client is held weakly, so queueing one never keeps alive anything the + collector would otherwise have reclaimed first. + + ``needs_loop`` is set for a client whose close is a coroutine; those can only + be closed from the event loop they were evicted on, recorded in ``loop_id``. + A client that closes synchronously carries neither constraint. + """ + + client_ref: "weakref.ref[object]" + loop_id: int | None + needs_loop: bool + close_after: float + + +def _bucket_key(pending: _PendingClose) -> _BucketKey: + """Which reaps can close this entry: any at all, any running a loop, or one loop's.""" + if not pending.needs_loop: + return _CLOSABLE_ANYWHERE + if pending.loop_id is None: + return _CLOSABLE_ON_ANY_LOOP + return pending.loop_id + + +def _running_loop_id() -> int | None: + try: + return id(asyncio.get_running_loop()) + except RuntimeError: + return None + + +def _close_function(client: object) -> Callable[[], object] | None: + close_fn: Callable[[], object] | None = getattr(client, "aclose", None) or getattr(client, "close", None) + return close_fn + + +def _transport_of(client: object) -> object: + """The httpx transport behind an SDK wrapper, a litellm handler, or a bare client.""" + for holder in (getattr(client, "_client", None), getattr(client, "client", None), client): + transport: object = getattr(holder, "_transport", None) + if transport is not None: + return transport + return None + + +def _connection_is_idle(connection: object) -> bool: + """A pooled connection is idle unless it is servicing a request.""" + is_idle: object = getattr(connection, "is_idle", None) + return bool(is_idle()) if callable(is_idle) else True + + +def _pool_has_busy_connection(transport: object) -> bool | None: + """Whether the httpcore pool behind the transport is servicing a request. + + ``None`` when there is no such pool, so the caller can ask the other backend. + """ + pooled: object = getattr(getattr(transport, "_pool", None), "connections", None) + if not isinstance(pooled, (list, tuple)): + return None + return any( + not _connection_is_idle(connection) # pyright: ignore[reportUnknownArgumentType] # untyped pool list + for connection in pooled # pyright: ignore[reportUnknownVariableType] # untyped pool list + ) + + +def _has_connection_in_flight(client: object) -> bool: + """Whether the client is servicing a request right now. + + Both connection backends litellm uses already account for the connections + they have handed out, so this reads the client's own lease accounting rather + than inferring it from elapsed time: httpcore reports a non-idle connection + for the whole of a response including a stream, and aiohttp holds the + connection in ``_acquired`` over the same span. + + A client that cannot answer is reported as idle, which leaves the grace + window as the only guard, exactly as it was before this check existed. + """ + try: + transport = _transport_of(client) + pooled_busy = _pool_has_busy_connection(transport) + if pooled_busy is not None: + return pooled_busy + session: object = getattr(transport, "client", None) + return bool(getattr(getattr(session, "connector", None), "_acquired", None)) + except Exception: # noqa: BLE001 - a client that cannot report its state is treated as idle + return False + + +async def _close_quietly(closing: Awaitable[object]) -> None: + try: + await closing + except Exception: # noqa: BLE001 - a discarded client's close must never surface to callers + pass + + +class EvictedClientCloser: + """Closes evicted, litellm-owned clients once they are idle and out of grace.""" + + def __init__( + self, + grace_seconds: float = EVICTED_LLM_CLIENT_CLOSE_GRACE_SECONDS, + max_pending: int = EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING, + clock: Callable[[], float] = time.monotonic, + ) -> None: + self._grace_seconds = grace_seconds + self._max_pending = max_pending + self._clock = clock + self._owned: weakref.WeakSet[object] = weakref.WeakSet() + self._buckets: dict[_BucketKey, deque[_PendingClose]] = {} # mutable-ok: deadline-ordered queues + self._pending_count = 0 + self._queue_lock = threading.Lock() # the cache is reachable from every worker thread's loop + self._close_tasks: set[asyncio.Task[None]] = set() # mutable-ok: strong refs to running closes + + def mark_owned(self, client: object) -> None: + """Record that litellm created this client, so it may be closed on eviction.""" + try: + self._owned.add(client) + except TypeError: + pass # values that cannot be weak-referenced are never litellm clients + + def _is_owned(self, client: object) -> bool: + try: + return client in self._owned + except TypeError: + return False # unhashable values are never litellm clients + + def schedule(self, client: object) -> None: + """Queue an evicted client for closing once it is idle and out of grace. + + Past ``max_pending`` the client is left to the collector instead, so a + workload that churns the cache cannot grow this queue without bound. + Every queued entry comes due within one grace window, so the capacity it + occupies is returned within that window rather than held. + """ + if client is None or not self._is_owned(client): + return + close_fn = _close_function(client) + if close_fn is None: + return + if self._pending_count >= self._max_pending: + return + self._enqueue( + _PendingClose( + client_ref=weakref.ref(client), + loop_id=_running_loop_id(), + needs_loop=inspect.iscoroutinefunction(close_fn), + close_after=self._clock() + self._grace_seconds, + ) + ) + + def reap(self) -> None: + """Close every queued client that is due, idle, and closable from here. + + Called from the cache's read path, so the empty-queue exit comes first and + the work done past it is proportional to what is due, not to the queue. + """ + if not self._pending_count: + return + now = self._clock() + for pending in self._take_due(_running_loop_id(), now): + client = pending.client_ref() + if client is None: + continue + if _has_connection_in_flight(client): + self._enqueue(replace(pending, close_after=now + self._grace_seconds)) + continue + self._close(client) + + @property + def pending_count(self) -> int: + return self._pending_count + + def _enqueue(self, pending: _PendingClose) -> None: + """Append to the entry's bucket, dropping any dead entries it queues behind. + + Deadlines only ever move forward, so appending keeps each bucket ordered + by deadline, and entries whose client the collector already took sit at + the front rather than having to be searched for. + """ + with self._queue_lock: + bucket = self._buckets.setdefault(_bucket_key(pending), deque()) # mutable-ok: FIFO by design + while bucket and bucket[0].client_ref() is None: + bucket.popleft() + self._pending_count -= 1 + bucket.append(pending) + self._pending_count += 1 + + def _take_due(self, loop_id: int | None, now: float) -> tuple[_PendingClose, ...]: + buckets = (_CLOSABLE_ANYWHERE,) if loop_id is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_id) + with self._queue_lock: + return tuple(pending for key in buckets for pending in self._drain_locked(key, now)) + + def _drain_locked(self, key: _BucketKey, now: float) -> Iterator[_PendingClose]: + bucket = self._buckets.get(key) + if bucket is None: + return + while bucket and bucket[0].close_after <= now: + self._pending_count -= 1 + yield bucket.popleft() + if not bucket: + del self._buckets[key] + + def _close(self, client: object) -> None: + close_fn = _close_function(client) + if close_fn is None: + return + try: + closing = close_fn() + except Exception: # noqa: BLE001 - a discarded client's close must never surface to callers + return + if not inspect.isawaitable(closing): + return + task = asyncio.get_running_loop().create_task(_close_quietly(closing)) + self._close_tasks.add(task) + task.add_done_callback(self._close_tasks.discard) + + +default_evicted_client_closer = EvictedClientCloser() diff --git a/litellm/caching/llm_caching_handler.py b/litellm/caching/llm_caching_handler.py index c2274713bb9..7eae8ee3749 100644 --- a/litellm/caching/llm_caching_handler.py +++ b/litellm/caching/llm_caching_handler.py @@ -4,21 +4,44 @@ Add the event loop to the cache key, to prevent event loop closed errors. import asyncio +from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer from .in_memory_cache import InMemoryCache class LLMClientCache(InMemoryCache): """Cache for LLM HTTP clients (OpenAI, Azure, httpx, etc.). - IMPORTANT: This cache intentionally does NOT close clients on eviction. - Evicted clients may still be in use by in-flight requests. Closing them - eagerly causes ``RuntimeError: Cannot send a request, as the client has - been closed.`` errors in production after the TTL (1 hour) expires. + An evicted client is never closed on the spot: a request handed the client + just before eviction is still using it, and closing it there raises + ``RuntimeError: Cannot send a request, as the client has been closed.`` - Clients that are no longer referenced will be garbage-collected normally. - For explicit shutdown cleanup, use ``close_litellm_async_clients()``. + Nor can eviction be left to rely on garbage collection. The SDK clients are + reference cycles, so an evicted client and its open TCP connections survive + until a generational collection runs. Instead a client litellm created is + handed to ``EvictedClientCloser``, which closes it once a grace window has + passed. Clients the caller supplied are left untouched. """ + def __init__( + self, + max_size_in_memory: int | None = 200, + default_ttl: int | None = 600, + max_size_per_item: int | None = 1024, + evicted_client_closer: EvictedClientCloser | None = None, + ): + super().__init__( + max_size_in_memory=max_size_in_memory, + default_ttl=default_ttl, + max_size_per_item=max_size_per_item, + ) + self.evicted_client_closer = evicted_client_closer or default_evicted_client_closer + + def _remove_key(self, key: str) -> None: + evicted: object = self.cache_dict.get(key) + super()._remove_key(key) + self.evicted_client_closer.schedule(evicted) + self.evicted_client_closer.reap() + def update_cache_key_with_event_loop(self, key): """ Add the event loop to the cache key, to prevent event loop closed errors. @@ -31,16 +54,22 @@ class LLMClientCache(InMemoryCache): except RuntimeError: # handle no current running event loop return key - def set_cache(self, key, value, **kwargs): + def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs): + """``litellm_owned_client`` marks a client litellm built, so it may be closed once evicted.""" + if litellm_owned_client: + self.evicted_client_closer.mark_owned(value) key = self.update_cache_key_with_event_loop(key) return super().set_cache(key, value, **kwargs) - async def async_set_cache(self, key, value, **kwargs): + async def async_set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs): + if litellm_owned_client: + self.evicted_client_closer.mark_owned(value) key = self.update_cache_key_with_event_loop(key) return await super().async_set_cache(key, value, **kwargs) def get_cache(self, key, **kwargs): key = self.update_cache_key_with_event_loop(key) + self.evicted_client_closer.reap() return super().get_cache(key, **kwargs) diff --git a/litellm/constants.py b/litellm/constants.py index d46f62af000..06421e6ed6a 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -197,6 +197,16 @@ RUNWAYML_POLLING_TIMEOUT = int(os.getenv("RUNWAYML_POLLING_TIMEOUT", 600)) # 10 ########## Networking constants ############################################################## _DEFAULT_TTL_FOR_HTTPX_CLIENTS = 3600 # 1 hour, re-use the same httpx client for 1 hour +# The earliest an evicted, litellm-created client may be closed. A request handed the +# client just before eviction is still using it, so nothing is closed inside this window; +# past it, the client is closed once it reports no connection in flight. +EVICTED_LLM_CLIENT_CLOSE_GRACE_SECONDS = 900 + +# How many evicted clients may be queued for closing at once. Past this, an evicted client +# is left to the collector rather than letting a cache-churning workload grow the queue +# without bound. Each queued entry is ~100 bytes and comes due within one grace window. +EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING = 10_000 + # Aiohttp connection pooling - prevents memory leaks from unbounded connection growth # Set to 0 for unlimited (not recommended for production) AIOHTTP_CONNECTOR_LIMIT = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 1000)) diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index 8e0bd363a8a..8db422e00ff 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -508,6 +508,8 @@ class BaseAzureLLM(BaseOpenAILLM): openai_client=openai_client, client_initialization_params=client_initialization_params, client_type="azure", + litellm_owned_client=client is None + and self.owns_wrapped_http_client(azure_client_params.get("http_client")), ) return openai_client diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 046840e6fd0..3e34b483002 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -1411,6 +1411,7 @@ def get_async_httpx_client( key=_cache_key_name, value=_new_client, ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS, + litellm_owned_client=True, ) return _new_client @@ -1456,5 +1457,6 @@ def _get_httpx_client(params: dict | None = None) -> HTTPHandler: key=_cache_key_name, value=_new_client, ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS, + litellm_owned_client=True, ) return _new_client diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index e72680f387d..082764df208 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -134,13 +134,33 @@ class BaseOpenAILLM: _cached_client = litellm.in_memory_llm_clients_cache.get_cache(_cache_key) return _cached_client + @staticmethod + def owns_wrapped_http_client(http_client: Optional[Union[httpx.Client, httpx.AsyncClient]]) -> bool: + """Whether litellm may close an SDK client built around ``http_client``. + + ``_get_async_http_client`` / ``_get_sync_http_client`` hand back + ``litellm.aclient_session`` / ``litellm.client_session`` when the caller + configured one. The SDK's ``close()`` closes whatever http client it was + given, so an SDK client wrapping one of those shared sessions must never be + closed on eviction; the caller goes on using the session. ``None`` means the + SDK built its own http client, which litellm does own. + """ + if http_client is None: + return True + return http_client is not litellm.aclient_session and http_client is not litellm.client_session + @staticmethod def set_cached_openai_client( openai_client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI, client_type: Literal["openai", "azure"], client_initialization_params: dict, + litellm_owned_client: bool = False, ): - """Stores the OpenAI client in the in-memory cache for _DEFAULT_TTL_FOR_HTTPX_CLIENTS SECONDS""" + """Stores the OpenAI client in the in-memory cache for _DEFAULT_TTL_FOR_HTTPX_CLIENTS SECONDS + + ``litellm_owned_client`` says litellm built this client, so the cache may close it once it + is evicted. A client the caller supplied stays open, since litellm does not own it. + """ _cache_key = BaseOpenAILLM.get_openai_client_cache_key( client_initialization_params=client_initialization_params, client_type=client_type, @@ -149,6 +169,7 @@ class BaseOpenAILLM: key=_cache_key, value=openai_client, ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS, + litellm_owned_client=litellm_owned_client, ) @staticmethod diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 845ad22589f..7096cdbf8fd 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -366,11 +366,16 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): if cached_client: if isinstance(cached_client, OpenAI) or isinstance(cached_client, AsyncOpenAI): return cached_client + http_client: Optional[Union[httpx.Client, httpx.AsyncClient]] = ( + OpenAIChatCompletion._get_async_http_client(shared_session=shared_session) + if is_async + else OpenAIChatCompletion._get_sync_http_client() + ) if is_async: _new_client: OpenAI | AsyncOpenAI = AsyncOpenAI( api_key=api_key, base_url=api_base, - http_client=OpenAIChatCompletion._get_async_http_client(shared_session=shared_session), + http_client=http_client, timeout=timeout, max_retries=max_retries, organization=organization, @@ -379,7 +384,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): _new_client = OpenAI( api_key=api_key, base_url=api_base, - http_client=OpenAIChatCompletion._get_sync_http_client(), + http_client=http_client, timeout=timeout, max_retries=max_retries, organization=organization, @@ -390,6 +395,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): openai_client=_new_client, client_initialization_params=client_initialization_params, client_type="openai", + litellm_owned_client=self.owns_wrapped_http_client(http_client), ) return _new_client diff --git a/tests/test_litellm/caching/test_evicted_client_closer.py b/tests/test_litellm/caching/test_evicted_client_closer.py new file mode 100644 index 00000000000..a08fd58079d --- /dev/null +++ b/tests/test_litellm/caching/test_evicted_client_closer.py @@ -0,0 +1,409 @@ +""" +Tests for EvictedClientCloser. + +An evicted client must stay open long enough for a request that already holds it +to finish, and must then actually be closed, otherwise its connection pool is +retained until a generational collection runs. A client the caller supplied is +never closed, because litellm does not own its lifecycle. +""" + +import asyncio +import gc +import weakref + +import httpx +import pytest + +from litellm.caching.evicted_client_closer import EvictedClientCloser +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + +class FakeClock: + """Hand-advanced monotonic clock, so grace windows need no real waiting.""" + + def __init__(self) -> None: + self.now = 1000.0 + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + self.now += seconds + + +class AsyncClient: + def __init__(self) -> None: + self.closed = False + + async def close(self) -> None: + self.closed = True + + +class SyncClient: + def __init__(self) -> None: + self.closed = False + + def close(self) -> None: + self.closed = True + + +class CountingDeadline(float): + """A clock reading that tallies every deadline comparison made against it. + + Deadline comparisons are the work a reap does, so counting them says whether + that work tracks the entries that are due or the size of the whole queue. + """ + + comparisons = 0 + + def __add__(self, other: float) -> "CountingDeadline": + return CountingDeadline(float(self) + other) + + def __le__(self, other: float) -> bool: + CountingDeadline.comparisons += 1 + return float(self) <= float(other) + + def __gt__(self, other: float) -> bool: + CountingDeadline.comparisons += 1 + return float(self) > float(other) + + +def make_closer(clock: FakeClock, grace_seconds: float = 60.0) -> EvictedClientCloser: + return EvictedClientCloser(grace_seconds=grace_seconds, clock=clock) + + +async def _trickling_upstream(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + """Serves a chunked body slowly, so a request stays on the wire long enough to observe.""" + await reader.read(4096) + writer.write(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n") + await writer.drain() + for _ in range(6): + writer.write(b"5\r\nhello\r\n") + await writer.drain() + await asyncio.sleep(0.1) + writer.write(b"0\r\n\r\n") + await writer.drain() + + +@pytest.mark.asyncio +async def test_owned_client_is_closed_once_the_grace_window_elapses(): + clock = FakeClock() + closer = make_closer(clock) + client = AsyncClient() + + closer.mark_owned(client) + closer.schedule(client) + clock.advance(61.0) + closer.reap() + await asyncio.sleep(0.05) + + assert client.closed is True + assert closer.pending_count == 0 + + +@pytest.mark.asyncio +async def test_owned_client_stays_open_inside_the_grace_window(): + """A request handed the client just before eviction is still using it.""" + clock = FakeClock() + closer = make_closer(clock) + client = AsyncClient() + + closer.mark_owned(client) + closer.schedule(client) + clock.advance(59.0) + closer.reap() + await asyncio.sleep(0.05) + + assert client.closed is False + assert closer.pending_count == 1 + + +@pytest.mark.asyncio +async def test_caller_supplied_client_is_never_closed(): + clock = FakeClock() + closer = make_closer(clock) + client = AsyncClient() + + closer.schedule(client) + clock.advance(3600.0) + closer.reap() + await asyncio.sleep(0.05) + + assert client.closed is False + assert closer.pending_count == 0 + + +@pytest.mark.asyncio +async def test_sync_client_is_closed_once_the_grace_window_elapses(): + clock = FakeClock() + closer = make_closer(clock) + client = SyncClient() + + closer.mark_owned(client) + closer.schedule(client) + clock.advance(61.0) + closer.reap() + + assert client.closed is True + + +@pytest.mark.asyncio +async def test_a_failing_close_does_not_propagate_or_block_the_others(): + class ExplodingClient: + async def close(self) -> None: + raise RuntimeError("connection already gone") + + clock = FakeClock() + closer = make_closer(clock) + exploding, healthy = ExplodingClient(), AsyncClient() + + for client in (exploding, healthy): + closer.mark_owned(client) + closer.schedule(client) + clock.advance(61.0) + closer.reap() + await asyncio.sleep(0.05) + + assert healthy.closed is True + + +@pytest.mark.asyncio +async def test_an_unhashable_cached_value_does_not_break_eviction(): + """The cache holds arbitrary values; an ownership test must never raise on one.""" + + class Unhashable: + __hash__ = None # pyright: ignore[reportAssignmentType] # unhashable by construction + + clock = FakeClock() + closer = make_closer(clock) + + closer.mark_owned(Unhashable()) + closer.schedule(Unhashable()) + + assert closer.pending_count == 0 + + +@pytest.mark.asyncio +async def test_values_with_nothing_to_close_are_never_queued(): + """The cache holds plain values too; those have nothing to reclaim.""" + + class NotAClient: + pass + + clock = FakeClock() + closer = make_closer(clock) + value = NotAClient() + + closer.mark_owned(value) + closer.schedule(value) + + assert closer.pending_count == 0 + + +@pytest.mark.asyncio +async def test_a_queued_client_is_not_kept_alive_by_the_queue(): + """Waiting out a grace window must not retain what the collector would free first.""" + clock = FakeClock() + closer = make_closer(clock) + client = AsyncClient() + gone = weakref.ref(client) + + closer.mark_owned(client) + closer.schedule(client) + del client + gc.collect() + + assert gone() is None, "the pending queue is holding the client alive" + + clock.advance(61.0) + closer.reap() + assert closer.pending_count == 0 + + +def test_sync_client_evicted_outside_an_event_loop_is_still_closed(): + """The sync httpx handler is cached and evicted from call sites with no loop.""" + clock = FakeClock() + closer = make_closer(clock) + client = SyncClient() + + closer.mark_owned(client) + closer.schedule(client) + assert closer.pending_count == 1 + + clock.advance(61.0) + closer.reap() + + assert client.closed is True + assert closer.pending_count == 0 + + +@pytest.mark.asyncio +async def test_an_async_client_waits_for_a_loop_rather_than_being_dropped(): + clock = FakeClock() + closer = make_closer(clock) + client = AsyncClient() + closer.mark_owned(client) + + def schedule_outside_a_loop() -> None: + closer.schedule(client) + clock.advance(61.0) + closer.reap() + + await asyncio.to_thread(schedule_outside_a_loop) + assert client.closed is False, "no loop was running, so it could not have been closed" + assert closer.pending_count == 1 + + closer.reap() + await asyncio.sleep(0.05) + + assert client.closed is True + + +@pytest.mark.asyncio +async def test_a_client_evicted_on_another_event_loop_is_left_alone(): + """Closing a client bound to a different loop would schedule work on that loop.""" + clock = FakeClock() + closer = make_closer(clock) + client = AsyncClient() + closer.mark_owned(client) + + def schedule_on_its_own_loop() -> None: + asyncio.run(_schedule()) + + async def _schedule() -> None: + closer.schedule(client) + + await asyncio.to_thread(schedule_on_its_own_loop) + assert closer.pending_count == 1 + + clock.advance(61.0) + closer.reap() + await asyncio.sleep(0.05) + + assert client.closed is False + assert closer.pending_count == 1 + + +@pytest.mark.asyncio +async def test_a_client_serving_a_request_is_not_closed_when_its_grace_window_ends(): + """The grace window on its own cannot promise that a request has finished. + + ``litellm.request_timeout`` defaults to 6000 seconds and a streaming response + is bounded only by how long the upstream keeps sending, so a client past its + deadline is closed only once its own pool reports nothing in flight. + """ + server = await asyncio.start_server(_trickling_upstream, "127.0.0.1", 0) + port = server.sockets[0].getsockname()[1] + clock = FakeClock() + closer = make_closer(clock) + client = httpx.AsyncClient() + + closer.mark_owned(client) + closer.schedule(client) + + async def read_the_stream() -> int: + received = 0 + async with client.stream("GET", f"http://127.0.0.1:{port}/") as response: + async for chunk in response.aiter_bytes(): + received += len(chunk) + return received + + streaming = asyncio.create_task(read_the_stream()) + await asyncio.sleep(0.25) # the request is on the wire + clock.advance(3600.0) # and its grace window is long gone + closer.reap() + await asyncio.sleep(0.05) + + assert client.is_closed is False, "closed a client that was serving a request" + assert await streaming > 0, "the in-flight request did not survive the reap" + + clock.advance(3600.0) + closer.reap() + await asyncio.sleep(0.05) + + assert client.is_closed is True, "an idle client past its grace window must be closed" + assert closer.pending_count == 0 + server.close() + + +@pytest.mark.asyncio +async def test_the_aiohttp_backed_handler_is_not_closed_mid_request(): + """The default async path is aiohttp-backed, whose pool accounts for its own leases.""" + server = await asyncio.start_server(_trickling_upstream, "127.0.0.1", 0) + port = server.sockets[0].getsockname()[1] + clock = FakeClock() + closer = make_closer(clock) + handler = AsyncHTTPHandler() + + closer.mark_owned(handler) + closer.schedule(handler) + + request = asyncio.create_task(handler.get(f"http://127.0.0.1:{port}/")) + await asyncio.sleep(0.25) + clock.advance(3600.0) + closer.reap() + await asyncio.sleep(0.05) + + assert handler.client.is_closed is False, "closed a handler that was serving a request" + assert (await request).status_code == 200 + + clock.advance(3600.0) + closer.reap() + await asyncio.sleep(0.05) + + assert handler.client.is_closed is True + server.close() + + +def test_the_pending_queue_cannot_grow_past_its_bound(): + """A caller that churns the client cache must not be able to grow this queue.""" + clock = FakeClock() + closer = EvictedClientCloser(grace_seconds=60.0, max_pending=8, clock=clock) + clients = tuple(SyncClient() for _ in range(50)) + + for client in clients: + closer.mark_owned(client) + closer.schedule(client) + + assert closer.pending_count == 8, "the queue grew past max_pending" + + clock.advance(61.0) + closer.reap() + + assert closer.pending_count == 0 + assert sum(client.closed for client in clients) == 8, "everything queued should have been closed" + + +def test_a_reap_looks_at_what_is_due_rather_than_at_the_whole_queue(): + """Sustained churn evicts a client per request, and every read of the cache reaps. + + So the cost of a reap has to track the entries that are due, not the length of + the queue; a reap that filters the whole queue makes the pair quadratic. Each + bucket is ordered by deadline, so an up-to-date reap compares one entry per + bucket and stops. Counting the comparisons measures that directly, where a + wall-clock budget would only measure the machine. + """ + evictions = 1_000 + clock = FakeClock() + closer = EvictedClientCloser( + grace_seconds=60.0, + max_pending=evictions, + clock=lambda: CountingDeadline(clock.now), + ) + clients = tuple(SyncClient() for _ in range(evictions)) + for client in clients: + closer.mark_owned(client) + + CountingDeadline.comparisons = 0 + for client in clients: + closer.schedule(client) + closer.reap() # nothing is due yet, which is the hot path + clock.advance(61.0) + closer.reap() + + assert closer.pending_count == 0 + assert all(client.closed for client in clients) + assert CountingDeadline.comparisons < 10 * evictions, ( + f"{CountingDeadline.comparisons} deadline comparisons for {evictions} evictions; " + "a reap is walking the whole queue" + ) diff --git a/tests/test_litellm/caching/test_llm_caching_handler.py b/tests/test_litellm/caching/test_llm_caching_handler.py index 8e6a94945b0..5f0e82dbb80 100644 --- a/tests/test_litellm/caching/test_llm_caching_handler.py +++ b/tests/test_litellm/caching/test_llm_caching_handler.py @@ -19,6 +19,7 @@ sys.path.insert( 0, os.path.abspath("../../..") ) # Adds the parent directory to the system path +from litellm.caching.evicted_client_closer import EvictedClientCloser from litellm.caching.llm_caching_handler import LLMClientCache @@ -156,6 +157,71 @@ def test_remove_key_no_event_loop(): assert "test-key" not in cache.cache_dict +class _FakeClock: + """Hand-advanced monotonic clock, so grace windows need no real waiting.""" + + def __init__(self) -> None: + self.now = 1000.0 + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + self.now += seconds + + +@pytest.mark.asyncio +async def test_evicted_litellm_owned_client_is_closed_once_the_grace_window_elapses(): + """ + Eviction only drops the cache's reference. The SDK clients are reference + cycles, so without an explicit close the client keeps its connection pool + open until a generational collection runs. + """ + clock = _FakeClock() + cache = LLMClientCache( + max_size_in_memory=2, + evicted_client_closer=EvictedClientCloser(grace_seconds=60.0, clock=clock), + ) + + client = MockAsyncClient() + cache.set_cache("client-key", client, litellm_owned_client=True, ttl=600) + + cache.ttl_dict = {key: 0 for key in cache.ttl_dict} + cache.expiration_heap = [(0, key) for _, key in cache.expiration_heap] + cache.evict_cache() + await asyncio.sleep(0.1) + assert client.closed is False, "an in-flight request may still hold the client" + + clock.advance(61.0) + cache.get_cache("any-key") + await asyncio.sleep(0.1) + + assert client.closed is True + + +@pytest.mark.asyncio +async def test_evicted_caller_supplied_client_is_never_closed(): + """litellm does not own a client the caller passed in, so it must stay open.""" + clock = _FakeClock() + cache = LLMClientCache( + max_size_in_memory=2, + evicted_client_closer=EvictedClientCloser(grace_seconds=60.0, clock=clock), + ) + + client = MockAsyncClient() + cache.set_cache("client-key", client, ttl=600) + + cache.ttl_dict = {key: 0 for key in cache.ttl_dict} + cache.expiration_heap = [(0, key) for _, key in cache.expiration_heap] + cache.evict_cache() + + clock.advance(3600.0) + cache.get_cache("any-key") + await asyncio.sleep(0.1) + + assert client.closed is False + + def test_remove_key_removes_plain_values(): """ _remove_key correctly removes non-client values (strings, dicts, etc.). diff --git a/tests/test_litellm/llms/azure/test_azure_common_utils.py b/tests/test_litellm/llms/azure/test_azure_common_utils.py index a3280b90fe3..c5d4bd044cc 100644 --- a/tests/test_litellm/llms/azure/test_azure_common_utils.py +++ b/tests/test_litellm/llms/azure/test_azure_common_utils.py @@ -2034,3 +2034,74 @@ def test_azure_traditional_api_uses_azure_openai_client(): assert isinstance( async_client, AsyncAzureOpenAI ), f"Expected AsyncAzureOpenAI client for api_version={api_version}" + + +def test_evicting_an_azure_client_built_on_the_callers_session_leaves_it_open(monkeypatch): + """`initialize_azure_sdk_client` puts `litellm.aclient_session` on the SDK client. + + That session belongs to the caller. `AsyncAzureOpenAI.close()` closes whatever + http client it was handed, so treating the wrapper as litellm's to close would + close the caller's shared session out from under them. + """ + import httpx + + from litellm.caching.evicted_client_closer import EvictedClientCloser + from litellm.caching.llm_caching_handler import LLMClientCache + + shared_session = httpx.AsyncClient() + closer = EvictedClientCloser(grace_seconds=0.0) + monkeypatch.setattr(litellm, "aclient_session", shared_session) + monkeypatch.setattr( + litellm, + "in_memory_llm_clients_cache", + LLMClientCache(evicted_client_closer=closer), + ) + + wrapper = BaseAzureLLM().get_azure_openai_client( + api_key="not-a-real-key", + api_base="https://litellm.openai.azure.com", + api_version="2024-02-01", + litellm_params={}, + _is_async=True, + ) + + assert wrapper is not None + assert wrapper._client is shared_session, "the wrapper should be built on the caller's session" + + closer.schedule(wrapper) + closer.reap() + + assert closer.pending_count == 0, "a wrapper around the caller's session must never be queued" + assert shared_session.is_closed is False, "closed the session the caller configured" + + +def test_an_azure_client_litellm_built_its_own_http_client_for_is_still_closed(monkeypatch): + """The ownership check must not turn the reclaim off for the ordinary case.""" + from litellm.caching.evicted_client_closer import EvictedClientCloser + from litellm.caching.llm_caching_handler import LLMClientCache + + closer = EvictedClientCloser(grace_seconds=0.0) + monkeypatch.setattr(litellm, "aclient_session", None) + monkeypatch.setattr(litellm, "client_session", None) + monkeypatch.setattr( + litellm, + "in_memory_llm_clients_cache", + LLMClientCache(evicted_client_closer=closer), + ) + + wrapper = BaseAzureLLM().get_azure_openai_client( + api_key="not-a-real-key", + api_base="https://litellm.openai.azure.com", + api_version="2024-02-01", + litellm_params={}, + _is_async=False, + ) + + assert wrapper is not None + closer.schedule(wrapper) + + assert closer.pending_count == 1, "litellm built this client's http client, so it owns it" + + closer.reap() + + assert wrapper.is_closed() is True diff --git a/tests/test_litellm/llms/openai/test_openai_common_utils.py b/tests/test_litellm/llms/openai/test_openai_common_utils.py index ce25f7e9af6..a099b5c659f 100644 --- a/tests/test_litellm/llms/openai/test_openai_common_utils.py +++ b/tests/test_litellm/llms/openai/test_openai_common_utils.py @@ -175,3 +175,75 @@ def test_get_openai_client_cache_key(client_type): ) assert isinstance(key, str) assert "api_key=sk-test" in key + + +def test_evicting_a_client_built_on_the_callers_session_leaves_that_session_open(monkeypatch): + """`litellm.aclient_session` belongs to the caller, who goes on using it. + + `_get_async_http_client` hands that session straight back, so the SDK client + litellm builds around it is only a wrapper. The SDK's `close()` closes + whatever http client it was given, so treating the wrapper as litellm's to + close would close the caller's shared session out from under them. + """ + import httpx + + from litellm.caching.evicted_client_closer import EvictedClientCloser + from litellm.caching.llm_caching_handler import LLMClientCache + from litellm.llms.openai.openai import OpenAIChatCompletion + + shared_session = httpx.AsyncClient() + closer = EvictedClientCloser(grace_seconds=0.0) + monkeypatch.setattr(litellm, "aclient_session", shared_session) + monkeypatch.setattr( + litellm, + "in_memory_llm_clients_cache", + LLMClientCache(evicted_client_closer=closer), + ) + + wrapper = OpenAIChatCompletion()._get_openai_client( + is_async=True, + api_key="sk-not-a-real-key", + api_base="https://api.openai.com/v1", + max_retries=2, + ) + + assert wrapper is not None + assert wrapper._client is shared_session, "the wrapper should be built on the caller's session" + + closer.schedule(wrapper) + closer.reap() + + assert closer.pending_count == 0, "a wrapper around the caller's session must never be queued" + assert shared_session.is_closed is False, "closed the session the caller configured" + + +def test_a_client_litellm_built_its_own_http_client_for_is_still_closed(monkeypatch): + """The ownership check must not turn the reclaim off for the ordinary case.""" + from litellm.caching.evicted_client_closer import EvictedClientCloser + from litellm.caching.llm_caching_handler import LLMClientCache + from litellm.llms.openai.openai import OpenAIChatCompletion + + closer = EvictedClientCloser(grace_seconds=0.0) + monkeypatch.setattr(litellm, "aclient_session", None) + monkeypatch.setattr(litellm, "client_session", None) + monkeypatch.setattr( + litellm, + "in_memory_llm_clients_cache", + LLMClientCache(evicted_client_closer=closer), + ) + + wrapper = OpenAIChatCompletion()._get_openai_client( + is_async=False, + api_key="sk-not-a-real-key", + api_base="https://api.openai.com/v1", + max_retries=2, + ) + + assert wrapper is not None + closer.schedule(wrapper) + + assert closer.pending_count == 1, "litellm built this client's http client, so it owns it" + + closer.reap() + + assert wrapper.is_closed() is True From 9d5984b35836c9be130e69d5bcd37eeeb1796148 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 3 Aug 2026 13:39:07 -0700 Subject: [PATCH 045/124] refactor(ui): rename the create MCP server component to PascalCase (#35686) Pure rename, no behavior change. create_mcp_server.tsx and its test move to CreateMCPServer, the two importers and one stale e2e comment follow, and the local/filename-pascal-case suppression drops now that the file passes the rule on its own. The rename is scoped to this one component rather than the whole directory because three PRs are currently open against its snake_case siblings; the rest can follow once those land. --- tests/e2e/ui/tests/mcp/mcpServers.spec.ts | 2 +- ui/litellm-dashboard/eslint-suppressions.json | 31 +++++++++---------- ...rver.test.tsx => CreateMCPServer.test.tsx} | 2 +- ...ate_mcp_server.tsx => CreateMCPServer.tsx} | 0 .../mcp-servers/_components/mcp_discovery.tsx | 2 +- .../mcp-servers/_components/mcp_servers.tsx | 2 +- 6 files changed, 18 insertions(+), 21 deletions(-) rename ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/{create_mcp_server.test.tsx => CreateMCPServer.test.tsx} (99%) rename ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/{create_mcp_server.tsx => CreateMCPServer.tsx} (100%) diff --git a/tests/e2e/ui/tests/mcp/mcpServers.spec.ts b/tests/e2e/ui/tests/mcp/mcpServers.spec.ts index 37aabf9c057..43f21e77fbd 100644 --- a/tests/e2e/ui/tests/mcp/mcpServers.spec.ts +++ b/tests/e2e/ui/tests/mcp/mcpServers.spec.ts @@ -36,7 +36,7 @@ test.describe("MCP Servers", () => { await formModal.locator('input[id="url"]').fill("https://e2e-fake-mcp.test.local/mcp"); // Authentication: None - // The auth_type Form.Item has no label prop (create_mcp_server.tsx:795), so + // The auth_type Form.Item has no label prop (CreateMCPServer.tsx), so // it can't be anchored by label text. Scope via the enclosing Collapse // panel ("Authentication") instead — that anchor is stable even if the // placeholder copy changes. diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 8c6d940cfea..6c67f593aec 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -779,6 +779,20 @@ "count": 1 } }, + "src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx": { + "max-lines": { + "count": 1 + }, + "no-nested-ternary": { + "count": 1 + }, + "no-restricted-imports": { + "count": 2 + }, + "react-hooks/set-state-in-effect": { + "count": 4 + } + }, "src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx": { "no-restricted-imports": { "count": 1 @@ -900,23 +914,6 @@ "count": 1 } }, - "src/app/(dashboard)/mcp-servers/_components/create_mcp_server.tsx": { - "local/filename-pascal-case": { - "count": 1 - }, - "max-lines": { - "count": 1 - }, - "no-nested-ternary": { - "count": 1 - }, - "no-restricted-imports": { - "count": 2 - }, - "react-hooks/set-state-in-effect": { - "count": 4 - } - }, "src/app/(dashboard)/mcp-servers/_components/index.tsx": { "local/filename-pascal-case": { "count": 1 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/create_mcp_server.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.test.tsx similarity index 99% rename from ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/create_mcp_server.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.test.tsx index e6b70e170d2..45da71ed301 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/create_mcp_server.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.test.tsx @@ -3,7 +3,7 @@ import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; import * as networking from "@/components/networking"; import { setToken } from "@/utils/mcpTokenStore"; -import CreateMCPServer from "./create_mcp_server"; +import CreateMCPServer from "./CreateMCPServer"; import { selectAntOption } from "./testUtils"; vi.mock("@/components/networking", () => ({ diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/create_mcp_server.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/create_mcp_server.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_discovery.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_discovery.tsx index 5094f1a6761..b0b8b2eed93 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_discovery.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_discovery.tsx @@ -7,7 +7,7 @@ import { Skeleton } from "@/components/ui/skeleton"; import { cn } from "@/lib/cva.config"; import { fetchDiscoverableMCPServers } from "@/components/networking"; import { DiscoverableMCPServer, DiscoverMCPServersResponse } from "@/components/mcp_tools/types"; -import { mcpLogoImg } from "./create_mcp_server"; +import { mcpLogoImg } from "./CreateMCPServer"; import { resolveLogoSrc } from "@/lib/assetPaths"; interface MCPDiscoveryProps { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx index ebb1d710bf2..193246aab3c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx @@ -24,7 +24,7 @@ import NotificationsManager from "@/components/molecules/notifications_manager"; import { deleteMCPServer } from "@/components/networking"; import { MCPSubmissionsTab } from "./MCPSubmissionsTab"; import { MCPToolsetsTab } from "./MCPToolsetsTab"; -import CreateMCPServer from "./create_mcp_server"; +import CreateMCPServer from "./CreateMCPServer"; import MCPConnect from "./mcp_connect"; import MCPServerCard from "./MCPServerCard"; import { MCPServerView } from "./mcp_server_view"; From b03803b9189ba38d3d9805d5e342793ea721dc05 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 3 Aug 2026 13:21:52 -0700 Subject: [PATCH 046/124] refactor(ui): extract the MCP create form's logic and field groups Pulls four modules out of the 1398-line create component, which drops to 896 lines. No behavior changes: CreateMCPServer.test.tsx is untouched and all 77 of its tests pass against the refactored component, which is the review contract for this PR. createServerPayload.ts is a pure form-values-to-payload function whose failures are a tagged union instead of inline notification calls, so the transformation is reachable without a DOM. createOAuthUiState.ts owns the snapshot that survives the OAuth authorize redirect, keeping every presence guard the inline version had. AwsSigV4Fields and OpenApiByokFields are the two largest JSX blocks, moved verbatim so they can be diffed as moves. The create/edit setToken divergence, the mcpLogoImg export, and the untyped form-values bag are left alone on purpose; each is a behavior or cross-file change that does not belong in a move. --- ui/litellm-dashboard/eslint-suppressions.json | 10 + .../_components/AwsSigV4Fields.tsx | 155 +++++ .../_components/CreateMCPServer.tsx | 583 +++--------------- .../_components/OpenApiByokFields.tsx | 91 +++ .../_components/createOAuthUiState.ts | 96 +++ .../_components/createServerPayload.ts | 233 +++++++ 6 files changed, 666 insertions(+), 502 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createServerPayload.ts diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 6c67f593aec..107f66b8f1a 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -779,6 +779,11 @@ "count": 1 } }, + "src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx": { + "no-restricted-imports": { + "count": 1 + } + }, "src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx": { "max-lines": { "count": 1 @@ -870,6 +875,11 @@ "count": 1 } }, + "src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx": { + "no-restricted-imports": { + "count": 1 + } + }, "src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.test.tsx": { "no-restricted-imports": { "count": 1 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx new file mode 100644 index 00000000000..d4ae537bffa --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx @@ -0,0 +1,155 @@ +import React from "react"; +import { Form, Input, Tooltip } from "antd"; +import { InfoCircleOutlined } from "@ant-design/icons"; + +const AwsSigV4Fields: React.FC = () => ( + <> +

+ For MCP servers hosted on AWS Bedrock AgentCore.{" "} + + View docs → + +

+ + AWS Region + + + + + } + name={["credentials", "aws_region_name"]} + rules={[{ required: true, message: "AWS region is required for SigV4 auth" }]} + > + + + + AWS Service Name + + + + + } + name={["credentials", "aws_service_name"]} + > + + + + AWS Access Key ID + + + + + } + name={["credentials", "aws_access_key_id"]} + dependencies={[["credentials", "aws_secret_access_key"]]} + rules={[ + ({ getFieldValue }) => ({ + validator(_, value) { + const secretKey = getFieldValue(["credentials", "aws_secret_access_key"]); + if (secretKey && !value) { + return Promise.reject(new Error("Access Key ID is required when Secret Access Key is provided")); + } + return Promise.resolve(); + }, + }), + ]} + > + + + + AWS Secret Access Key + + + + + } + name={["credentials", "aws_secret_access_key"]} + dependencies={[["credentials", "aws_access_key_id"]]} + rules={[ + ({ getFieldValue }) => ({ + validator(_, value) { + const accessKeyId = getFieldValue(["credentials", "aws_access_key_id"]); + if (accessKeyId && !value) { + return Promise.reject(new Error("Secret Access Key is required when Access Key ID is provided")); + } + return Promise.resolve(); + }, + }), + ]} + > + + + + AWS Session Token + + + + + } + name={["credentials", "aws_session_token"]} + > + + + + AWS Role ARN + + + + + } + name={["credentials", "aws_role_name"]} + > + + + + AWS Session Name + + + + + } + name={["credentials", "aws_session_name"]} + > + + + +); + +export default AwsSigV4Fields; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx index 1d0262acdca..0785dd142ff 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx @@ -1,5 +1,5 @@ import React, { useState } from "react"; -import { Modal, Tooltip, Form, Select, Input, InputNumber, Switch, Collapse } from "antd"; +import { Modal, Tooltip, Form, Select, Input, InputNumber, Collapse } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; import { Button, TextInput } from "@tremor/react"; import { createMCPServer, registerMCPServer, storeMCPOAuthUserCredential } from "@/components/networking"; @@ -13,15 +13,22 @@ import { TRANSPORT, getMcpOAuthMode, MCP_OAUTH2_FLOW_M2M, - MCP_OAUTH2_FLOW_INTERACTIVE, isClientForwardedTokenMode, getOAuthAuthorizationIdentity, CLEARED_ON_INVALIDATION, isHeldOAuthTokenStale, preservedAdminCredentials, preservedDeclaredAppCredentials, - withoutMintedTokenCredentials, } from "@/components/mcp_tools/types"; +import { + AUTH_TYPES_REQUIRING_AUTH_VALUE, + BuildCreatePayloadResult, + buildCreateServerPayload, + reduceStaticHeaders, +} from "./createServerPayload"; +import { readCreateUiSnapshot, writeCreateUiSnapshot } from "./createOAuthUiState"; +import AwsSigV4Fields from "./AwsSigV4Fields"; +import OpenApiByokFields from "./OpenApiByokFields"; import OAuthFormFields from "./OAuthFormFields"; import TruePassthroughWarning from "./TruePassthroughWarning"; import PassthroughAuthorizeSection from "./PassthroughAuthorizeSection"; @@ -36,11 +43,10 @@ import OpenAPIFormSection, { OpenAPIKeyTool } from "./OpenAPIFormSection"; import MCPLogoSelector from "./MCPLogoSelector"; import EnvVarsSection from "./EnvVarsSection"; import { isAdminRole } from "@/utils/roles"; -import { validateMCPServerUrl, validateMCPServerName, normalizeEnvVars, TOOL_DISPLAY_NAME_PATTERN } from "./utils"; +import { validateMCPServerUrl, validateMCPServerName } from "./utils"; import NotificationsManager from "@/components/molecules/notifications_manager"; import { useMcpOAuthFlow } from "@/hooks/useMcpOAuthFlow"; import { useTestMCPConnection } from "@/hooks/useTestMCPConnection"; -import { getSecureItem, setSecureItem } from "@/utils/secureStorage"; import mcpLogo from "../../../../../public/assets/logos/mcp_logo.png"; export const mcpLogoImg = mcpLogo.src; @@ -57,25 +63,15 @@ interface CreateMCPServerProps { onBackToDiscovery?: () => void; } -const AUTH_TYPES_REQUIRING_AUTH_VALUE = [AUTH_TYPE.API_KEY, AUTH_TYPE.BEARER_TOKEN, AUTH_TYPE.TOKEN, AUTH_TYPE.BASIC]; -const AUTH_TYPES_REQUIRING_CREDENTIALS = [ - ...AUTH_TYPES_REQUIRING_AUTH_VALUE, - AUTH_TYPE.OAUTH2, - AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE, - AUTH_TYPE.OAUTH2_ID_JAG, - AUTH_TYPE.AWS_SIGV4, - AUTH_TYPE.TRUE_PASSTHROUGH, - AUTH_TYPE.OAUTH_DELEGATE, -]; -const CREATE_OAUTH_UI_STATE_KEY = "litellm-mcp-oauth-create-state"; - -const reduceStaticHeaders = (list: unknown): Record => { - if (!Array.isArray(list)) return {}; - return list.reduce((acc: Record, entry: Record) => { - const header = entry?.header?.trim(); - if (header) acc[header] = (entry?.value ?? "").trim(); - return acc; - }, {}); +const payloadErrorMessage = (result: Exclude): string => { + switch (result.kind) { + case "invalid_tool_display_name": + return `Tool display name "${result.displayName}" is invalid. Only letters, digits, underscores, and hyphens are allowed (no spaces).`; + case "invalid_stdio_json": + return "Invalid JSON in stdio configuration"; + case "invalid_token_validation_json": + return "Invalid JSON in Token Validation Rules"; + } }; const CreateMCPServer: React.FC = ({ @@ -147,29 +143,18 @@ const CreateMCPServer: React.FC = ({ const isM2MFlow = isOAuthAuthType && formValues.oauth_flow_type === OAUTH_FLOW.M2M; const persistCreateUiState = () => { - if (typeof window === "undefined") { - return; - } - try { - const values = form.getFieldsValue(true); - const uiState = { - modalVisible: isModalVisible, - formValues: values, - transportType, - costConfig, - allowedTools, - hasToolAllowlistInteraction, - searchValue, - aliasManuallyEdited, - logoUrl, - // Persist the identity so invalidation stays armed across the OAuth redirect round trip: a - // post-restore url/mode edit must still discard the held token instead of silently keeping it. - authorizedIdentity, - }; - setSecureItem(CREATE_OAUTH_UI_STATE_KEY, JSON.stringify(uiState)); - } catch (err) { - console.warn("Failed to persist MCP create state", err); - } + writeCreateUiSnapshot({ + modalVisible: isModalVisible, + formValues: form.getFieldsValue(true), + transportType, + costConfig, + allowedTools, + hasToolAllowlistInteraction, + searchValue, + aliasManuallyEdited, + logoUrl, + authorizedIdentity, + }); }; const { @@ -308,59 +293,39 @@ const CreateMCPServer: React.FC = ({ }; React.useEffect(() => { - if (typeof window === "undefined") { + const restored = readCreateUiSnapshot(); + if (!restored) { return; } - const storedState = getSecureItem(CREATE_OAUTH_UI_STATE_KEY); - if (!storedState) { - return; + if (restored.modalVisible) { + setModalVisible(true); } - - try { - const parsed = JSON.parse(storedState); - if (parsed.modalVisible) { - setModalVisible(true); - } - const restoredTransport = parsed.formValues?.transport || parsed.transportType || ""; - if (restoredTransport) { - setTransportType(restoredTransport); - } - if (parsed.formValues) { - // Assign the cleaned credentials (strip minted token material so a stale token never rehydrates); - // the declared app the admin typed is kept. Create has no server-side stored app to merge. - const restoredValues = { - ...parsed.formValues, - credentials: withoutMintedTokenCredentials(parsed.formValues.credentials), - }; - setPendingRestoredValues({ values: restoredValues, transport: restoredTransport }); - } - if (typeof parsed.authorizedIdentity === "string") { - // Re-arm invalidation: without this the remounted form has authorizedIdentity=undefined, so a - // post-restore mode/url edit would never fire the stale-token discard. - setAuthorizedIdentity(parsed.authorizedIdentity); - } - if (parsed.costConfig) { - setCostConfig(parsed.costConfig); - } - if (parsed.allowedTools) { - setAllowedTools(parsed.allowedTools); - } - if (typeof parsed.hasToolAllowlistInteraction === "boolean") { - setHasToolAllowlistInteraction(parsed.hasToolAllowlistInteraction); - } - if (parsed.searchValue) { - setSearchValue(parsed.searchValue); - } - if (typeof parsed.aliasManuallyEdited === "boolean") { - setAliasManuallyEdited(parsed.aliasManuallyEdited); - } - if (parsed.logoUrl) { - setLogoUrl(parsed.logoUrl); - } - } catch (err) { - console.error("Failed to restore MCP create state", err); - } finally { - window.sessionStorage.removeItem(CREATE_OAUTH_UI_STATE_KEY); + if (restored.transportType) { + setTransportType(restored.transportType); + } + if (restored.formValues) { + setPendingRestoredValues({ values: restored.formValues, transport: restored.transportType }); + } + if (restored.authorizedIdentity !== undefined) { + setAuthorizedIdentity(restored.authorizedIdentity); + } + if (restored.costConfig) { + setCostConfig(restored.costConfig); + } + if (restored.allowedTools) { + setAllowedTools([...restored.allowedTools]); + } + if (restored.hasToolAllowlistInteraction !== undefined) { + setHasToolAllowlistInteraction(restored.hasToolAllowlistInteraction); + } + if (restored.searchValue) { + setSearchValue(restored.searchValue); + } + if (restored.aliasManuallyEdited !== undefined) { + setAliasManuallyEdited(restored.aliasManuallyEdited); + } + if (restored.logoUrl) { + setLogoUrl(restored.logoUrl); } }, [form, setModalVisible]); @@ -422,169 +387,25 @@ const CreateMCPServer: React.FC = ({ setAliasManuallyEdited(false); }, [isModalVisible, prefillData, form]); - const handleCreate = async (values: Record) => { - const invalidDisplayName = Object.entries(toolNameToDisplayName).find( - ([, displayName]) => displayName && !TOOL_DISPLAY_NAME_PATTERN.test(displayName), - ); - if (invalidDisplayName) { - NotificationsManager.fromBackend( - `Tool display name "${invalidDisplayName[1]}" is invalid. Only letters, digits, underscores, and hyphens are allowed (no spaces).`, - ); + const handleCreate = async (values: Record) => { + const built = buildCreateServerPayload(values, { + transportType, + costConfig, + allowedTools, + hasToolAllowlistInteraction, + toolNameToDisplayName, + toolNameToDescription, + logoUrl, + dcrClient: dcrClientRef.current, + }); + if (built.kind !== "ok") { + NotificationsManager.fromBackend(payloadErrorMessage(built)); return; } + const payload = built.payload; + setIsLoading(true); try { - const { - static_headers: staticHeadersList, - env_vars: envVarsList, - stdio_config: rawStdioConfig, - credentials: credentialValues, - allow_all_keys: allowAllKeysRaw, - available_on_public_internet: availableOnPublicInternetRaw, - delegate_auth_to_upstream: delegateAuthToUpstreamRaw, - oauth_passthrough: oauthPassthroughRaw, - dcr_bridge: dcrBridgeRaw, - token_validation_json: rawTokenValidationJson, - ...restValues - } = values; - - // Transform access groups into objects with name property - const accessGroups = restValues.mcp_access_groups; - - const staticHeaders = reduceStaticHeaders(staticHeadersList); - const envVars = normalizeEnvVars(envVarsList); - - const credentialsPayload = - credentialValues && typeof credentialValues === "object" - ? Object.entries(credentialValues).reduce((acc: Record, [key, value]) => { - if (value === undefined || value === null || value === "") { - return acc; - } - if (key === "scopes") { - if (Array.isArray(value)) { - const filteredScopes = value.filter((scope) => scope != null && scope !== ""); - if (filteredScopes.length > 0) { - acc[key] = filteredScopes; - } - } - } else { - acc[key] = value; - } - return acc; - }, {}) - : undefined; - - // Process stdio configuration if present - let stdioFields = {}; - if (rawStdioConfig && transportType === "stdio") { - try { - const stdioConfig = JSON.parse(rawStdioConfig); - - // Handle both formats: - // 1. Full mcpServers structure: {"mcpServers": {"server-name": {...}}} - // 2. Direct config: {"command": "...", "args": [...], "env": {...}} - - let actualConfig = stdioConfig; - - // If it's the full mcpServers structure, extract the first server config - if (stdioConfig.mcpServers && typeof stdioConfig.mcpServers === "object") { - const serverNames = Object.keys(stdioConfig.mcpServers); - if (serverNames.length > 0) { - const firstServerName = serverNames[0]; - actualConfig = stdioConfig.mcpServers[firstServerName]; - - // If no alias is provided, use the server name from the JSON - if (!restValues.server_name) { - restValues.server_name = firstServerName.replace(/-/g, "_"); // Replace hyphens with underscores - } - } - } - - stdioFields = { - command: actualConfig.command, - args: actualConfig.args, - env: actualConfig.env, - }; - } catch (error) { - NotificationsManager.fromBackend("Invalid JSON in stdio configuration"); - return; - } - } - - // Map "openapi" transport to "http" for the backend - if (restValues.transport === TRANSPORT.OPENAPI) { - restValues.transport = "http"; - } - - // Parse token_validation JSON if provided - let tokenValidation: Record | null = null; - if (rawTokenValidationJson && rawTokenValidationJson.trim() !== "") { - try { - tokenValidation = JSON.parse(rawTokenValidationJson); - } catch { - NotificationsManager.fromBackend("Invalid JSON in Token Validation Rules"); - setIsLoading(false); - return; - } - } - - // Prepare the payload with cost configuration and allowed tools - const payload: Record = { - ...restValues, - ...stdioFields, - // Remove the raw stdio_config field as we've extracted its components - stdio_config: undefined, - mcp_info: { - server_name: restValues.server_name || restValues.url, - description: restValues.description, - logo_url: logoUrl || undefined, - mcp_server_cost_info: Object.keys(costConfig).length > 0 ? costConfig : null, - tool_allowlist_enforced: hasToolAllowlistInteraction || allowedTools.length > 0, - }, - mcp_access_groups: accessGroups, - alias: restValues.alias, - allowed_tools: allowedTools, - tool_name_to_display_name: toolNameToDisplayName, - tool_name_to_description: toolNameToDescription, - allow_all_keys: Boolean(allowAllKeysRaw), - available_on_public_internet: Boolean(availableOnPublicInternetRaw), - delegate_auth_to_upstream: Boolean(delegateAuthToUpstreamRaw), - oauth_passthrough: Boolean(oauthPassthroughRaw), - // ``dcr_bridge`` is only meaningful for the client-forwarded token - // modes (true_passthrough / oauth_delegate) and defaults on when the - // toggle is shown; force false for any other auth type so a stale - // ``true`` is never persisted. Mirrors the sibling flags above. - dcr_bridge: isClientForwardedTokenMode(restValues.auth_type) ? Boolean(dcrBridgeRaw ?? true) : false, - ...(restValues.auth_type === AUTH_TYPE.OAUTH2 - ? { - oauth2_flow: - values.oauth_flow_type === OAUTH_FLOW.M2M ? MCP_OAUTH2_FLOW_M2M : MCP_OAUTH2_FLOW_INTERACTIVE, - } - : {}), - static_headers: staticHeaders, - env_vars: envVars, - ...(tokenValidation !== null && { token_validation: tokenValidation }), - }; - - const includeCredentials = - restValues.auth_type && AUTH_TYPES_REQUIRING_CREDENTIALS.includes(restValues.auth_type); - - // Client-forwarded rows persist ONLY the declared app; strip any token material that lingered in - // the form (e.g. from a prior oauth2 authorize on the same session) so it can never reach the row. - const submitCredentials = isClientForwardedTokenMode(restValues.auth_type) - ? preservedAdminCredentials(credentialsPayload) - : credentialsPayload; - - if (includeCredentials && submitCredentials && Object.keys(submitCredentials).length > 0) { - payload.credentials = submitCredentials; - } - - // An interactive (oauth2) create persists its DCR-minted client from the ref (kept out of the - // form store); reuse a re-authorize's registered client instead of re-registering. - if (restValues.auth_type === AUTH_TYPE.OAUTH2 && dcrClientRef.current) { - payload.credentials = { ...(payload.credentials ?? {}), ...dcrClientRef.current }; - } - if (accessToken != null) { const response = isAdmin ? await createMCPServer(accessToken, payload) @@ -596,9 +417,9 @@ const CreateMCPServer: React.FC = ({ // forwards a browser-held token, so it stays in sessionStorage only. if (oauthTokenResponse?.access_token && response?.server_id) { const oauthMode = getMcpOAuthMode({ - auth_type: restValues.auth_type, + auth_type: values.auth_type as string | undefined, oauth2_flow: values.oauth_flow_type === OAUTH_FLOW.M2M ? MCP_OAUTH2_FLOW_M2M : null, - delegate_auth_to_upstream: Boolean(delegateAuthToUpstreamRaw), + delegate_auth_to_upstream: Boolean(values.delegate_auth_to_upstream), }); if (oauthMode === "authorization_code") { const scope = oauthTokenResponse.scope; @@ -953,94 +774,7 @@ const CreateMCPServer: React.FC = ({ )} {/* BYOK toggle - only for OpenAPI */} - {transportType === TRANSPORT.OPENAPI && ( - <> - - BYOK (Bring Your Own Key) - - - - - } - name="is_byok" - valuePropName="checked" - > - - - - prev.is_byok !== cur.is_byok || prev.auth_type !== cur.auth_type} - > - {({ getFieldValue }) => - getFieldValue("is_byok") ? ( - <> - {/* Auth format hint */} - {getFieldValue("auth_type") && getFieldValue("auth_type") !== "none" && ( -
- - - User keys will be sent as:{" "} - - {getFieldValue("auth_type") === "bearer_token" && "Authorization: Bearer {key}"} - {getFieldValue("auth_type") === "token" && "Authorization: token {key}"} - {getFieldValue("auth_type") === "api_key" && "x-api-key: {key}"} - {getFieldValue("auth_type") === "basic" && "Authorization: Basic {key}"} - {getFieldValue("auth_type") === "authorization" && "Authorization: {key}"} - - {!getFieldValue("auth_type") && "Set Authentication Type below to specify the format."} - -
- )} - {!getFieldValue("auth_type") && ( -
- - - Set the Authentication Type below to specify how user keys are sent - (e.g., Bearer Token, API Key header). - -
- )} - - Access Description - - - - - } - name="byok_description" - > - - - - ) : null - } - - - )} + {transportType === TRANSPORT.OPENAPI && } = ({ /> )} - {transportType !== "stdio" && transportType !== "" && isAwsSigV4AuthType && ( - <> -

- For MCP servers hosted on AWS Bedrock AgentCore.{" "} - - View docs → - -

- - AWS Region - - - - - } - name={["credentials", "aws_region_name"]} - rules={[{ required: true, message: "AWS region is required for SigV4 auth" }]} - > - - - - AWS Service Name - - - - - } - name={["credentials", "aws_service_name"]} - > - - - - AWS Access Key ID - - - - - } - name={["credentials", "aws_access_key_id"]} - dependencies={[["credentials", "aws_secret_access_key"]]} - rules={[ - ({ getFieldValue }) => ({ - validator(_, value) { - const secretKey = getFieldValue(["credentials", "aws_secret_access_key"]); - if (secretKey && !value) { - return Promise.reject( - new Error("Access Key ID is required when Secret Access Key is provided"), - ); - } - return Promise.resolve(); - }, - }), - ]} - > - - - - AWS Secret Access Key - - - - - } - name={["credentials", "aws_secret_access_key"]} - dependencies={[["credentials", "aws_access_key_id"]]} - rules={[ - ({ getFieldValue }) => ({ - validator(_, value) { - const accessKeyId = getFieldValue(["credentials", "aws_access_key_id"]); - if (accessKeyId && !value) { - return Promise.reject( - new Error("Secret Access Key is required when Access Key ID is provided"), - ); - } - return Promise.resolve(); - }, - }), - ]} - > - - - - AWS Session Token - - - - - } - name={["credentials", "aws_session_token"]} - > - - - - AWS Role ARN - - - - - } - name={["credentials", "aws_role_name"]} - > - - - - AWS Session Name - - - - - } - name={["credentials", "aws_session_name"]} - > - - - - )} + {transportType !== "stdio" && transportType !== "" && isAwsSigV4AuthType && } {/* Stdio Configuration - only show for stdio transport */} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx new file mode 100644 index 00000000000..2ac4279e20a --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx @@ -0,0 +1,91 @@ +import React from "react"; +import { Form, Input, Select, Switch, Tooltip } from "antd"; +import { InfoCircleOutlined } from "@ant-design/icons"; + +const OpenApiByokFields: React.FC = () => ( + <> + + BYOK (Bring Your Own Key) + + + + + } + name="is_byok" + valuePropName="checked" + > + + + + prev.is_byok !== cur.is_byok || prev.auth_type !== cur.auth_type}> + {({ getFieldValue }) => + getFieldValue("is_byok") ? ( + <> + {/* Auth format hint */} + {getFieldValue("auth_type") && getFieldValue("auth_type") !== "none" && ( +
+ + + User keys will be sent as:{" "} + + {getFieldValue("auth_type") === "bearer_token" && "Authorization: Bearer {key}"} + {getFieldValue("auth_type") === "token" && "Authorization: token {key}"} + {getFieldValue("auth_type") === "api_key" && "x-api-key: {key}"} + {getFieldValue("auth_type") === "basic" && "Authorization: Basic {key}"} + {getFieldValue("auth_type") === "authorization" && "Authorization: {key}"} + + {!getFieldValue("auth_type") && "Set Authentication Type below to specify the format."} + +
+ )} + {!getFieldValue("auth_type") && ( +
+ + + Set the Authentication Type below to specify how user keys are sent (e.g., Bearer + Token, API Key header). + +
+ )} + + Access Description + + + + + } + name="byok_description" + > + + + + ) : null + } + + +); + +export default OpenApiByokFields; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.ts new file mode 100644 index 00000000000..f6475b97830 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.ts @@ -0,0 +1,96 @@ +import { MCPServerCostInfo, withoutMintedTokenCredentials } from "@/components/mcp_tools/types"; +import { getSecureItem, setSecureItem } from "@/utils/secureStorage"; + +const CREATE_OAUTH_UI_STATE_KEY = "litellm-mcp-oauth-create-state"; + +// Everything the create modal needs to look untouched after the OAuth authorize redirect reloads the +// page. `authorizedIdentity` is part of it so invalidation stays armed across the round trip: without +// it the remounted form starts with no identity, and a post-restore url/mode edit would never fire the +// stale-token discard. +export interface CreateUiSnapshot { + readonly modalVisible: boolean; + readonly formValues: Record; + readonly transportType: string; + readonly costConfig: MCPServerCostInfo; + readonly allowedTools: readonly string[]; + readonly hasToolAllowlistInteraction: boolean; + readonly searchValue: string; + readonly aliasManuallyEdited: boolean; + readonly logoUrl: string | undefined; + readonly authorizedIdentity: string | undefined; +} + +// Only the fields that survived their own presence check. A key absent here means "leave the freshly +// mounted state alone", which is why every field is optional rather than defaulted. +export type RestoredUiSnapshot = { + readonly modalVisible?: boolean; + readonly formValues?: Record; + readonly transportType?: string; + readonly costConfig?: MCPServerCostInfo; + readonly allowedTools?: readonly string[]; + readonly hasToolAllowlistInteraction?: boolean; + readonly searchValue?: string; + readonly aliasManuallyEdited?: boolean; + readonly logoUrl?: string; + readonly authorizedIdentity?: string; +}; + +export const writeCreateUiSnapshot = (snapshot: CreateUiSnapshot): void => { + if (typeof window === "undefined") { + return; + } + try { + setSecureItem(CREATE_OAUTH_UI_STATE_KEY, JSON.stringify(snapshot)); + } catch (err) { + console.warn("Failed to persist MCP create state", err); + } +}; + +/** + * Read and validate the snapshot left before the authorize redirect, then drop it so a later mount + * cannot replay it. Returns null when there is nothing to restore (or the payload was unparseable), + * in which case the stored value is left in place for an in-flight flow to time out naturally. + */ +export const readCreateUiSnapshot = (): RestoredUiSnapshot | null => { + if (typeof window === "undefined") { + return null; + } + const storedState = getSecureItem(CREATE_OAUTH_UI_STATE_KEY); + if (!storedState) { + return null; + } + + try { + const parsed = JSON.parse(storedState); + const restoredTransport = parsed.formValues?.transport || parsed.transportType || ""; + + return { + ...(parsed.modalVisible ? { modalVisible: true } : {}), + ...(restoredTransport ? { transportType: restoredTransport } : {}), + ...(parsed.formValues + ? { + // Strip minted token material so a stale token never rehydrates; the declared app the + // admin typed is kept. Create has no server-side stored app to merge. + formValues: { + ...parsed.formValues, + credentials: withoutMintedTokenCredentials(parsed.formValues.credentials), + }, + } + : {}), + ...(typeof parsed.authorizedIdentity === "string" ? { authorizedIdentity: parsed.authorizedIdentity } : {}), + ...(parsed.costConfig ? { costConfig: parsed.costConfig } : {}), + ...(parsed.allowedTools ? { allowedTools: parsed.allowedTools } : {}), + ...(typeof parsed.hasToolAllowlistInteraction === "boolean" + ? { hasToolAllowlistInteraction: parsed.hasToolAllowlistInteraction } + : {}), + ...(parsed.searchValue ? { searchValue: parsed.searchValue } : {}), + ...(typeof parsed.aliasManuallyEdited === "boolean" ? { aliasManuallyEdited: parsed.aliasManuallyEdited } : {}), + ...(parsed.logoUrl ? { logoUrl: parsed.logoUrl } : {}), + }; + } catch (err) { + console.error("Failed to restore MCP create state", err); + return null; + } finally { + window.sessionStorage.removeItem(CREATE_OAUTH_UI_STATE_KEY); + } +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createServerPayload.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createServerPayload.ts new file mode 100644 index 00000000000..f45857fcc8b --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createServerPayload.ts @@ -0,0 +1,233 @@ +import { + AUTH_TYPE, + MCPServerCostInfo, + MCP_OAUTH2_FLOW_INTERACTIVE, + MCP_OAUTH2_FLOW_M2M, + OAUTH_FLOW, + TRANSPORT, + isClientForwardedTokenMode, + preservedAdminCredentials, +} from "@/components/mcp_tools/types"; +import { TOOL_DISPLAY_NAME_PATTERN, normalizeEnvVars } from "./utils"; + +export const AUTH_TYPES_REQUIRING_AUTH_VALUE = [ + AUTH_TYPE.API_KEY, + AUTH_TYPE.BEARER_TOKEN, + AUTH_TYPE.TOKEN, + AUTH_TYPE.BASIC, +]; + +export const AUTH_TYPES_REQUIRING_CREDENTIALS = [ + ...AUTH_TYPES_REQUIRING_AUTH_VALUE, + AUTH_TYPE.OAUTH2, + AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE, + AUTH_TYPE.OAUTH2_ID_JAG, + AUTH_TYPE.AWS_SIGV4, + AUTH_TYPE.TRUE_PASSTHROUGH, + AUTH_TYPE.OAUTH_DELEGATE, +]; + +export interface DcrClient { + readonly client_id: string; + readonly client_secret?: string; +} + +export interface CreateServerUiState { + readonly transportType: string; + readonly costConfig: MCPServerCostInfo; + readonly allowedTools: readonly string[]; + readonly hasToolAllowlistInteraction: boolean; + readonly toolNameToDisplayName: Readonly>; + readonly toolNameToDescription: Readonly>; + readonly logoUrl: string | undefined; + readonly dcrClient: DcrClient | null; +} + +export type BuildCreatePayloadResult = + | { readonly kind: "ok"; readonly payload: Record } + | { readonly kind: "invalid_tool_display_name"; readonly displayName: string } + | { readonly kind: "invalid_stdio_json" } + | { readonly kind: "invalid_token_validation_json" }; + +export type StdioParseResult = + | { readonly kind: "ok"; readonly fields: Record; readonly derivedServerName?: string } + | { readonly kind: "invalid" }; + +type JsonParseResult = + | { readonly kind: "ok"; readonly value: Record | null } + | { readonly kind: "invalid" }; + +const tryParseJson = (raw: string): JsonParseResult => { + try { + return { kind: "ok", value: JSON.parse(raw) }; + } catch { + return { kind: "invalid" }; + } +}; + +export const reduceStaticHeaders = (list: unknown): Record => { + if (!Array.isArray(list)) return {}; + return list.reduce((acc: Record, entry: Record) => { + const header = entry?.header?.trim(); + if (header) acc[header] = (entry?.value ?? "").trim(); + return acc; + }, {}); +}; + +// Accepts both the full `{"mcpServers": {"name": {...}}}` shape a user copies out of a client config +// and a bare `{"command": ..., "args": ..., "env": ...}`. A non-object JSON body (null, a number) +// falls through to the invalid branch, which is what the caller surfaces to the admin. +export const parseStdioConfig = (raw: string): StdioParseResult => { + try { + const stdioConfig = JSON.parse(raw); + const nestedName = + stdioConfig.mcpServers && typeof stdioConfig.mcpServers === "object" + ? Object.keys(stdioConfig.mcpServers)[0] + : undefined; + const actualConfig = nestedName === undefined ? stdioConfig : stdioConfig.mcpServers[nestedName]; + + return { + kind: "ok", + fields: { command: actualConfig.command, args: actualConfig.args, env: actualConfig.env }, + // The JSON's own server key is the fallback name when the admin left the field blank. + ...(nestedName === undefined ? {} : { derivedServerName: nestedName.replace(/-/g, "_") }), + }; + } catch { + return { kind: "invalid" }; + } +}; + +const filterCredentials = (credentialValues: unknown): Record | undefined => { + if (!credentialValues || typeof credentialValues !== "object") return undefined; + return Object.entries(credentialValues as Record).reduce( + (acc: Record, [key, value]) => { + if (value === undefined || value === null || value === "") { + return acc; + } + if (key === "scopes") { + if (Array.isArray(value)) { + const filteredScopes = value.filter((scope) => scope != null && scope !== ""); + if (filteredScopes.length > 0) { + acc[key] = filteredScopes; + } + } + } else { + acc[key] = value; + } + return acc; + }, + {}, + ); +}; + +const firstInvalidToolDisplayName = (toolNameToDisplayName: Readonly>): string | undefined => + Object.entries(toolNameToDisplayName).find( + ([, displayName]) => displayName && !TOOL_DISPLAY_NAME_PATTERN.test(displayName), + )?.[1]; + +export const buildCreateServerPayload = ( + values: Record, + ui: CreateServerUiState, +): BuildCreatePayloadResult => { + const badDisplayName = firstInvalidToolDisplayName(ui.toolNameToDisplayName); + if (badDisplayName !== undefined) { + return { kind: "invalid_tool_display_name", displayName: badDisplayName }; + } + + const { + static_headers: staticHeadersList, + env_vars: envVarsList, + stdio_config: rawStdioConfig, + credentials: credentialValues, + allow_all_keys: allowAllKeysRaw, + available_on_public_internet: availableOnPublicInternetRaw, + delegate_auth_to_upstream: delegateAuthToUpstreamRaw, + oauth_passthrough: oauthPassthroughRaw, + dcr_bridge: dcrBridgeRaw, + token_validation_json: rawTokenValidationJson, + ...restValues + } = values; + + const stdio: StdioParseResult = + rawStdioConfig && ui.transportType === "stdio" + ? parseStdioConfig(rawStdioConfig as string) + : { kind: "ok", fields: {} }; + if (stdio.kind === "invalid") { + return { kind: "invalid_stdio_json" }; + } + + const rawTokenValidation = rawTokenValidationJson as string | undefined; + const tokenValidationResult: JsonParseResult = + rawTokenValidation && rawTokenValidation.trim() !== "" + ? tryParseJson(rawTokenValidation) + : { kind: "ok", value: null }; + if (tokenValidationResult.kind === "invalid") { + return { kind: "invalid_token_validation_json" }; + } + const tokenValidation = tokenValidationResult.value; + + const serverName = (restValues.server_name as string | undefined) || stdio.derivedServerName; + // "openapi" is a UI-only transport; the backend stores those servers as plain http. + const transport = restValues.transport === TRANSPORT.OPENAPI ? "http" : restValues.transport; + const authType = restValues.auth_type as string | undefined; + + const credentialsPayload = filterCredentials(credentialValues); + const includeCredentials = authType !== undefined && AUTH_TYPES_REQUIRING_CREDENTIALS.includes(authType); + // Client-forwarded rows persist ONLY the declared app; strip any token material that lingered in + // the form (e.g. from a prior oauth2 authorize on the same session) so it can never reach the row. + const submitCredentials = isClientForwardedTokenMode(authType) + ? preservedAdminCredentials(credentialsPayload) + : credentialsPayload; + const persistedCredentials = + includeCredentials && submitCredentials && Object.keys(submitCredentials).length > 0 + ? submitCredentials + : undefined; + // An interactive (oauth2) create persists its DCR-minted client from the ref (kept out of the + // form store); reuse a re-authorize's registered client instead of re-registering. + const credentials = + authType === AUTH_TYPE.OAUTH2 && ui.dcrClient + ? { ...(persistedCredentials ?? {}), ...ui.dcrClient } + : persistedCredentials; + + return { + kind: "ok", + payload: { + ...restValues, + ...stdio.fields, + ...(serverName === restValues.server_name ? {} : { server_name: serverName }), + ...(transport === restValues.transport ? {} : { transport }), + // Remove the raw stdio_config field as we've extracted its components + stdio_config: undefined, + mcp_info: { + server_name: serverName || restValues.url, + description: restValues.description, + logo_url: ui.logoUrl || undefined, + mcp_server_cost_info: Object.keys(ui.costConfig).length > 0 ? ui.costConfig : null, + tool_allowlist_enforced: ui.hasToolAllowlistInteraction || ui.allowedTools.length > 0, + }, + mcp_access_groups: restValues.mcp_access_groups, + alias: restValues.alias, + allowed_tools: [...ui.allowedTools], + tool_name_to_display_name: ui.toolNameToDisplayName, + tool_name_to_description: ui.toolNameToDescription, + allow_all_keys: Boolean(allowAllKeysRaw), + available_on_public_internet: Boolean(availableOnPublicInternetRaw), + delegate_auth_to_upstream: Boolean(delegateAuthToUpstreamRaw), + oauth_passthrough: Boolean(oauthPassthroughRaw), + // ``dcr_bridge`` is only meaningful for the client-forwarded token + // modes (true_passthrough / oauth_delegate) and defaults on when the + // toggle is shown; force false for any other auth type so a stale + // ``true`` is never persisted. Mirrors the sibling flags above. + dcr_bridge: isClientForwardedTokenMode(authType) ? Boolean(dcrBridgeRaw ?? true) : false, + ...(authType === AUTH_TYPE.OAUTH2 + ? { + oauth2_flow: values.oauth_flow_type === OAUTH_FLOW.M2M ? MCP_OAUTH2_FLOW_M2M : MCP_OAUTH2_FLOW_INTERACTIVE, + } + : {}), + static_headers: reduceStaticHeaders(staticHeadersList), + env_vars: normalizeEnvVars(envVarsList), + ...(tokenValidation !== null && { token_validation: tokenValidation }), + ...(credentials === undefined ? {} : { credentials }), + }, + }; +}; From a6d4654261d97d7757721b1ff5af6e9d38279ad6 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 3 Aug 2026 14:10:35 -0700 Subject: [PATCH 047/124] fix(openai): restore httpx client union type on owns_wrapped_http_client (#35706) PR #35492 was authored before the ruff sweep removed Union from the typing imports in litellm/llms/openai/common_utils.py, so the merge landed an annotation referencing Union without an import. The annotation is evaluated at class-definition time, so importing litellm raises NameError and every test shard on litellm_internal_staging fails at collection. Rewrites the annotation (and the same latent one in openai.py) as httpx.Client | httpx.AsyncClient | None, matching the file's PEP 604 style, so no typing import is needed at all. --- litellm/llms/openai/common_utils.py | 2 +- litellm/llms/openai/openai.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 082764df208..808998ddaf6 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -135,7 +135,7 @@ class BaseOpenAILLM: return _cached_client @staticmethod - def owns_wrapped_http_client(http_client: Optional[Union[httpx.Client, httpx.AsyncClient]]) -> bool: + def owns_wrapped_http_client(http_client: httpx.Client | httpx.AsyncClient | None) -> bool: """Whether litellm may close an SDK client built around ``http_client``. ``_get_async_http_client`` / ``_get_sync_http_client`` hand back diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 7096cdbf8fd..f01730a06a5 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -366,7 +366,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): if cached_client: if isinstance(cached_client, OpenAI) or isinstance(cached_client, AsyncOpenAI): return cached_client - http_client: Optional[Union[httpx.Client, httpx.AsyncClient]] = ( + http_client: httpx.Client | httpx.AsyncClient | None = ( OpenAIChatCompletion._get_async_http_client(shared_session=shared_session) if is_async else OpenAIChatCompletion._get_sync_http_client() From 32eb0720e3b6d9277e500555ba04ee2f5f1fcdff Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:14:58 -0700 Subject: [PATCH 048/124] fix(openai): drop the undefined Union from owns_wrapped_http_client (#35704) Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> From b7843193a0a1aa355105069b06d6bd969652d2b0 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 3 Aug 2026 14:27:40 -0700 Subject: [PATCH 049/124] chore(ui): note Google's Agent Platform rename in vector store setup (#28076) Google Cloud has renamed Vertex AI RAG Engine to "RAG Engine" and Vertex AI Search to "Agent Search" in its console. Users following our setup instructions hit a naming mismatch when they cross-reference the GCP console. Keep "Vertex AI" as the primary term (the generic new names would make our provider UI ambiguous) and surface the new names as secondary asides only where users leave the UI for the console. Resolves LIT-3081 --- .../_components/VectorStoreForm.tsx | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx index 6cdb895b98d..67ba469f68d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx @@ -176,6 +176,10 @@ const VectorStoreForm: React.FC = ({ description={

To use Vertex AI RAG Engine:

+

+ Note: Google Cloud has renamed this to "RAG Engine" in its console — the steps below still + apply. +

  1. Set up your Vertex AI RAG Engine corpus following the guide:{" "} @@ -188,7 +192,9 @@ const VectorStoreForm: React.FC = ({
  2. Create a corpus in your Google Cloud project
    3. -
  3. Note the corpus ID from the Vertex AI console
    4. +
  4. + Note the corpus ID from the Vertex AI console (now labeled "RAG Engine" in Google Cloud) +
  5. Enter the corpus ID in the Vector Store ID field below
@@ -206,6 +212,10 @@ const VectorStoreForm: React.FC = ({ description={

To use Vertex AI Search (Discovery Engine):

+

+ Note: Google Cloud has renamed this to "Agent Search" in its console — the steps below still + apply. +

  1. Enable the Discovery Engine API on your Google Cloud project and create a data store following the @@ -254,11 +264,11 @@ const VectorStoreForm: React.FC = ({ From 8cf2e2eb4385d1b3ea7232865596b81a1522e9de Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Mon, 3 Aug 2026 15:09:47 -0700 Subject: [PATCH 050/124] fix(proxy): apply key/team router_settings.model_group_alias (#35486) Key and team `router_settings.model_group_alias` was accepted, persisted and echoed back by `/key/info`, but never applied at request time, so the request ran on the group the caller asked for. `route_request` forwards only the settings the Router accepts as per-request kwargs, and `model_group_alias` is not one of them: the Router resolves aliases from its own instance attribute, which holds the global config map and is shared across requests. Resolve the alias in the proxy instead, alongside the existing model-alias rewrites and ahead of the pre-call hooks, so per-model limits and guardrails key off the group that actually serves the request. Authorize the alias target before the rewrite; model access was checked against the requested group, so a key whose alias points at a group it cannot call gets the usual 403 rather than being quietly served it. Resolves LIT-4879 --- litellm/proxy/common_request_processing.py | 81 +++++-- litellm/router.py | 12 +- litellm/router_utils/common_utils.py | 21 ++ .../proxy/proxy_server/test_proxy_config.py | 42 ++++ .../proxy/test_common_request_processing.py | 229 ++++++++++++++++++ .../proxy/test_model_level_guardrails.py | 59 +++-- .../test_router_utils_common_utils.py | 42 ++++ 7 files changed, 427 insertions(+), 59 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index cb688860280..bc3bcd233f0 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -43,6 +43,7 @@ from litellm.litellm_core_utils.llm_response_utils.get_headers import ( ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import ProxyException, UserAPIKeyAuth +from litellm.proxy.auth.auth_checks import can_key_call_resolved_model from litellm.proxy.auth.auth_utils import check_response_size_is_safe from litellm.proxy.common_utils.callback_utils import ( get_logging_caching_headers, @@ -53,6 +54,7 @@ from litellm.proxy.route_llm_request import route_request from litellm.proxy.utils import ProxyLogging, _check_and_merge_model_level_guardrails from litellm.router import Router from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict +from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.types.guardrails import GuardrailEventHooks from litellm.types.router import RouterRateLimitError from litellm.types.utils import ServerToolUse @@ -384,6 +386,39 @@ async def _authorize_response_file_search_vector_stores( ) +async def _resolve_per_request_model_group_alias( + requested_model: object, + router_settings: Mapping[str, object], + user_api_key_dict: UserAPIKeyAuth, + llm_router: Router, +) -> str | None: + """ + Resolve ``router_settings.model_group_alias`` coming from a key or team. + + The Router only ever resolves aliases from its own instance attribute, which + holds the global config map and is shared across requests, so a per-request + map has to be applied here instead of being forwarded to the Router. + + Model access was authorized against the requested group, so the target is + authorized in its own right before the rewrite; a key that may not call the + target gets the usual 403 rather than being quietly served it. + + Returns the target model group, or None when no alias applies. + """ + if not isinstance(requested_model, str): + return None + target = resolve_model_group_alias(router_settings.get("model_group_alias"), requested_model) + if target is None or target == requested_model: + return None + await can_key_call_resolved_model( + model=target, + llm_model_list=llm_router.model_list, + valid_token=user_api_key_dict, + llm_router=llm_router, + ) + return target + + async def _parse_event_data_for_error(event_line: str | bytes) -> int | None: """Parses an event line and returns an error code if present, else None.""" event_line = event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line @@ -1285,6 +1320,35 @@ class ProxyBaseLLMRequestProcessing: ): self.data["model"] = user_api_key_dict.aliases[self.data["model"]] + # Apply hierarchical router_settings (Key > Team) + # Global router_settings are already on the Router object itself. + # This sits with the other alias rewrites, and ahead of the guardrail + # merge and the pre-call hooks, so everything that keys off the model + # group -- model-level guardrails, per-model budgets and rate limits, + # the logging object -- sees the group that will actually serve. + if llm_router is not None and proxy_config is not None: + from litellm.proxy.proxy_server import prisma_client + + router_settings = await proxy_config._get_hierarchical_router_settings( + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + ) + + # If router_settings found (from key or team), apply them + # Pass settings as per-request overrides instead of creating a new Router + # This avoids expensive Router instantiation on each request + if router_settings is not None: + self.data["router_settings_override"] = router_settings + alias_target = await _resolve_per_request_model_group_alias( + requested_model=self.data.get("model"), + router_settings=router_settings, + user_api_key_dict=user_api_key_dict, + llm_router=llm_router, + ) + if alias_target is not None: + self.data["model"] = alias_target + self.data["litellm_call_id"] = request.headers.get("x-litellm-call-id", str(uuid.uuid4())) DDSpanTagger.tag_call_id(self.data.get("litellm_call_id")) DDSpanTagger.tag_request( @@ -1339,23 +1403,6 @@ class ProxyBaseLLMRequestProcessing: call_type=route_type, # type: ignore ) - # Apply hierarchical router_settings (Key > Team) - # Global router_settings are already on the Router object itself. - if llm_router is not None and proxy_config is not None: - from litellm.proxy.proxy_server import prisma_client - - router_settings = await proxy_config._get_hierarchical_router_settings( - user_api_key_dict=user_api_key_dict, - prisma_client=prisma_client, - proxy_logging_obj=proxy_logging_obj, - ) - - # If router_settings found (from key or team), apply them - # Pass settings as per-request overrides instead of creating a new Router - # This avoids expensive Router instantiation on each request - if router_settings is not None: - self.data["router_settings_override"] = router_settings - if "messages" in self.data and self.data["messages"]: logging_obj.update_messages(self.data["messages"]) diff --git a/litellm/router.py b/litellm/router.py index e6613e1d302..6bf1bdfc670 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -115,6 +115,7 @@ from litellm.router_utils.common_utils import ( _is_proxy_admin_request, filter_team_based_models, filter_web_search_deployments, + resolve_model_group_alias, ) from litellm.router_utils.cooldown_cache import CooldownCache from litellm.router_utils.cooldown_handlers import ( @@ -10331,16 +10332,7 @@ class Router: - str, the litellm model name - None, if model is not in model group alias """ - if model not in self.model_group_alias: - return None - - _item = self.model_group_alias[model] - if isinstance(_item, str): - model = _item - else: - model = _item["model"] - - return model + return resolve_model_group_alias(self.model_group_alias, model) def _get_deployment_by_litellm_model(self, model: str) -> list: """ diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 189296a8955..bf1c814c049 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -22,6 +22,27 @@ def _is_proxy_admin_request(request_kwargs: Mapping[str, object] | None) -> bool return getattr(user_api_key_auth, "user_role", None) == "proxy_admin" +def resolve_model_group_alias(model_group_alias: object, model: str) -> str | None: + """ + Resolve ``model`` through a ``model_group_alias`` map. + + Handles both supported entry shapes, the plain string form + ``{"alias": "target"}`` and the item form + ``{"alias": {"model": "target", "hidden": true}}``, and tolerates malformed + entries: the map can come from a key or team row rather than from validated + config, so a bad value must not raise mid-request. + + Returns the target model group, or None when the map does not rewrite ``model``. + """ + if not isinstance(model_group_alias, Mapping): + return None + entry = model_group_alias.get(model) + target = entry.get("model") if isinstance(entry, Mapping) else entry + if not isinstance(target, str) or not target: + return None + return target + + def get_litellm_params_sensitive_credential_hash(litellm_params: dict) -> str: """ Hash of the credential params, used for mapping the file id to the right model diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 9a3702d2355..28d4d87e26f 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -2016,6 +2016,48 @@ async def test_ProxyConfig__get_hierarchical_router_settings_missing_returns_non assert out is None +@pytest.mark.asyncio +async def test_ProxyConfig__get_hierarchical_router_settings_falls_back_to_team(monkeypatch): + """A key with no router_settings inherits the team's, so a team-level + model_group_alias reaches the request path at all.""" + pc = ProxyConfig() + fake_key = SimpleNamespace(router_settings=None, team_id="team-1") + team_settings = {"model_group_alias": {"group-a": "group-b"}} + monkeypatch.setattr( + "litellm.proxy.proxy_server.get_team_object", + AsyncMock(return_value=SimpleNamespace(router_settings=team_settings)), + ) + + out = await pc._get_hierarchical_router_settings( + user_api_key_dict=fake_key, + prisma_client=None, + proxy_logging_obj=None, + ) + + assert out == team_settings + + +@pytest.mark.asyncio +async def test_ProxyConfig__get_hierarchical_router_settings_key_shadows_team_entirely(monkeypatch): + """Resolution returns whichever object it finds first, it does not merge + per field, so a key that sets any router setting hides every team setting + including an alias the key itself never set.""" + pc = ProxyConfig() + fake_key = SimpleNamespace(router_settings={"num_retries": 3}, team_id="team-1") + team_lookup = AsyncMock(return_value=SimpleNamespace(router_settings={"model_group_alias": {"group-a": "group-b"}})) + monkeypatch.setattr("litellm.proxy.proxy_server.get_team_object", team_lookup) + + out = await pc._get_hierarchical_router_settings( + user_api_key_dict=fake_key, + prisma_client=None, + proxy_logging_obj=None, + ) + + assert out == {"num_retries": 3} + assert "model_group_alias" not in out + team_lookup.assert_not_called() + + # --------------------------------------------------------------------------- # ProxyConfig._add_router_settings_from_db_config # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 3bb84e095a0..4d98a05da8d 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -1,6 +1,7 @@ import asyncio import copy import datetime +from types import SimpleNamespace from typing import AsyncGenerator, Callable, Optional from unittest.mock import AsyncMock, MagicMock, patch @@ -28,11 +29,14 @@ from litellm.proxy.common_request_processing import ( _is_azure_model_router_request, _override_openai_response_model, _parse_event_data_for_error, + _resolve_per_request_model_group_alias, _should_return_raw_model_name, _UpstreamClosingStreamingResponse, create_response, ) from litellm.proxy.dd_span_tagger import DDSpanTagger +from litellm.proxy._types import ProxyException +from litellm.proxy._types import UserAPIKeyAuth as ProxyUserAPIKeyAuth from litellm.proxy.utils import ProxyLogging @@ -5354,3 +5358,228 @@ class TestModelDeploymentsSupportStreamOptions: def test_non_string_model_is_not_injected(self): assert self._support(None, None) is False + + +class TestPerRequestModelGroupAlias: + """``router_settings.model_group_alias`` on a key or team has to be resolved + by the proxy: the Router resolves aliases from its own shared instance + attribute, which only ever holds the global config map.""" + + @staticmethod + def _router() -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": "group-a", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"}, + }, + { + "model_name": "group-b", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-fake"}, + }, + ] + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "alias_map, expected", + [ + ({"group-a": "group-b"}, "group-b"), + ({"group-a": {"model": "group-b", "hidden": True}}, "group-b"), + ({"group-b": "group-a"}, None), + ({"group-a": "group-a"}, None), + ({"group-a": {"hidden": True}}, None), + ({}, None), + (None, None), + ], + ) + async def test_resolves_alias_for_the_requested_model_group(self, alias_map, expected): + resolved = await _resolve_per_request_model_group_alias( + requested_model="group-a", + router_settings={"model_group_alias": alias_map}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash", models=[]), + llm_router=self._router(), + ) + + assert resolved == expected + + @pytest.mark.asyncio + async def test_alias_target_outside_the_key_allowlist_is_rejected(self): + """Access was authorized against the requested group, so a rewrite that + the key could not have requested directly must not be served.""" + with pytest.raises(ProxyException) as exc_info: + await _resolve_per_request_model_group_alias( + requested_model="group-a", + router_settings={"model_group_alias": {"group-a": "group-b"}}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash", models=["group-a"]), + llm_router=self._router(), + ) + + assert exc_info.value.code == "403" + assert "group-b" in exc_info.value.message + + @pytest.mark.asyncio + async def test_alias_target_inside_the_key_allowlist_resolves(self): + resolved = await _resolve_per_request_model_group_alias( + requested_model="group-a", + router_settings={"model_group_alias": {"group-a": "group-b"}}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash", models=["group-a", "group-b"]), + llm_router=self._router(), + ) + + assert resolved == "group-b" + + @pytest.mark.asyncio + @pytest.mark.parametrize("requested_model", [None, ["group-a", "group-b"]]) + async def test_non_string_requested_model_is_left_alone(self, requested_model): + """The routed model is not always a string (a batch request carries a + list), and an unhashable one must not blow up the alias lookup.""" + resolved = await _resolve_per_request_model_group_alias( + requested_model=requested_model, + router_settings={"model_group_alias": {"group-a": "group-b"}}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash", models=[]), + llm_router=self._router(), + ) + + assert resolved is None + + @pytest.mark.asyncio + async def test_pre_call_logic_rewrites_the_requested_model(self, monkeypatch): + """End to end through the request path: a key carrying the alias must + leave pre-call processing pointing at the alias target, not at the + group the caller asked for.""" + processing_obj = ProxyBaseLLMRequestProcessing(data={"model": "group-a"}) + mock_request = MagicMock(spec=Request) + mock_request.headers = {} + + async def mock_add_litellm_data_to_request(*args, **kwargs): + return kwargs.get("data", {}) + + async def passthrough_pre_call_hook(user_api_key_dict, data, call_type): + return copy.deepcopy(data) + + mock_proxy_logging_obj = MagicMock(spec=ProxyLogging) + mock_proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=passthrough_pre_call_hook) + monkeypatch.setattr( + litellm.proxy.common_request_processing, + "add_litellm_data_to_request", + mock_add_litellm_data_to_request, + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", MagicMock()) + + mock_proxy_config = MagicMock(spec=ProxyConfig) + mock_proxy_config._get_hierarchical_router_settings = AsyncMock( + return_value={"model_group_alias": {"group-a": "group-b"}} + ) + + returned_data, _ = await processing_obj.common_processing_pre_call_logic( + request=mock_request, + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash", models=[]), + proxy_logging_obj=mock_proxy_logging_obj, + proxy_config=mock_proxy_config, + route_type="acompletion", + llm_router=self._router(), + ) + + assert returned_data["model"] == "group-b" + assert returned_data["router_settings_override"] == {"model_group_alias": {"group-a": "group-b"}} + # The rewrite has to land before the pre-call hooks: they are where + # per-model budgets and rate limits are enforced, so resolving later + # applies the requested group's limits to a call the target serves. + assert mock_proxy_logging_obj.pre_call_hook.call_args.kwargs["data"]["model"] == "group-b" + + @pytest.mark.asyncio + async def test_team_level_alias_rewrites_the_requested_model(self, monkeypatch): + """The team path is separate resolution, not a variant of the key path: + settings are looked up on the team only when the key carries none. Runs + the real hierarchical lookup rather than mocking it, so this covers the + team half of the fix end to end.""" + from litellm.proxy.proxy_server import ProxyConfig as RealProxyConfig + + processing_obj = ProxyBaseLLMRequestProcessing(data={"model": "group-a"}) + mock_request = MagicMock(spec=Request) + mock_request.headers = {} + + async def mock_add_litellm_data_to_request(*args, **kwargs): + return kwargs.get("data", {}) + + async def passthrough_pre_call_hook(user_api_key_dict, data, call_type): + return copy.deepcopy(data) + + mock_proxy_logging_obj = MagicMock(spec=ProxyLogging) + mock_proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=passthrough_pre_call_hook) + monkeypatch.setattr( + litellm.proxy.common_request_processing, + "add_litellm_data_to_request", + mock_add_litellm_data_to_request, + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", MagicMock()) + monkeypatch.setattr( + "litellm.proxy.proxy_server.get_team_object", + AsyncMock(return_value=SimpleNamespace(router_settings={"model_group_alias": {"group-a": "group-b"}})), + ) + + returned_data, _ = await processing_obj.common_processing_pre_call_logic( + request=mock_request, + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash", models=[], team_id="team-1"), + proxy_logging_obj=mock_proxy_logging_obj, + proxy_config=RealProxyConfig(), + route_type="acompletion", + llm_router=self._router(), + ) + + assert returned_data["model"] == "group-b" + + @pytest.mark.asyncio + async def test_model_level_guardrails_resolve_against_the_alias_target(self, monkeypatch): + """Model-level guardrails are merged by model group name, so the merge + must see the target rather than the group the caller named.""" + processing_obj = ProxyBaseLLMRequestProcessing(data={"model": "group-a"}) + mock_request = MagicMock(spec=Request) + mock_request.headers = {} + + async def mock_add_litellm_data_to_request(*args, **kwargs): + return kwargs.get("data", {}) + + async def passthrough_pre_call_hook(user_api_key_dict, data, call_type): + return copy.deepcopy(data) + + mock_proxy_logging_obj = MagicMock(spec=ProxyLogging) + mock_proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=passthrough_pre_call_hook) + monkeypatch.setattr( + litellm.proxy.common_request_processing, + "add_litellm_data_to_request", + mock_add_litellm_data_to_request, + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", MagicMock()) + + merged_for: list = [] + + def recording_merge(data, llm_router, trust_client_model_info=True): + merged_for.append(data.get("model")) + return data + + monkeypatch.setattr( + litellm.proxy.common_request_processing, + "_check_and_merge_model_level_guardrails", + recording_merge, + ) + + mock_proxy_config = MagicMock(spec=ProxyConfig) + mock_proxy_config._get_hierarchical_router_settings = AsyncMock( + return_value={"model_group_alias": {"group-a": "group-b"}} + ) + + await processing_obj.common_processing_pre_call_logic( + request=mock_request, + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash", models=[]), + proxy_logging_obj=mock_proxy_logging_obj, + proxy_config=mock_proxy_config, + route_type="acompletion", + llm_router=self._router(), + ) + + assert merged_for == ["group-b"] diff --git a/tests/test_litellm/proxy/test_model_level_guardrails.py b/tests/test_litellm/proxy/test_model_level_guardrails.py index 48163bf5ed5..a1278e399b5 100644 --- a/tests/test_litellm/proxy/test_model_level_guardrails.py +++ b/tests/test_litellm/proxy/test_model_level_guardrails.py @@ -598,10 +598,15 @@ async def test_pre_call_merges_model_level_guardrails_before_pre_call_hook(): } ) - captured_pre_call_data: dict = {} + captured_pre_call_guardrails: list = [] async def fake_pre_call_hook(*, user_api_key_dict, data, call_type): - captured_pre_call_data.update(data) + # Snapshot the list rather than the dict: metadata is shared by + # reference, so a merge that happens after this point would otherwise + # show up here retroactively and the assertion would pass either way. + captured_pre_call_guardrails.extend( + (data.get("metadata") or {}).get("guardrails") or data.get("guardrails") or [] + ) return data proxy_logging = MagicMock() @@ -616,13 +621,9 @@ async def test_pre_call_merges_model_level_guardrails_before_pre_call_hook(): proxy_config = MagicMock() proxy_config._get_hierarchical_router_settings = AsyncMock(return_value=None) - # Stop the function before any post-pre_call_hook logic so we can keep - # the test focused. Raising _StopAfterPreCall in the next await fires - # right after the guardrail merge + pre_call_hook complete. - class _StopAfterPreCall(Exception): - pass - - proxy_config._get_hierarchical_router_settings.side_effect = _StopAfterPreCall() + # Assert on what pre_call_hook was handed rather than short-circuiting the + # function part way through: a sentinel keyed to one particular later call + # silently stops testing the ordering as soon as that call moves. with ( patch( @@ -640,30 +641,24 @@ async def test_pre_call_merges_model_level_guardrails_before_pre_call_hook(): ): from litellm.proxy._types import UserAPIKeyAuth - try: - await processing.common_processing_pre_call_logic( - request=MagicMock(headers={}, url=MagicMock(path="/v1/chat/completions")), - general_settings={}, - user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), - proxy_logging_obj=proxy_logging, - proxy_config=proxy_config, - route_type="acompletion", - version=None, - user_model=None, - user_temperature=None, - user_request_timeout=None, - user_max_tokens=None, - user_api_base=None, - model=None, - llm_router=mock_router, - ) - except _StopAfterPreCall: - pass + await processing.common_processing_pre_call_logic( + request=MagicMock(headers={}, url=MagicMock(path="/v1/chat/completions")), + general_settings={}, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + proxy_logging_obj=proxy_logging, + proxy_config=proxy_config, + route_type="acompletion", + version=None, + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + model=None, + llm_router=mock_router, + ) # The pre_call_hook must have received data with the model-level # guardrail already merged in. Before the fix, this assertion fails # because pre_call_hook saw the original data without merge. - merged = (captured_pre_call_data.get("metadata") or {}).get("guardrails") or ( - captured_pre_call_data.get("guardrails") or [] - ) - assert "my-pre-call-guardrail" in merged + assert "my-pre-call-guardrail" in captured_pre_call_guardrails diff --git a/tests/test_litellm/router_utils/test_router_utils_common_utils.py b/tests/test_litellm/router_utils/test_router_utils_common_utils.py index efa5f2382dc..7d453c72652 100644 --- a/tests/test_litellm/router_utils/test_router_utils_common_utils.py +++ b/tests/test_litellm/router_utils/test_router_utils_common_utils.py @@ -10,6 +10,7 @@ from litellm.router_utils.common_utils import ( add_model_file_id_mappings, filter_team_based_models, filter_web_search_deployments, + resolve_model_group_alias, ) @@ -516,3 +517,44 @@ class TestAddModelFileIdMappings: def test_should_return_empty_mapping_when_given_empty_list(self): result = add_model_file_id_mappings([], []) assert result == {} + + +class TestResolveModelGroupAlias: + """``model_group_alias`` maps reach this helper from validated config and + from key/team rows, so both entry shapes must resolve and malformed entries + must not raise mid-request.""" + + @pytest.mark.parametrize( + "alias_map, expected", + [ + ({"group-a": "group-b"}, "group-b"), + ({"group-a": {"model": "group-b", "hidden": True}}, "group-b"), + ({"group-a": {"model": "group-b"}}, "group-b"), + ({"other": "group-b"}, None), + ({}, None), + (None, None), + ("not-a-map", None), + ({"group-a": {"hidden": True}}, None), + ({"group-a": {"model": 5}}, None), + ({"group-a": 5}, None), + ({"group-a": None}, None), + ({"group-a": ""}, None), + ], + ) + def test_resolves_both_entry_shapes_and_tolerates_malformed_entries(self, alias_map, expected): + assert resolve_model_group_alias(alias_map, "group-a") == expected + + def test_router_alias_resolution_uses_the_shared_helper(self): + router = Router( + model_list=[ + { + "model_name": "group-b", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"}, + } + ], + model_group_alias={"group-a": "group-b", "group-item": {"model": "group-b", "hidden": True}}, + ) + + assert router._get_model_from_alias("group-a") == "group-b" + assert router._get_model_from_alias("group-item") == "group-b" + assert router._get_model_from_alias("group-b") is None From 8ad5d144a1fe0c6b06b35a0777e871277c8f8a47 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 3 Aug 2026 15:18:27 -0700 Subject: [PATCH 051/124] feat(complexity_router): default session affinity off and expose it in the UI (#35714) * feat(ui): expose an Auto-Router session affinity toggle session_affinity on ComplexityRouterConfig defaults to True, and neither the create form nor the edit modal ever emitted the key, so every auto-router built in the UI silently pinned each session to its first turn's model for an hour with no way to see or change that. Adds an "Advanced: Session Affinity" switch to both surfaces, defaulted on to match the backend field. Both paths now write the key explicitly instead of falling through to the backend default, so a stored config states what the router actually does. A stored config with the key absent hydrates as on, since those routers are running with affinity enabled today; showing them as off would report the opposite of reality and persist it on the next save. * feat(complexity_router): default session affinity off and expose it in the UI session_affinity defaulted to True and the Auto-Router UI never emitted the key, so every router built there silently pinned each session to whatever model its first turn classified into for an hour, refreshed on every hit. There was no way to see that from the UI and no way to change it without hand-editing config.yaml. The default flips to False, so every turn is classified on its own merits and lands on the cheapest adequate tier. Pinning is now opt-in. The toggle added in the previous commit follows the field: it renders off, and both the create tab and the edit modal keep writing the key explicitly, so a stored config states what the router does instead of inheriting a default that can move under it. Behavior change for existing routers: those created before this have no session_affinity key stored, so they pick up the new default and start reclassifying every turn. That gives up the provider prompt cache the pin was preserving, and a multi-turn session can now change model between turns. Set session_affinity: true to keep the old behavior. --- .../complexity_router/config.py | 8 +- .../router_strategy/test_complexity_router.py | 26 +++---- .../add_model/ComplexityRouterConfig.tsx | 28 +++++++ .../add_model/add_auto_router_tab.test.tsx | 36 +++++++++ .../add_model/add_auto_router_tab.tsx | 3 + .../build_complexity_router_config.test.ts | 18 ++++- .../build_complexity_router_config.ts | 4 + ...d_updated_complexity_router_config.test.ts | 25 +++++++ .../edit_auto_router_modal.test.ts | 2 + .../edit_auto_router_modal.test.tsx | 73 +++++++++++++++++++ .../edit_auto_router_modal.tsx | 7 ++ 11 files changed, 209 insertions(+), 21 deletions(-) diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 970d8de8575..1fa98f13c25 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -426,13 +426,13 @@ class ComplexityRouterConfig(BaseModel): # Session affinity: pin the first turn's routed model for the rest of the session session_affinity: bool = Field( - default=True, + default=False, description=( "When True and a session_id is resolvable on the request, pin the model chosen on the " "session's first turn and reuse it for every later turn, skipping re-classification. " - "On by default so multi-turn sessions stay on one model, preserving provider prompt " - "caches and avoiding cross-model conversation-history errors. Set False to reclassify " - "every turn." + "Off by default so every turn is classified on its own merits and routed to the cheapest " + "adequate tier. Set True to keep a multi-turn session on one model, which preserves " + "provider prompt caches and avoids cross-model conversation-history errors." ), ) session_affinity_ttl_seconds: int = Field( diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index cc73273450a..3a94b1e0f85 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -2903,7 +2903,7 @@ class TestRoutingDecisionCauseLogging: class TestSessionAffinity: - """Test the session_affinity sticky-routing behavior (on by default).""" + """Test the session_affinity sticky-routing behavior (off by default).""" REASONING_MESSAGE = [ { @@ -2917,18 +2917,14 @@ class TestSessionAffinity: def session_affinity_config(self, basic_config) -> Dict: return {**basic_config, "session_affinity": True} - @pytest.fixture - def session_affinity_disabled_config(self, basic_config) -> Dict: - return {**basic_config, "session_affinity": False} - @staticmethod def _request_kwargs(session_id: str) -> Dict: return {"metadata": {"session_id": session_id}} @pytest.mark.asyncio - async def test_enabled_by_default_pins_model(self, mock_router_instance, basic_config): - """Regression: session_affinity defaults to True, so a shared session_id pins the - first turn's model and later turns reuse it instead of reclassifying.""" + async def test_disabled_by_default_reclassifies_every_turn(self, mock_router_instance, basic_config): + """Regression: session_affinity defaults to False, so a shared session_id must NOT + pin the first turn's model; every turn is classified on its own merits.""" assert "session_affinity" not in basic_config mock_router_instance.cache = DualCache() router = ComplexityRouter( @@ -2944,19 +2940,17 @@ class TestSessionAffinity: model="test-model", request_kwargs=request_kwargs, messages=self.SIMPLE_MESSAGE ) assert first.model == "o1-preview" - assert second.model == "o1-preview" + assert second.model == "gpt-4o-mini" @pytest.mark.asyncio - async def test_can_be_disabled_reclassifies_every_turn( - self, mock_router_instance, session_affinity_disabled_config - ): - """Regression: session_affinity=False must still reclassify every turn even when a - shared session_id is present, so the opt-out keeps working.""" + async def test_can_be_enabled_to_pin_every_later_turn(self, mock_router_instance, session_affinity_config): + """Regression: session_affinity=True is the opt-in, so a shared session_id reuses the + first turn's model instead of reclassifying.""" mock_router_instance.cache = DualCache() router = ComplexityRouter( model_name="test-router", litellm_router_instance=mock_router_instance, - complexity_router_config=session_affinity_disabled_config, + complexity_router_config=session_affinity_config, ) request_kwargs = self._request_kwargs("session-1") first = await router.async_pre_routing_hook( @@ -2966,7 +2960,7 @@ class TestSessionAffinity: model="test-model", request_kwargs=request_kwargs, messages=self.SIMPLE_MESSAGE ) assert first.model == "o1-preview" - assert second.model == "gpt-4o-mini" + assert second.model == "o1-preview" @pytest.mark.asyncio async def test_pins_model_after_first_turn(self, mock_router_instance, session_affinity_config): diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 0503c0c9c6d..b83808b0728 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -14,6 +14,7 @@ export const DEFAULT_CLASSIFIER_TIMEOUT_MS = 3000; export const DEFAULT_TIER_DISTANCE_PENALTY = 0.5; export const DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE = 3; export const DEFAULT_CLASSIFIER_CONTEXT_PER_TURN_CHARS = 200; +export const DEFAULT_SESSION_AFFINITY = false; export interface ComplexityTiers { SIMPLE: string[]; @@ -45,6 +46,7 @@ export interface ComplexityRouterConfigValue { classifier_context_window_size?: number; classifier_context_per_turn_chars?: number; classifier_context_include_assistant_turns?: boolean; + session_affinity?: boolean; adaptive?: boolean; adaptive_weights?: AdaptiveRouterWeights; tier_distance_penalty?: number; @@ -224,6 +226,32 @@ const ComplexityRouterConfig: React.FC = ({ ), children: , }, + { + key: "session-affinity", + label: ( + + Advanced: Session Affinity + + ), + children: ( + <> +
    + onChange({ ...value, session_affinity: sessionAffinity })} + aria-label="Pin a session to its first model" + /> + Pin a session to its first model +
    + + Off by default: every turn is classified on its own merits and routed to the cheapest adequate tier. + Turn this on to reuse the model chosen on a session's first turn for every later turn, which + preserves provider prompt caches and avoids cross-model conversation-history errors, at the cost of + keeping the whole session on the first turn's tier. + + + ), + }, { key: "response", label: ( diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index fa31c7d9c9d..f7cc9a1deae 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -111,4 +111,40 @@ describe("AddAutoRouterTab", () => { expect(await screen.findByText("Please select a team to continue")).toBeInTheDocument(); expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled(); }); + + it("defaults a new router to session affinity off, matching the backend field default", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "affinity-router"); + await user.click(screen.getByText("Advanced: Session Affinity")); + expect(await screen.findByRole("switch", { name: "Pin a session to its first model" })).not.toBeChecked(); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({ + session_affinity: false, + }); + }); + + it("carries session affinity turned on through to the create payload", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "affinity-router"); + await user.click(screen.getByText("Advanced: Session Affinity")); + await user.click(await screen.findByRole("switch", { name: "Pin a session to its first model" })); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({ + session_affinity: true, + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 4593f6a6a2e..ea75bd8e283 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -10,6 +10,7 @@ import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_m import ComplexityRouterConfig, { ComplexityRouterConfigValue, DEFAULT_ADAPTIVE_WEIGHTS, + DEFAULT_SESSION_AFFINITY, DEFAULT_TIER_DISTANCE_PENALTY, } from "./ComplexityRouterConfig"; import { KeywordTierRule } from "./KeywordTierRules"; @@ -102,6 +103,7 @@ const AddAutoRouterTab: React.FC = ({ classifier_context_window_size: classifierContextWindowSize, classifier_context_per_turn_chars: classifierContextPerTurnChars, classifier_context_include_assistant_turns: classifierContextIncludeAssistantTurns, + session_affinity: sessionAffinity = DEFAULT_SESSION_AFFINITY, adaptive = false, adaptive_weights: adaptiveWeights = DEFAULT_ADAPTIVE_WEIGHTS, tier_distance_penalty: tierDistancePenalty = DEFAULT_TIER_DISTANCE_PENALTY, @@ -148,6 +150,7 @@ const AddAutoRouterTab: React.FC = ({ classifierContextWindowSize, classifierContextPerTurnChars, classifierContextIncludeAssistantTurns, + sessionAffinity, customTechnicalKeywords, keywordTierRules, semanticMatchingEnabled, diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index e939ce12904..9d784b57903 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -19,6 +19,7 @@ const baseParams: BuildComplexityRouterConfigParams = { classifierContextWindowSize: undefined, classifierContextPerTurnChars: undefined, classifierContextIncludeAssistantTurns: undefined, + sessionAffinity: false, customTechnicalKeywords: [], keywordTierRules: [], semanticMatchingEnabled: false, @@ -35,7 +36,12 @@ const baseParams: BuildComplexityRouterConfigParams = { describe("buildComplexityRouterConfig", () => { it("emits tiers, classifier_type, and escalation_keywords when nothing else is configured", () => { const config = buildComplexityRouterConfig(baseParams); - expect(config).toEqual({ tiers, classifier_type: "heuristic", escalation_keywords: ["LITELLM ESCALATE"] }); + expect(config).toEqual({ + tiers, + classifier_type: "heuristic", + session_affinity: false, + escalation_keywords: ["LITELLM ESCALATE"], + }); }); it("trims escalation keywords and drops blank entries", () => { @@ -219,6 +225,16 @@ describe("buildComplexityRouterConfig", () => { expect(config.return_raw_model_name).toBeUndefined(); }); + it("writes session_affinity=true so turning the toggle on overrides the backend's off-by-default", () => { + const config = buildComplexityRouterConfig({ ...baseParams, sessionAffinity: true }); + expect(config.session_affinity).toBe(true); + }); + + it("writes session_affinity explicitly when off, so the stored config never relies on the backend default", () => { + const config = buildComplexityRouterConfig({ ...baseParams, sessionAffinity: false }); + expect(config.session_affinity).toBe(false); + }); + it("includes return_raw_model_name when enabled", () => { const config = buildComplexityRouterConfig({ ...baseParams, returnRawModelName: true }); expect(config.return_raw_model_name).toBe(true); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 192e71b4597..cd6c697b377 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -15,6 +15,7 @@ export interface BuildComplexityRouterConfigParams { classifierContextWindowSize: number | undefined; classifierContextPerTurnChars: number | undefined; classifierContextIncludeAssistantTurns: boolean | undefined; + sessionAffinity: boolean; customTechnicalKeywords: string[]; keywordTierRules: KeywordTierRule[]; semanticMatchingEnabled: boolean; @@ -35,6 +36,7 @@ export interface ComplexityRouterConfigPayload { classifier_context_window_size?: number; classifier_context_per_turn_chars?: number; classifier_context_include_assistant_turns?: boolean; + session_affinity: boolean; custom_technical_keywords?: string[]; keyword_tier_rules?: { keywords: string[]; tier: KeywordTierRule["tier"] }[]; semantic_keyword_matching?: boolean; @@ -78,6 +80,7 @@ export const buildComplexityRouterConfig = ({ classifierContextWindowSize, classifierContextPerTurnChars, classifierContextIncludeAssistantTurns, + sessionAffinity, customTechnicalKeywords, keywordTierRules, semanticMatchingEnabled, @@ -110,6 +113,7 @@ export const buildComplexityRouterConfig = ({ classifierContextIncludeAssistantTurns !== undefined && { classifier_context_include_assistant_turns: classifierContextIncludeAssistantTurns, }), + session_affinity: sessionAffinity, ...(customTechnicalKeywords.length > 0 && { custom_technical_keywords: customTechnicalKeywords }), ...(cleanedKeywordTierRules.length > 0 && { keyword_tier_rules: cleanedKeywordTierRules }), escalation_keywords: cleanedEscalationKeywords, diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index f8d46f9ddd6..971c833a0de 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -199,3 +199,28 @@ describe("buildUpdatedComplexityRouterConfig assistant turns", () => { expect(result.classifier_context_include_assistant_turns).toBeUndefined(); }); }); + +describe("buildUpdatedComplexityRouterConfig session affinity", () => { + it("writes session_affinity=false when the toggle is off", () => { + const result = buildUpdatedComplexityRouterConfig(STORED, { ...FORM_VALUE, session_affinity: false }); + expect(result.session_affinity).toBe(false); + }); + + it("writes session_affinity=true when the toggle is on", () => { + const result = buildUpdatedComplexityRouterConfig(STORED, { ...FORM_VALUE, session_affinity: true }); + expect(result.session_affinity).toBe(true); + }); + + it("re-asserts the backend's off-by-default when the form value is absent, rather than dropping the key", () => { + const result = buildUpdatedComplexityRouterConfig({ ...STORED, session_affinity: true }, FORM_VALUE); + expect(result.session_affinity).toBe(false); + }); + + it("stops a stored session_affinity=true from surviving a save that turned the toggle back off", () => { + const result = buildUpdatedComplexityRouterConfig( + { ...STORED, session_affinity: true }, + { ...FORM_VALUE, session_affinity: false }, + ); + expect(result.session_affinity).toBe(false); + }); +}); diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts index 17fa810b529..eb5bb46f0e8 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts @@ -47,6 +47,7 @@ const expectedClassifiedTierConfig = { semantic_keyword_matching: true, embedding_model: "voyage-4-large", match_threshold: 0.65, + session_affinity: false, adaptive: true, adaptive_weights: { quality: 0.4, cost: 0.6 }, adaptive_eligible: "classified_tier", @@ -66,6 +67,7 @@ const expectedAdaptiveDisabledConfig = { semantic_keyword_matching: true, embedding_model: "voyage-4-large", match_threshold: 0.65, + session_affinity: false, }; describe("buildUpdatedComplexityRouterConfig", () => { diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx index 98b7ac519f5..c0806befa52 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx @@ -243,3 +243,76 @@ describe("EditAutoRouterModal assistant turns", () => { expect(savedConfig().classifier_context_include_assistant_turns).toBe(false); }); }); + +describe("EditAutoRouterModal session affinity", () => { + beforeEach(() => { + modelPatchUpdateCall.mockClear(); + }); + + const renderWithStoredConfig = (complexity_router_config: Record) => + renderWithProviders( + , + ); + + // A stored config with no session_affinity key now runs with affinity OFF, because the backend + // field defaults to False. The toggle has to render what the router actually does, and an + // untouched save must not flip it. + it("shows a stored config with no session_affinity key as off", async () => { + const user = userEvent.setup(); + renderWithStoredConfig(STORED_CONFIG); + + await user.click(await screen.findByText("Advanced: Session Affinity")); + expect(await screen.findByRole("switch", { name: "Pin a session to its first model" })).not.toBeChecked(); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().session_affinity).toBe(false); + }); + + it("shows a stored session_affinity=true as on and preserves it through an untouched save", async () => { + const user = userEvent.setup(); + renderWithStoredConfig({ ...STORED_CONFIG, session_affinity: true }); + + await user.click(await screen.findByText("Advanced: Session Affinity")); + expect(await screen.findByRole("switch", { name: "Pin a session to its first model" })).toBeChecked(); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().session_affinity).toBe(true); + }); + + it("persists turning session affinity on", async () => { + const user = userEvent.setup(); + renderWithStoredConfig(STORED_CONFIG); + + await user.click(await screen.findByText("Advanced: Session Affinity")); + await user.click(await screen.findByRole("switch", { name: "Pin a session to its first model" })); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().session_affinity).toBe(true); + }); + + it("persists turning session affinity back off", async () => { + const user = userEvent.setup(); + renderWithStoredConfig({ ...STORED_CONFIG, session_affinity: true }); + + await user.click(await screen.findByText("Advanced: Session Affinity")); + await user.click(await screen.findByRole("switch", { name: "Pin a session to its first model" })); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().session_affinity).toBe(false); + }); +}); diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index 99b5ff178b1..a70fc31d6fe 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -13,6 +13,7 @@ import { hydrateKeywordTierRules, serializeKeywordTierRules } from "../add_model import ComplexityRouterConfig, { ComplexityRouterConfigValue, DEFAULT_ADAPTIVE_WEIGHTS, + DEFAULT_SESSION_AFFINITY, DEFAULT_TIER_DISTANCE_PENALTY, } from "../add_model/ComplexityRouterConfig"; import NotificationsManager from "../molecules/notifications_manager"; @@ -36,6 +37,7 @@ const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "classifier_context_window_size", "classifier_context_per_turn_chars", "classifier_context_include_assistant_turns", + "session_affinity", "adaptive", "adaptive_weights", "tier_distance_penalty", @@ -101,6 +103,7 @@ export const buildUpdatedComplexityRouterConfig = ( value.classifier_context_include_assistant_turns !== undefined && { classifier_context_include_assistant_turns: value.classifier_context_include_assistant_turns, }), + session_affinity: value.session_affinity ?? DEFAULT_SESSION_AFFINITY, ...(customTechnicalKeywords && customTechnicalKeywords.length > 0 && { custom_technical_keywords: customTechnicalKeywords, @@ -218,6 +221,10 @@ const EditAutoRouterModal: React.FC = ({ typeof parsedConfig.classifier_context_include_assistant_turns === "boolean" ? parsedConfig.classifier_context_include_assistant_turns : undefined, + session_affinity: + typeof parsedConfig.session_affinity === "boolean" + ? parsedConfig.session_affinity + : DEFAULT_SESSION_AFFINITY, adaptive: parsedConfig.adaptive || false, adaptive_weights: parsedConfig.adaptive_weights, tier_distance_penalty: parsedConfig.tier_distance_penalty, From 7dab1ff75f8fb105bc4a3f783742798ebb5b61cd Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 3 Aug 2026 16:19:56 -0700 Subject: [PATCH 052/124] fix(datadog): read team callback dd_* params from kwargs instead of blocked dynamic params (#35115) (#35687) Team-scoped DD credentials (dd_api_key, dd_site) set via POST /team/{id}/callback were silently dropped because _request_blocked_callback_params blocks them from standard_callback_dynamic_params. The security block is correct for request-level injection, but team callback_vars are admin-configured and trusted. Store the raw init kwargs on the Logging instance and read dd_* params from there in _process_dynamic_callback_list instead of from standard_callback_dynamic_params. Adds an integration test that exercises the full Logging.__init__ flow with team callback_vars to prevent regression. Co-authored-by: Aanchal Khandelwal --- .../initialize_dynamic_callback_params.py | 38 ++-- litellm/litellm_core_utils/litellm_logging.py | 11 +- litellm/proxy/litellm_pre_call_utils.py | 45 ++++- litellm/types/utils.py | 7 + .../datadog/test_datadog_team_handler.py | 102 ++++++++++- .../proxy/test_litellm_pre_call_utils.py | 169 ++++++++++++++++++ 6 files changed, 354 insertions(+), 18 deletions(-) diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index 11668acb21e..171165d01be 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -1,7 +1,7 @@ -from collections.abc import Iterator +from collections.abc import Iterator, Mapping from typing import Any -from litellm.types.utils import StandardCallbackDynamicParams +from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, StandardCallbackDynamicParams _CLIENT_CALLBACK_METADATA_SLOTS: tuple[str, ...] = ("litellm_metadata", "metadata") @@ -75,14 +75,32 @@ _supported_callback_params = [ "turn_off_message_logging", ] -_request_blocked_callback_params = { - "gcs_bucket_name", - "gcs_path_service_account", - "dd_api_key", - "dd_site", - "dd_agent_host", - "dd_agent_port", -} +_request_blocked_callback_params = frozenset( + { + "gcs_bucket_name", + "gcs_path_service_account", + "dd_api_key", + "dd_site", + "dd_agent_host", + "dd_agent_port", + } +) + + +def get_trusted_callback_params(kwargs: Mapping[str, Any] | None) -> tuple[tuple[str, str], ...]: + """ + Read callback params the proxy itself stamped from admin-configured team/key callback settings. + + Request-body values never reach this field: the proxy strips it from client input before + setting it, so callbacks can consume credentials and destinations here without re-validating. + + Returned as pairs rather than a mapping because the caller keeps this on the Logging object, + which the proxy deep-copies; a mappingproxy is not copyable and a dict would be mutable. + """ + trusted_vars = kwargs.get(TRUSTED_CALLBACK_VARS_FIELD) if kwargs else None + if not isinstance(trusted_vars, Mapping): + return () + return tuple((key, str(value)) for key, value in trusted_vars.items() if isinstance(key, str)) def initialize_standard_callback_dynamic_params( diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index b00130653c5..66d82bd18f1 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -166,6 +166,9 @@ from ..integrations.s3_v2 import S3Logger as S3V2Logger from ..integrations.supabase import Supabase from ..integrations.traceloop import TraceloopLogger from .exception_mapping_utils import _get_response_headers +from .initialize_dynamic_callback_params import ( + get_trusted_callback_params, +) from .initialize_dynamic_callback_params import ( initialize_standard_callback_dynamic_params as _initialize_standard_callback_dynamic_params, ) @@ -362,6 +365,7 @@ class Logging(LiteLLMLoggingBaseClass): self.standard_callback_dynamic_params: StandardCallbackDynamicParams = ( self.initialize_standard_callback_dynamic_params(kwargs) ) + self._trusted_callback_vars: tuple[tuple[str, str], ...] = get_trusted_callback_params(kwargs) # Process dynamic callbacks (after standard_callback_dynamic_params is initialized, # so team-scoped credentials are available for callback initialization) @@ -459,9 +463,10 @@ class Logging(LiteLLMLoggingBaseClass): # pass only the relevant dynamic params as custom_logger_init_args. _custom_logger_init_args: dict | None = None if callback == "datadog": - _custom_logger_init_args = { - k: v for k, v in self.standard_callback_dynamic_params.items() if k.startswith("dd_") - } + # dd_* params are blocked from standard_callback_dynamic_params + # (request-level security); only the proxy-stamped team/key + # callback vars are admin-configured and trusted. + _custom_logger_init_args = {k: v for k, v in self._trusted_callback_vars if k.startswith("dd_")} callback_class = _init_custom_logger_compatible_class( callback, # type: ignore[arg-type] diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index d58f953d2ef..13eb41af751 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -20,6 +20,8 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + TRUSTED_CALLBACK_VARS_FIELD, + _request_blocked_callback_params, iter_client_callback_metadata_dicts, ) from litellm.litellm_core_utils.safe_json_loads import safe_json_loads @@ -356,6 +358,39 @@ def _strip_client_message_redaction_opt_out(data: dict[str, Any]) -> None: ) +def _strip_client_callback_credentials( + data: dict[str, Any], # mutable-ok: strips in place on the request body the pre-call pipeline threads through +) -> None: + """Drop callback credentials and destinations supplied by the caller. + + ``_request_blocked_callback_params`` (Datadog + GCS credentials, sites and agent + hosts) are already ignored when building ``standard_callback_dynamic_params``. + Strip them from the body and every client metadata slot as well, so a caller + cannot pair its own ``dd_site``/``dd_agent_host`` with the team's admin-configured + ``dd_api_key`` and have the resulting logs shipped to a host it controls. + + ``TRUSTED_CALLBACK_VARS_FIELD`` is proxy-owned; it is cleared here and repopulated + from team/key callback settings in ``add_litellm_data_to_request``. + """ + containers = (("body", data), *iter_client_callback_metadata_dicts(data)) + stripped = tuple( + f"{label}.{field}" + for label, container in containers + for field in _request_blocked_callback_params + if field in container + ) + for _, container in containers: + for field in _request_blocked_callback_params: + container.pop(field, None) + data.pop(TRUSTED_CALLBACK_VARS_FIELD, None) + if stripped: + verbose_proxy_logger.debug( + "Stripped client-supplied callback credentials from request: %s. " + "Configure these on the team or key callback settings instead.", + ", ".join(sorted(stripped)), + ) + + def _strip_client_pricing_overrides(data: dict[str, Any]) -> None: """Drop pricing overrides from the request body and any metadata variant. @@ -524,7 +559,8 @@ def safe_add_api_version_from_query_params(data: dict, request: Request): def convert_key_logging_metadata_to_callback( - data: AddTeamCallback, team_callback_settings_obj: TeamCallbackMetadata | None + data: AddTeamCallback, + team_callback_settings_obj: TeamCallbackMetadata | None, ) -> TeamCallbackMetadata: if team_callback_settings_obj is None: team_callback_settings_obj = TeamCallbackMetadata() @@ -1563,6 +1599,10 @@ async def add_litellm_data_to_request( if not _key_or_team_allows_client_pricing_override(user_api_key_dict): _strip_client_pricing_overrides(data) + # Same reason as the strips above: runs after the metadata string-to-dict parse + # so JSON-string metadata cannot smuggle callback credentials past the dict guard. + _strip_client_callback_credentials(data) + if not _allow_client_message_redaction_opt_out and litellm.turn_off_message_logging is True: _strip_client_message_redaction_opt_out(data) @@ -1771,6 +1811,9 @@ async def add_litellm_data_to_request( # unpack callback_vars in data for k, v in callback_settings_obj.callback_vars.items(): data[k] = v + # Callbacks that must not honour request-supplied credentials read this + # proxy-owned field instead of the raw request kwargs. + data[TRUSTED_CALLBACK_VARS_FIELD] = callback_settings_obj.callback_vars # Add disabled callbacks from key metadata if user_api_key_dict.metadata and "litellm_disabled_callbacks" in user_api_key_dict.metadata: diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 18991f53e6f..3539ac0f27a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3286,8 +3286,15 @@ agentic_loop_internal_litellm_params = [ "_code_interpreter_interception_converted_stream", ] +# Proxy-owned callback credentials, stamped from admin-configured team/key callback +# settings. Listed in all_litellm_params for the same reason as the agentic-loop +# fields above: an unrecognized top-level key is swept into extra_body and sent to +# the provider. +TRUSTED_CALLBACK_VARS_FIELD = "litellm_trusted_callback_vars" + all_litellm_params = ( agentic_loop_internal_litellm_params + + [TRUSTED_CALLBACK_VARS_FIELD] + [ "metadata", "litellm_metadata", diff --git a/tests/test_litellm/integrations/datadog/test_datadog_team_handler.py b/tests/test_litellm/integrations/datadog/test_datadog_team_handler.py index 772e993c132..09d6f51e0a8 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_team_handler.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_team_handler.py @@ -6,6 +6,7 @@ Verifies that DataDogLogger can be instantiated with per-team credentials and that the DataDogHandler correctly resolves and caches per-team loggers. """ +import copy from unittest.mock import patch import pytest @@ -13,7 +14,9 @@ import pytest from litellm.integrations.datadog.datadog import DataDogLogger from litellm.integrations.datadog.datadog_team_handler import ( DataDogHandler, - DatadogLoggingConfig, +) +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + TRUSTED_CALLBACK_VARS_FIELD, ) from litellm.litellm_core_utils.specialty_caches.dynamic_logging_cache import ( DynamicLoggingCache, @@ -94,9 +97,7 @@ class TestDataDogLoggerCredentialKwargs: assert logger.DD_API_KEY is None assert "attacker.example.com" in logger.intake_url - def test_direct_api_mode_does_not_leak_env_api_key_when_disallowed( - self, datadog_env - ): + def test_direct_api_mode_does_not_leak_env_api_key_when_disallowed(self, datadog_env): """With allow_env_credentials=False and no explicit key, init must fail rather than reuse env key.""" with pytest.raises(Exception, match="DD_API_KEY"): with patch("asyncio.create_task"): @@ -261,3 +262,96 @@ class TestStandardCallbackDynamicParamsIncludesDatadog: assert "dd_site" in annotations assert "dd_agent_host" in annotations assert "dd_agent_port" in annotations + + +def _build_logging_obj(kwargs: dict, *, with_datadog_callback: bool = True): + from litellm.litellm_core_utils.litellm_logging import Logging + + with patch("asyncio.create_task"): + return Logging( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="completion", + start_time="2026-01-01", + litellm_call_id="test-call-id", + function_id="test-func", + dynamic_success_callbacks=["datadog"] if with_datadog_callback else None, + kwargs=kwargs, + ) + + +def _dd_loggers(logging_obj) -> list[DataDogLogger]: + return [cb for cb in (logging_obj.dynamic_success_callbacks or []) if isinstance(cb, DataDogLogger)] + + +class TestTeamCallbackFlowPassesDDCredentials: + """ + dd_* credentials reach DataDogHandler only from the proxy-stamped trusted field. + + Team callback_vars are admin-configured, so they must survive + _request_blocked_callback_params; anything the caller put in the request body + must not, or a caller could pair its own dd_site with the team's dd_api_key. + """ + + def test_trusted_callback_vars_reach_datadog_handler(self, datadog_env): + trusted_vars = {"dd_api_key": "team-dd-key-123", "dd_site": "us5.datadoghq.com"} + logging_obj = _build_logging_obj( + { + TRUSTED_CALLBACK_VARS_FIELD: trusted_vars, + "model": "gpt-4", + "litellm_params": {"metadata": {}}, + } + ) + + dd_loggers = _dd_loggers(logging_obj) + assert len(dd_loggers) == 1, "DataDogLogger should be initialized from team callback_vars" + assert dd_loggers[0].DD_API_KEY == "team-dd-key-123" + assert "us5.datadoghq.com" in dd_loggers[0].intake_url + + def test_request_kwargs_dd_params_are_ignored(self, datadog_env): + """Top-level dd_* in the call kwargs are caller-controlled and must never be honoured.""" + logging_obj = _build_logging_obj( + { + "dd_api_key": "caller-dd-key", + "dd_site": "attacker.example.com", + "dd_agent_host": "attacker.example.com", + "model": "gpt-4", + "litellm_params": {"metadata": {}}, + } + ) + + dd_loggers = _dd_loggers(logging_obj) + assert len(dd_loggers) == 1 + assert dd_loggers[0].DD_API_KEY == "global_api_key" + assert "attacker.example.com" not in dd_loggers[0].intake_url + assert "us1.datadoghq.com" in dd_loggers[0].intake_url + + def test_logging_object_stays_deepcopyable(self): + """The proxy deep-copies request data, and the Logging object rides along in it.""" + logging_obj = _build_logging_obj( + { + TRUSTED_CALLBACK_VARS_FIELD: {"dd_api_key": "team-dd-key-123", "dd_site": "us5.datadoghq.com"}, + "model": "gpt-4", + "litellm_params": {"metadata": {}}, + }, + with_datadog_callback=False, + ) + + assert copy.deepcopy(logging_obj)._trusted_callback_vars == logging_obj._trusted_callback_vars + + def test_caller_cannot_redirect_team_credentials(self, datadog_env): + """The exfil shape: caller's dd_site paired with the team's dd_api_key.""" + logging_obj = _build_logging_obj( + { + TRUSTED_CALLBACK_VARS_FIELD: {"dd_api_key": "team-dd-key-123"}, + "dd_site": "attacker.example.com", + "model": "gpt-4", + "litellm_params": {"metadata": {}}, + } + ) + + dd_loggers = _dd_loggers(logging_obj) + assert len(dd_loggers) == 1 + assert dd_loggers[0].DD_API_KEY == "team-dd-key-123" + assert "attacker.example.com" not in dd_loggers[0].intake_url diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index bceefae3a9f..37642605088 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -28,6 +28,9 @@ from litellm.proxy.litellm_pre_call_utils import ( check_if_token_is_service_account, clean_headers, ) +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + TRUSTED_CALLBACK_VARS_FIELD, +) from litellm.types.utils import CredentialItem sys.path.insert( @@ -5554,3 +5557,169 @@ def test_warn_stale_team_alias_once_evicts_oldest_key_beyond_cap(monkeypatch): pre_call_utils._warn_stale_team_alias_once("key-3", "stale alias") assert list(pre_call_utils._STALE_TEAM_ALIAS_WARNING_KEYS) == ["key-2", "key-3"] + + +def _callback_credential_request_mock() -> MagicMock: + request_mock = MagicMock(spec=Request) + request_mock.url = MagicMock() + request_mock.url.path = "/v1/chat/completions" + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + return request_mock + + +_DATADOG_TEAM_KEY = UserAPIKeyAuth( + api_key="hashed-key", + team_id="team-1", + team_metadata={ + "logging": [ + { + "callback_name": "datadog", + "callback_type": "success", + "callback_vars": {"dd_api_key": "team-dd-key"}, + } + ] + }, +) + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_strips_caller_supplied_callback_credentials(): + """ + The team admin sets dd_api_key only; a caller pairing its own dd_site with that key + would ship the team's Datadog credential to a host it controls. + """ + caller_destinations = {"dd_site": "attacker.example.com", "dd_agent_host": "attacker.example.com"} + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hello"}], + **caller_destinations, + "gcs_bucket_name": "attacker-bucket", + TRUSTED_CALLBACK_VARS_FIELD: {"dd_site": "smuggled.example.com"}, + "metadata": {**caller_destinations, "safe_user_metadata": "kept"}, + "litellm_metadata": dict(caller_destinations), + "litellm_params": {"metadata": dict(caller_destinations)}, + } + + updated = await add_litellm_data_to_request( + data=data, + request=_callback_credential_request_mock(), + user_api_key_dict=_DATADOG_TEAM_KEY, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert "dd_site" not in updated + assert "dd_agent_host" not in updated + assert "gcs_bucket_name" not in updated + assert updated["dd_api_key"] == "team-dd-key" + assert updated[TRUSTED_CALLBACK_VARS_FIELD] == {"dd_api_key": "team-dd-key"} + for metadata_key in ("metadata", "litellm_metadata"): + assert "dd_site" not in updated[metadata_key] + assert "dd_agent_host" not in updated[metadata_key] + assert "dd_site" not in updated["litellm_params"]["metadata"] + assert updated["metadata"]["safe_user_metadata"] == "kept" + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_strips_caller_supplied_callback_credentials_with_clientside_creds_allowed(): + """`allow_client_side_credentials` opens the auth-layer ban; the strip must still hold.""" + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hello"}], + "dd_site": "attacker.example.com", + } + + updated = await add_litellm_data_to_request( + data=data, + request=_callback_credential_request_mock(), + user_api_key_dict=_DATADOG_TEAM_KEY, + proxy_config=MagicMock(), + general_settings={"allow_client_side_credentials": True}, + version="test-version", + ) + + assert "dd_site" not in updated + assert updated[TRUSTED_CALLBACK_VARS_FIELD] == {"dd_api_key": "team-dd-key"} + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_omits_trusted_callback_vars_without_team_callbacks(): + """Without team/key callback settings the trusted field must not exist for a callback to read.""" + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hello"}], + TRUSTED_CALLBACK_VARS_FIELD: {"dd_api_key": "caller-key", "dd_site": "attacker.example.com"}, + } + + updated = await add_litellm_data_to_request( + data=data, + request=_callback_credential_request_mock(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert TRUSTED_CALLBACK_VARS_FIELD not in updated + + +def test_trusted_callback_vars_never_reach_the_provider(): + """ + The stamped field rides the request body, so it has to be a recognised litellm param; + otherwise the OpenAI param builder sweeps it into extra_body and the provider 400s. + """ + from litellm.utils import get_non_default_completion_params + + non_default = get_non_default_completion_params( + { + "model": "gpt-4", + TRUSTED_CALLBACK_VARS_FIELD: {"dd_api_key": "team-dd-key"}, + "some_provider_param": "kept", + } + ) + + assert TRUSTED_CALLBACK_VARS_FIELD not in non_default + assert non_default["some_provider_param"] == "kept" + + +@pytest.mark.asyncio +async def test_key_level_callback_vars_survive_the_strip(): + """ + Key-level callbacks configure their own destination and credentials, and they replace + team settings rather than merging with them, so only the request body is untrusted. + """ + key_with_datadog_callback = UserAPIKeyAuth( + api_key="hashed-key", + metadata={ + "logging": [ + { + "callback_name": "datadog", + "callback_type": "success", + "callback_vars": {"dd_api_key": "key-dd-key", "dd_site": "us5.datadoghq.com"}, + } + ] + }, + ) + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hello"}], + "dd_site": "attacker.example.com", + } + + updated = await add_litellm_data_to_request( + data=data, + request=_callback_credential_request_mock(), + user_api_key_dict=key_with_datadog_callback, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert updated[TRUSTED_CALLBACK_VARS_FIELD] == {"dd_api_key": "key-dd-key", "dd_site": "us5.datadoghq.com"} + assert updated["dd_site"] == "us5.datadoghq.com" From cd3b7ef4271c622e55dfcdc4c5c4ed0b6744bb79 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 3 Aug 2026 13:31:45 -0700 Subject: [PATCH 053/124] test(ui): tier the MCP create tests into unit and integration Adds 61 unit tests on the modules #35694 extracted: 46 on the payload builder, 15 on the OAuth redirect snapshot. They run in 9ms against 240s for the 77 full-render tests they partly replace. Nine of nine mutants were killed when the extracted logic was deliberately broken, so the speed does not come at the cost of signal. Deletes six cases across four blocks that rendered the whole modal to assert one payload key belonging to a field they never touched. Every test that proves a form field reaches the right payload key stays; those cover field to form value to payload, which a unit test cannot reach. Replaces "should not render when user is not an admin", which asserted the admin title was absent and so passed for the wrong reason: the modal does render for a non-admin, retitled. registerMCPServer was mocked but never asserted anywhere, leaving the whole non-admin submission path uncovered. It now drives a real submit and asserts the call lands there and never on createMCPServer. Renames the slow file to CreateMCPServer.integration.test.tsx and documents the three tiers in the dashboard CLAUDE.md. No production code changes. --- ui/litellm-dashboard/CLAUDE.md | 6 + ...x => CreateMCPServer.integration.test.tsx} | 144 +++------ .../_components/createOAuthUiState.test.ts | 126 ++++++++ .../_components/createServerPayload.test.ts | 296 ++++++++++++++++++ 4 files changed, 476 insertions(+), 96 deletions(-) rename ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/{CreateMCPServer.test.tsx => CreateMCPServer.integration.test.tsx} (96%) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createServerPayload.test.ts diff --git a/ui/litellm-dashboard/CLAUDE.md b/ui/litellm-dashboard/CLAUDE.md index 5ec9392d2b0..701b37ec6aa 100644 --- a/ui/litellm-dashboard/CLAUDE.md +++ b/ui/litellm-dashboard/CLAUDE.md @@ -3,3 +3,9 @@ Never put LiteLLM tokens or API keys in `localStorage`. `localStorage` survives When you fix lint violations that are grandfathered in `eslint-suppressions.json`, run `eslint . --prune-suppressions` and commit the updated baseline so the gate ratchets down instead of leaving a stale suppression `src/lib/http/schema.d.ts` is generated from the proxy's OpenAPI spec; never hand-edit it. After changing a backend route or response model that the dashboard consumes, run `npm run gen:api` and commit the result (CI `Check UI API Types Sync` enforces this) + +Tests come in three tiers, named by the standard definitions. `Foo.test.tsx` is a unit test: one module, collaborators replaced by doubles, no multi-component tree, and it should run in milliseconds. `Foo.integration.test.tsx` renders a real component tree with real children and only stubs the network boundary; it costs seconds per case, so it earns its place by proving wiring that a unit test cannot reach. Browser-level tests live in `tests/e2e/ui/` as Playwright specs against a live proxy + +When a component holds logic worth asserting, extract the logic and unit-test it there rather than driving it through a render. `CreateMCPServer` is the worked example: its payload building lives in `createServerPayload.ts` with 46 unit tests that run in single-digit milliseconds, while `CreateMCPServer.integration.test.tsx` keeps only the cases that prove a form field reaches the right payload key. A test that renders a whole modal to assert the shape of one object belongs in the first category, not the second + +Most of the suite predates this split and is not yet classified, so an unsuffixed `*.test.tsx` is not evidence that a file is really a unit test. Classify what you touch diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.integration.test.tsx similarity index 96% rename from ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.integration.test.tsx index 45da71ed301..c9007e29c3e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.integration.test.tsx @@ -120,10 +120,48 @@ describe("CreateMCPServer", () => { expect(screen.getByText("Add New MCP Server")).toBeInTheDocument(); }); - it("should not render when user is not an admin", () => { + // The modal DOES render for a non-admin; it retitles and routes the submit to the review endpoint. + // The assertion this replaced only checked that the admin title was absent, which passed for the + // wrong reason and left the whole non-admin submission path uncovered. + it("routes a non-admin submission to the review endpoint instead of creating the server", async () => { render(); + expect(screen.getByText("Submit MCP Server for Review")).toBeInTheDocument(); expect(screen.queryByText("Add New MCP Server")).not.toBeInTheDocument(); + + await selectAntOption("Transport Type", "Streamable HTTP"); + await waitFor(() => { + expect(screen.getByPlaceholderText("https://your-mcp-server.com")).toBeInTheDocument(); + }); + await act(async () => { + fireEvent.change(getServerNameInput(), { target: { value: "Submitted_Server" } }); + }); + await act(async () => { + fireEvent.change(screen.getByPlaceholderText("https://your-mcp-server.com"), { + target: { value: "https://example.com/mcp" }, + }); + }); + await selectAntOption("Authentication", "None"); + + vi.mocked(networking.registerMCPServer).mockResolvedValue({ + server_id: "submitted-1", + server_name: "Submitted_Server", + alias: "Submitted_Server", + url: "https://example.com/mcp", + transport: "http", + auth_type: "none", + created_at: "2024-01-01T00:00:00Z", + created_by: "user-1", + updated_at: "2024-01-01T00:00:00Z", + updated_by: "user-1", + }); + + await act(async () => { + fireEvent.click(screen.getByRole("button", { name: "Add MCP Server" })); + }); + + await waitFor(() => expect(networking.registerMCPServer).toHaveBeenCalledTimes(1)); + expect(networking.createMCPServer).not.toHaveBeenCalled(); }); it("should show transport type options", async () => { @@ -1591,44 +1629,8 @@ describe("CreateMCPServer", () => { expect(payload.credentials?.client_secret).toBeUndefined(); }); - it("omits token_validation from payload when token_validation_json is empty", async () => { - vi.mocked(networking.createMCPServer).mockResolvedValue({ - server_id: "new-server-oauth", - server_name: "OAuth_Server", - alias: "OAuth_Server", - url: "https://example.com/mcp", - transport: "http", - auth_type: "oauth2", - created_at: "2024-01-01T00:00:00Z", - created_by: "user-1", - updated_at: "2024-01-01T00:00:00Z", - updated_by: "user-1", - }); - - await setupOAuthInteractive(); - - const nameInput = document.getElementById("server_name") as HTMLInputElement; - await act(async () => { - fireEvent.change(nameInput, { target: { value: "OAuth_Server" } }); - }); - const urlInput = screen.getByPlaceholderText("https://your-mcp-server.com"); - await act(async () => { - fireEvent.change(urlInput, { target: { value: "https://example.com/mcp" } }); - }); - - // Leave token_validation_json empty - const submitButton = screen.getByRole("button", { name: "Add MCP Server" }); - await act(async () => { - fireEvent.click(submitButton); - }); - - await waitFor(() => { - expect(networking.createMCPServer).toHaveBeenCalledTimes(1); - }); - - const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0]; - expect(payload.token_validation).toBeUndefined(); - }); + // Empty/whitespace token_validation is covered in createServerPayload.test.ts; the sibling + // test above still proves the textarea reaches token_validation_json. it("includes credentials.token_endpoint_auth_method in payload when client_secret_basic is selected", async () => { vi.mocked(networking.createMCPServer).mockResolvedValue({ @@ -1670,43 +1672,8 @@ describe("CreateMCPServer", () => { expect(payload.credentials?.token_endpoint_auth_method).toBe("client_secret_basic"); }); - it("omits token_endpoint_auth_method from credentials when left blank", async () => { - vi.mocked(networking.createMCPServer).mockResolvedValue({ - server_id: "new-server-oauth", - server_name: "OAuth_Server", - alias: "OAuth_Server", - url: "https://example.com/mcp", - transport: "http", - auth_type: "oauth2", - created_at: "2024-01-01T00:00:00Z", - created_by: "user-1", - updated_at: "2024-01-01T00:00:00Z", - updated_by: "user-1", - }); - - await setupOAuthInteractive(); - - const nameInput = document.getElementById("server_name") as HTMLInputElement; - await act(async () => { - fireEvent.change(nameInput, { target: { value: "OAuth_Server" } }); - }); - const urlInput = screen.getByPlaceholderText("https://your-mcp-server.com"); - await act(async () => { - fireEvent.change(urlInput, { target: { value: "https://example.com/mcp" } }); - }); - - const submitButton = screen.getByRole("button", { name: "Add MCP Server" }); - await act(async () => { - fireEvent.click(submitButton); - }); - - await waitFor(() => { - expect(networking.createMCPServer).toHaveBeenCalledTimes(1); - }); - - const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0]; - expect(payload.credentials?.token_endpoint_auth_method).toBeUndefined(); - }); + // Blank credential keys are dropped by the shared filter, covered in createServerPayload.test.ts; + // the sibling test above still proves the select reaches credentials.token_endpoint_auth_method. it("persists access + refresh token to the DB on submit for OBO mode", async () => { // "Authorize & Fetch" produced a token before submit. @@ -2052,14 +2019,8 @@ describe("CreateMCPServer oauth2_flow persistence", () => { expect(payload.oauth2_flow).toBe("client_credentials"); }); - it("sends no oauth2_flow for a non-oauth2 create", async () => { - vi.mocked(networking.createMCPServer).mockResolvedValue({ ...createdServer, auth_type: "none" }); - await setupHttpServerForm(); - await selectAntOption("Authentication", "None"); - - const payload = await submitCreate(); - expect(payload.oauth2_flow).toBeUndefined(); - }); + // oauth2_flow branch coverage lives in createServerPayload.test.ts; the two cases above keep + // the dropdown-to-payload wiring they uniquely prove. }); describe("CreateMCPServer dcr_bridge toggle", () => { @@ -2185,18 +2146,9 @@ describe("CreateMCPServer dcr_bridge toggle", () => { expect(payload.dcr_bridge).toBe(false); }); - it.each([ - ["none", "None"], - ["api_key", "API Key"], - ["oauth2", "OAuth"], - ])("forces an explicit dcr_bridge: false for %s", async (authType, optionLabel) => { - vi.mocked(networking.createMCPServer).mockResolvedValue({ ...createdServer, auth_type: authType }); - await setupHttpServerForm(); - await selectAntOption("Authentication", optionLabel); - - const payload = await submitCreate(); - expect(payload.dcr_bridge).toBe(false); - }); + // Forcing dcr_bridge false for every non-client-forwarded auth type is covered in + // createServerPayload.test.ts. The two form-state cases below stay: they prove the Form.Item + // unmounts on a switch away, and that the live value survives a client-forwarded swap. it("forces dcr_bridge: false when the auth type is switched away after toggling", async () => { vi.mocked(networking.createMCPServer).mockResolvedValue({ ...createdServer, auth_type: "none" }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.test.ts new file mode 100644 index 00000000000..e2e7814f0d7 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.test.ts @@ -0,0 +1,126 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { setSecureItem } from "@/utils/secureStorage"; +import { CreateUiSnapshot, readCreateUiSnapshot, writeCreateUiSnapshot } from "./createOAuthUiState"; + +const STORAGE_KEY = "litellm-mcp-oauth-create-state"; + +const fullSnapshot: CreateUiSnapshot = { + modalVisible: true, + formValues: { url: "https://example.com/mcp", auth_type: "oauth2", credentials: { client_id: "app-id" } }, + transportType: "http", + costConfig: { default_cost_per_query: 0.02 }, + allowedTools: ["search"], + hasToolAllowlistInteraction: true, + searchValue: "group-a", + aliasManuallyEdited: true, + logoUrl: "https://cdn/logo.png", + authorizedIdentity: "identity-abc", +}; + +const seedRaw = (value: unknown) => setSecureItem(STORAGE_KEY, JSON.stringify(value)); + +describe("createOAuthUiState", () => { + beforeEach(() => { + window.sessionStorage.clear(); + vi.restoreAllMocks(); + }); + + it("returns null and leaves storage untouched when nothing was persisted", () => { + expect(readCreateUiSnapshot()).toBeNull(); + }); + + it("round-trips a full snapshot through the redirect", () => { + writeCreateUiSnapshot(fullSnapshot); + expect(readCreateUiSnapshot()).toEqual(fullSnapshot); + }); + + it("does not store the snapshot in plaintext", () => { + writeCreateUiSnapshot(fullSnapshot); + // secureStorage base64-encodes; a readable url in the raw value would mean the encoding was lost. + expect(window.sessionStorage.getItem(STORAGE_KEY)).not.toContain("https://example.com/mcp"); + }); + + it("consumes the snapshot so a second mount cannot replay it", () => { + writeCreateUiSnapshot(fullSnapshot); + expect(readCreateUiSnapshot()).not.toBeNull(); + expect(readCreateUiSnapshot()).toBeNull(); + expect(window.sessionStorage.getItem(STORAGE_KEY)).toBeNull(); + }); + + it("strips minted token material so a stale token never rehydrates", () => { + writeCreateUiSnapshot({ + ...fullSnapshot, + formValues: { + url: "https://example.com/mcp", + credentials: { + client_id: "app-id", + client_secret: "app-secret", + access_token: "stale-tok", + refresh_token: "stale-refresh", + expires_in: 3600, + scope: "read", + }, + }, + }); + + const restored = readCreateUiSnapshot(); + expect(restored?.formValues?.credentials).toEqual({ client_id: "app-id", client_secret: "app-secret" }); + expect(JSON.stringify(restored)).not.toContain("stale-tok"); + expect(JSON.stringify(restored)).not.toContain("stale-refresh"); + }); + + it("re-arms invalidation by restoring the authorized identity", () => { + writeCreateUiSnapshot(fullSnapshot); + expect(readCreateUiSnapshot()?.authorizedIdentity).toBe("identity-abc"); + }); + + it("prefers the persisted form transport over the standalone transportType", () => { + seedRaw({ formValues: { transport: "sse" }, transportType: "http" }); + expect(readCreateUiSnapshot()?.transportType).toBe("sse"); + }); + + it("omits falsy scalars so a restore never blanks freshly mounted state", () => { + seedRaw({ searchValue: "", logoUrl: "", transportType: "", modalVisible: false }); + const restored = readCreateUiSnapshot(); + expect(restored).not.toHaveProperty("searchValue"); + expect(restored).not.toHaveProperty("logoUrl"); + expect(restored).not.toHaveProperty("transportType"); + expect(restored).not.toHaveProperty("modalVisible"); + }); + + it("restores an explicitly empty tool allowlist, which is a real admin choice", () => { + seedRaw({ allowedTools: [], hasToolAllowlistInteraction: true }); + const restored = readCreateUiSnapshot(); + expect(restored?.allowedTools).toEqual([]); + expect(restored?.hasToolAllowlistInteraction).toBe(true); + }); + + it.each([ + ["hasToolAllowlistInteraction", false], + ["aliasManuallyEdited", false], + ])("restores %s when it was persisted as false", (key, value) => { + seedRaw({ [key]: value }); + expect(readCreateUiSnapshot()).toHaveProperty(key, value); + }); + + it.each([["hasToolAllowlistInteraction"], ["aliasManuallyEdited"]])( + "ignores a non-boolean %s rather than coercing it", + (key) => { + seedRaw({ [key]: "yes" }); + expect(readCreateUiSnapshot()).not.toHaveProperty(key); + }, + ); + + it("ignores a non-string authorizedIdentity", () => { + seedRaw({ authorizedIdentity: 42 }); + expect(readCreateUiSnapshot()).not.toHaveProperty("authorizedIdentity"); + }); + + it("returns null on a corrupted payload but still clears it", () => { + vi.spyOn(console, "error").mockImplementation(() => {}); + setSecureItem(STORAGE_KEY, "{not json"); + + expect(readCreateUiSnapshot()).toBeNull(); + expect(window.sessionStorage.getItem(STORAGE_KEY)).toBeNull(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createServerPayload.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createServerPayload.test.ts new file mode 100644 index 00000000000..4573069ff07 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createServerPayload.test.ts @@ -0,0 +1,296 @@ +import { describe, expect, it } from "vitest"; +import { + BuildCreatePayloadResult, + CreateServerUiState, + buildCreateServerPayload, + parseStdioConfig, + reduceStaticHeaders, +} from "./createServerPayload"; + +const baseUi: CreateServerUiState = { + transportType: "http", + costConfig: {}, + allowedTools: [], + hasToolAllowlistInteraction: false, + toolNameToDisplayName: {}, + toolNameToDescription: {}, + logoUrl: undefined, + dcrClient: null, +}; + +/** Narrow to the success branch so a regression surfaces as a failed assertion, not a type error. */ +const payloadOf = (result: BuildCreatePayloadResult): Record => { + expect(result.kind).toBe("ok"); + if (result.kind !== "ok") throw new Error("unreachable"); + return result.payload; +}; + +const build = (values: Record, ui: Partial = {}) => + buildCreateServerPayload(values, { ...baseUi, ...ui }); + +describe("reduceStaticHeaders", () => { + it("returns an empty map for a non-array", () => { + expect(reduceStaticHeaders(undefined)).toEqual({}); + expect(reduceStaticHeaders("X-Api-Key: v")).toEqual({}); + }); + + it("trims header and value and drops rows with a blank header", () => { + expect( + reduceStaticHeaders([ + { header: " X-Api-Key ", value: " secret " }, + { header: " ", value: "orphaned" }, + { header: "X-Empty" }, + ]), + ).toEqual({ "X-Api-Key": "secret", "X-Empty": "" }); + }); + + it("keeps the last value when a header repeats", () => { + expect( + reduceStaticHeaders([ + { header: "X-Dup", value: "first" }, + { header: "X-Dup", value: "second" }, + ]), + ).toEqual({ "X-Dup": "second" }); + }); +}); + +describe("parseStdioConfig", () => { + it("reads a direct command/args/env config", () => { + const result = parseStdioConfig('{"command":"npx","args":["-y","srv"],"env":{"TOKEN":"t"}}'); + expect(result).toEqual({ + kind: "ok", + fields: { command: "npx", args: ["-y", "srv"], env: { TOKEN: "t" } }, + }); + }); + + it("unwraps the mcpServers form and derives the server name with underscores", () => { + const result = parseStdioConfig('{"mcpServers":{"my-github-server":{"command":"npx","args":["-y"]}}}'); + expect(result).toEqual({ + kind: "ok", + fields: { command: "npx", args: ["-y"], env: undefined }, + derivedServerName: "my_github_server", + }); + }); + + it("takes the first server when mcpServers holds several", () => { + const result = parseStdioConfig('{"mcpServers":{"first":{"command":"a"},"second":{"command":"b"}}}'); + expect(result).toMatchObject({ kind: "ok", fields: { command: "a" }, derivedServerName: "first" }); + }); + + it("treats an empty mcpServers object as a direct config rather than deriving a name", () => { + const result = parseStdioConfig('{"mcpServers":{},"command":"direct"}'); + expect(result).toEqual({ kind: "ok", fields: { command: "direct", args: undefined, env: undefined } }); + }); + + it.each([["not json{"], ["null"]])("reports %s as invalid", (raw) => { + expect(parseStdioConfig(raw)).toEqual({ kind: "invalid" }); + }); +}); + +describe("buildCreateServerPayload validation", () => { + it("rejects a tool display name containing a space and names the offender", () => { + const result = build({ auth_type: "none" }, { toolNameToDisplayName: { search: "My Tool" } }); + expect(result).toEqual({ kind: "invalid_tool_display_name", displayName: "My Tool" }); + }); + + it("accepts letters, digits, underscores and hyphens in a display name", () => { + const result = build({ auth_type: "none" }, { toolNameToDisplayName: { search: "my-tool_2" } }); + expect(result.kind).toBe("ok"); + }); + + it("rejects unparseable stdio JSON only when the stdio transport is selected", () => { + expect(build({ stdio_config: "{oops" }, { transportType: "stdio" })).toEqual({ kind: "invalid_stdio_json" }); + // The same bad string on an http server is an inert leftover field, not a submit blocker. + expect(build({ stdio_config: "{oops" }, { transportType: "http" }).kind).toBe("ok"); + }); + + it("rejects unparseable token validation JSON", () => { + expect(build({ token_validation_json: "not-valid-json{" })).toEqual({ kind: "invalid_token_validation_json" }); + }); + + it("ignores a whitespace-only token validation body", () => { + const payload = payloadOf(build({ token_validation_json: " " })); + expect(payload).not.toHaveProperty("token_validation"); + }); + + it("includes parsed token validation rules when the JSON is valid", () => { + const payload = payloadOf(build({ token_validation_json: '{"organization":"my-org","team.id":"42"}' })); + expect(payload.token_validation).toEqual({ organization: "my-org", "team.id": "42" }); + }); +}); + +describe("buildCreateServerPayload transport and naming", () => { + it("maps the UI-only openapi transport to http for the backend", () => { + const payload = payloadOf(build({ transport: "openapi", spec_path: "https://api.example.com/openapi.json" })); + expect(payload.transport).toBe("http"); + }); + + it("leaves http and sse transports untouched", () => { + expect(payloadOf(build({ transport: "sse" })).transport).toBe("sse"); + }); + + it("falls back to the stdio JSON's server key when the name field is blank", () => { + const payload = payloadOf( + build( + { transport: "stdio", stdio_config: '{"mcpServers":{"my-server":{"command":"npx"}}}' }, + { transportType: "stdio" }, + ), + ); + expect(payload.server_name).toBe("my_server"); + expect(payload.command).toBe("npx"); + }); + + it("keeps an explicit server name over the stdio JSON's key", () => { + const payload = payloadOf( + build( + { server_name: "Chosen", transport: "stdio", stdio_config: '{"mcpServers":{"my-server":{"command":"npx"}}}' }, + { transportType: "stdio" }, + ), + ); + expect(payload.server_name).toBe("Chosen"); + }); + + it("falls back to the url for mcp_info.server_name when no name is given", () => { + const payload = payloadOf(build({ url: "https://example.com/mcp" })); + expect((payload.mcp_info as Record).server_name).toBe("https://example.com/mcp"); + }); +}); + +describe("buildCreateServerPayload credentials", () => { + it("drops empty, null and undefined credential entries", () => { + const payload = payloadOf( + build({ auth_type: "api_key", credentials: { auth_value: "secret", client_id: "", client_secret: null } }), + ); + expect(payload.credentials).toEqual({ auth_value: "secret" }); + }); + + it("filters blank scopes and omits the key when none survive", () => { + expect( + payloadOf(build({ auth_type: "oauth2", credentials: { client_id: "c", scopes: ["read", "", null] } })) + .credentials, + ).toEqual({ client_id: "c", scopes: ["read"] }); + expect(payloadOf(build({ auth_type: "oauth2", credentials: { client_id: "c", scopes: [] } })).credentials).toEqual({ + client_id: "c", + }); + }); + + it("omits credentials entirely for an auth type that needs none", () => { + const payload = payloadOf(build({ auth_type: "none", credentials: { auth_value: "stale" } })); + expect(payload).not.toHaveProperty("credentials"); + }); + + it.each([["true_passthrough"], ["oauth_delegate"]])( + "persists only the declared app for %s, never minted token material", + (authType) => { + const payload = payloadOf( + build({ + auth_type: authType, + credentials: { + client_id: "org-app", + client_secret: "org-secret", + access_token: "upstream-tok", + refresh_token: "refresh-tok", + expires_in: 3600, + scope: "read", + }, + }), + ); + expect(payload.credentials).toEqual({ client_id: "org-app", client_secret: "org-secret" }); + expect(JSON.stringify(payload)).not.toContain("upstream-tok"); + expect(JSON.stringify(payload)).not.toContain("refresh-tok"); + }, + ); + + it("merges the DCR-minted client into an oauth2 payload", () => { + const payload = payloadOf( + build( + { auth_type: "oauth2", credentials: { access_token: "tok" } }, + { dcrClient: { client_id: "dcr-id", client_secret: "dcr-secret" } }, + ), + ); + expect(payload.credentials).toMatchObject({ + client_id: "dcr-id", + client_secret: "dcr-secret", + access_token: "tok", + }); + }); + + it("never leaks the DCR-minted client onto a non-oauth2 server", () => { + const payload = payloadOf( + build({ auth_type: "true_passthrough" }, { dcrClient: { client_id: "dcr-id", client_secret: "dcr-secret" } }), + ); + expect(JSON.stringify(payload)).not.toContain("dcr-id"); + }); +}); + +describe("buildCreateServerPayload flags", () => { + it.each([["true_passthrough"], ["oauth_delegate"]])("defaults dcr_bridge on for %s", (authType) => { + expect(payloadOf(build({ auth_type: authType })).dcr_bridge).toBe(true); + }); + + it.each([["true_passthrough"], ["oauth_delegate"]])("honours an explicit dcr_bridge false for %s", (authType) => { + expect(payloadOf(build({ auth_type: authType, dcr_bridge: false })).dcr_bridge).toBe(false); + }); + + it.each([["none"], ["api_key"], ["oauth2"]])( + "forces dcr_bridge off for %s even when the form still holds true", + (authType) => { + expect(payloadOf(build({ auth_type: authType, dcr_bridge: true })).dcr_bridge).toBe(false); + }, + ); + + it("stamps the interactive oauth2 flow by default", () => { + expect(payloadOf(build({ auth_type: "oauth2" })).oauth2_flow).toBe("authorization_code"); + }); + + it("stamps client_credentials for an M2M oauth2 server", () => { + expect(payloadOf(build({ auth_type: "oauth2", oauth_flow_type: "m2m" })).oauth2_flow).toBe("client_credentials"); + }); + + it("sends no oauth2_flow for a non-oauth2 server", () => { + expect(payloadOf(build({ auth_type: "api_key", oauth_flow_type: "m2m" }))).not.toHaveProperty("oauth2_flow"); + }); + + it.each([["allow_all_keys"], ["available_on_public_internet"], ["delegate_auth_to_upstream"], ["oauth_passthrough"]])( + "coerces %s to a boolean", + (key) => { + expect(payloadOf(build({ auth_type: "none" }))[key]).toBe(false); + expect(payloadOf(build({ auth_type: "none", [key]: true }))[key]).toBe(true); + }, + ); +}); + +describe("buildCreateServerPayload tool allowlist", () => { + it("marks the allowlist enforced once the admin has touched it, even with nothing selected", () => { + const payload = payloadOf(build({ auth_type: "none" }, { hasToolAllowlistInteraction: true })); + expect((payload.mcp_info as Record).tool_allowlist_enforced).toBe(true); + expect(payload.allowed_tools).toEqual([]); + }); + + it("marks the allowlist enforced when tools are selected without an explicit interaction", () => { + const payload = payloadOf(build({ auth_type: "none" }, { allowedTools: ["search"] })); + expect((payload.mcp_info as Record).tool_allowlist_enforced).toBe(true); + expect(payload.allowed_tools).toEqual(["search"]); + }); + + it("leaves the allowlist unenforced when untouched and empty", () => { + const payload = payloadOf(build({ auth_type: "none" })); + expect((payload.mcp_info as Record).tool_allowlist_enforced).toBe(false); + }); +}); + +describe("buildCreateServerPayload mcp_info", () => { + it("sends a null cost map when nothing is configured and the map when it is", () => { + expect( + (payloadOf(build({ auth_type: "none" })).mcp_info as Record).mcp_server_cost_info, + ).toBeNull(); + const priced = payloadOf(build({ auth_type: "none" }, { costConfig: { default_cost_per_query: 0.01 } })); + expect((priced.mcp_info as Record).mcp_server_cost_info).toEqual({ default_cost_per_query: 0.01 }); + }); + + it("carries the selected logo and drops the raw stdio_config field", () => { + const payload = payloadOf(build({ auth_type: "none", stdio_config: "{}" }, { logoUrl: "https://cdn/logo.png" })); + expect((payload.mcp_info as Record).logo_url).toBe("https://cdn/logo.png"); + expect(payload.stdio_config).toBeUndefined(); + }); +}); From 9c2c79f976bf78a7572c198b2840c62ca7620d3b Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 3 Aug 2026 16:25:35 -0700 Subject: [PATCH 054/124] fix(ui): render Responses API request and response in the logs drawer The Pretty view only parsed the Chat Completions shape (messages / choices[0].message), so any spend log storing the Responses API shape (input / output) rendered an empty Input card and the literal text "No response data available" even though the row held the full request and response. This also hit plain /v1/chat/completions callers, because litellm may route those over the Responses bridge and then store the upstream Responses-shaped body. Parsing now branches on a tagged union covering both shapes, which also replaces the any-typed key sniffing and the role guessing it relied on. --- ui/litellm-dashboard/eslint-suppressions.json | 5 - .../PrettyMessagesView.test.tsx | 120 +++++++++ .../LogDetailsDrawer/prettyMessagesTypes.ts | 16 +- .../LogDetailsDrawer/prettyMessagesUtils.ts | 236 ++++++++++++------ 4 files changed, 299 insertions(+), 78 deletions(-) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 107f66b8f1a..2602b8fa7ad 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -4142,11 +4142,6 @@ "count": 1 } }, - "src/components/view_logs/LogDetailsDrawer/prettyMessagesUtils.ts": { - "no-nested-ternary": { - "count": 1 - } - }, "src/components/view_logs/LogDetailsDrawer/useKeyboardNavigation.ts": { "react-hooks/immutability": { "count": 2 diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.test.tsx index e7295ed7a72..104c421acbf 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.test.tsx @@ -76,6 +76,126 @@ describe("PrettyMessagesView", () => { expect(modelElements.length).toBeGreaterThanOrEqual(1); }); + it("renders a Responses API log, whose body uses input/output instead of messages/choices", () => { + const request = { + model: "gpt-5.6", + input: [{ role: "user", content: "Reply with exactly: hello from responses api" }], + }; + const response = { + output: [ + { + id: "msg_070989277645d4ae", + role: "assistant", + type: "message", + status: "completed", + content: [{ text: "hello from responses api", type: "output_text", annotations: [] }], + }, + ], + }; + + render(); + expect(screen.getByText("Reply with exactly: hello from responses api")).toBeInTheDocument(); + expect(screen.getByText("hello from responses api")).toBeInTheDocument(); + expect(screen.queryByText("No response data available")).not.toBeInTheDocument(); + }); + + it("renders a Responses API tool call, whose output item is a function_call", () => { + const request = { + model: "gpt-5.6", + input: [{ role: "user", content: "What is the weather in San Francisco? Use the tool." }], + }; + const response = { + output: [ + { + id: "fc_08edf6c2312f1485", + name: "get_weather", + type: "function_call", + status: "completed", + call_id: "call_AtO0J9eNy5jgECXzBicMJM8W", + arguments: '{"city":"San Francisco"}', + }, + ], + }; + + render(); + expect(screen.getByText("What is the weather in San Francisco? Use the tool.")).toBeInTheDocument(); + expect(screen.getByText("get_weather")).toBeInTheDocument(); + expect(screen.queryByText("No response data available")).not.toBeInTheDocument(); + }); + + it("renders instructions as the system turn and a bare string input", () => { + const request = { model: "gpt-5.6", instructions: "You are terse.", input: "Say A" }; + const response = { + output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text: "A" }] }], + }; + + render(); + expect(screen.getByText("You are terse.")).toBeInTheDocument(); + expect(screen.getByText("Say A")).toBeInTheDocument(); + expect(screen.getByText("A")).toBeInTheDocument(); + }); + + it("skips reasoning output items rather than rendering them as empty turns", () => { + const request = { input: [{ role: "user", content: "Think then answer" }] }; + const response = { + output: [ + { type: "reasoning", id: "rs_1", summary: [] }, + { type: "message", role: "assistant", content: [{ type: "output_text", text: "answered" }] }, + ], + }; + + render(); + expect(screen.getByText("answered")).toBeInTheDocument(); + expect(screen.queryByText("No response data available")).not.toBeInTheDocument(); + }); + + it("renders a Responses API follow-up turn carrying a prior function_call and its output", () => { + const request = { + input: [ + { role: "user", content: "What is the weather in San Francisco? Use the tool." }, + { + type: "function_call", + name: "get_weather", + call_id: "call_AtO0J9eNy5jgECXzBicMJM8W", + arguments: '{"city":"San Francisco"}', + }, + { type: "function_call_output", call_id: "call_AtO0J9eNy5jgECXzBicMJM8W", output: '{"temp":18}' }, + ], + }; + const response = { + output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text: "It is 18 degrees." }] }], + }; + + render(); + expect(screen.getByText("It is 18 degrees.")).toBeInTheDocument(); + expect(screen.getByText('{"temp":18}')).toBeInTheDocument(); + expect(screen.getByText("TOOL")).toBeInTheDocument(); + }); + + it("maps the developer and legacy function roles onto the roles the drawer renders", () => { + const request = { + messages: [ + { role: "developer", content: "Stay terse." }, + { role: "user", content: "Weather?" }, + { role: "function", name: "get_weather", content: '{"temp":18}' }, + ], + }; + const response = { choices: [{ message: { role: "assistant", content: "18 degrees." } }] }; + + render(); + expect(screen.getByText("Stay terse.")).toBeInTheDocument(); + expect(screen.getByText("TOOL")).toBeInTheDocument(); + expect(screen.queryByText("FUNCTION")).not.toBeInTheDocument(); + }); + + it("still reports missing output when a Responses API log has an empty output array", () => { + const request = { input: [{ role: "user", content: "Hello" }] }; + + render(); + expect(screen.getByText("Hello")).toBeInTheDocument(); + expect(screen.getByText("No response data available")).toBeInTheDocument(); + }); + it("should render standard view when response has results but no realtime events", () => { const request = { messages: [{ role: "user", content: "Test" }], diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesTypes.ts b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesTypes.ts index da5c492e60f..463ba65d6ff 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesTypes.ts +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesTypes.ts @@ -2,17 +2,29 @@ * Type definitions for pretty messages view */ +export type MessageRole = "system" | "user" | "assistant" | "tool"; + export interface ParsedMessage { - role: "system" | "user" | "assistant" | "tool"; + role: MessageRole; content: string; toolCalls?: ToolCall[]; toolCallId?: string; } +export type RequestPayload = + | { kind: "chat"; messages: readonly unknown[] } + | { kind: "responses"; instructions: string; input: string | readonly unknown[] } + | { kind: "unknown" }; + +export type ResponsePayload = + | { kind: "chat"; choices: readonly unknown[] } + | { kind: "responses"; output: readonly unknown[] } + | { kind: "unknown" }; + export interface ToolCall { id: string; name: string; - arguments: Record; + arguments: Record; } export interface ParsedMessages { diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesUtils.ts b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesUtils.ts index 09b8f551c1d..1f73da1d30e 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesUtils.ts +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesUtils.ts @@ -2,7 +2,15 @@ * Utility functions for parsing and formatting messages for pretty view */ -import { ParsedMessage, ParsedMessages, RoleStyle } from "./prettyMessagesTypes"; +import { + MessageRole, + ParsedMessage, + ParsedMessages, + RequestPayload, + ResponsePayload, + RoleStyle, + ToolCall, +} from "./prettyMessagesTypes"; /** * Role color styles for message cards - minimal, professional design @@ -35,102 +43,188 @@ export const ROLE_STYLES: Record = { }, }; +type UnknownRecord = Record; + +const isRecord = (value: unknown): value is UnknownRecord => + typeof value === "object" && value !== null && !Array.isArray(value); + +const asString = (value: unknown): string => (typeof value === "string" ? value : ""); + +const ROLES: readonly MessageRole[] = ["system", "user", "assistant", "tool"]; + +const toRole = (value: unknown, fallback: MessageRole): MessageRole => { + if (value === "developer") return "system"; + if (value === "function") return "tool"; + return ROLES.includes(value as MessageRole) ? (value as MessageRole) : fallback; +}; + +const classifyRequest = (request: unknown): RequestPayload => { + if (Array.isArray(request)) return { kind: "chat", messages: request }; + if (!isRecord(request)) return { kind: "unknown" }; + if (Array.isArray(request.messages)) return { kind: "chat", messages: request.messages }; + const { input } = request; + if (typeof input === "string" || Array.isArray(input)) { + return { kind: "responses", instructions: asString(request.instructions), input }; + } + return { kind: "unknown" }; +}; + +const classifyResponse = (response: unknown): ResponsePayload => { + if (!isRecord(response)) return { kind: "unknown" }; + if (Array.isArray(response.choices)) return { kind: "chat", choices: response.choices }; + if (Array.isArray(response.output)) return { kind: "responses", output: response.output }; + return { kind: "unknown" }; +}; + /** * Parse request messages and response message from log data */ -export const parseMessages = (request: any, response: any): ParsedMessages => { - // Parse request messages. `request` is either the raw request body - // ({ messages: [...] }) or, when prompts come from cold storage, the bare - // messages array itself. - const requestMessages: ParsedMessage[] = []; +export const parseMessages = (request: unknown, response: unknown): ParsedMessages => ({ + requestMessages: parseRequestMessages(classifyRequest(request)), + responseMessage: parseResponseMessage(classifyResponse(response)), +}); - const requestMessageList = Array.isArray(request) - ? request - : Array.isArray(request?.messages) - ? request.messages - : []; - - requestMessageList.forEach((msg: any) => { - requestMessages.push({ - role: msg.role || "user", - content: parseMessageContent(msg.content), - toolCallId: msg.tool_call_id, - }); - }); - - // Parse response message - let responseMessage: ParsedMessage | null = null; - const responseMsg = response?.choices?.[0]?.message; - - if (responseMsg) { - responseMessage = { - role: responseMsg.role || "assistant", - content: responseMsg.content || "", - toolCalls: parseToolCalls(responseMsg.tool_calls), - }; +const parseRequestMessages = (payload: RequestPayload): ParsedMessage[] => { + switch (payload.kind) { + case "chat": + return payload.messages.map(parseChatMessage); + case "responses": { + const instructions: ParsedMessage[] = payload.instructions + ? [{ role: "system", content: payload.instructions }] + : []; + const input: ParsedMessage[] = + typeof payload.input === "string" + ? [{ role: "user", content: payload.input }] + : payload.input.flatMap(parseResponsesInputItem); + return [...instructions, ...input]; + } + case "unknown": + return []; } - - return { requestMessages, responseMessage }; }; +const parseResponseMessage = (payload: ResponsePayload): ParsedMessage | null => { + switch (payload.kind) { + case "chat": { + const choice = payload.choices[0]; + const message = isRecord(choice) ? choice.message : undefined; + if (!isRecord(message)) return null; + return { + role: toRole(message.role, "assistant"), + content: parseMessageContent(message.content), + toolCalls: parseChatToolCalls(message.tool_calls), + }; + } + case "responses": { + const content = payload.output + .filter((item): item is UnknownRecord => isRecord(item) && item.type === "message") + .map((item) => parseMessageContent(item.content)) + .filter((text) => text.length > 0) + .join("\n"); + const toolCalls = payload.output.filter(isResponsesFunctionCall).map(parseResponsesFunctionCall); + if (content.length === 0 && toolCalls.length === 0) return null; + return { role: "assistant", content, toolCalls: toolCalls.length > 0 ? toolCalls : undefined }; + } + case "unknown": + return null; + } +}; + +const parseChatMessage = (message: unknown): ParsedMessage => { + if (!isRecord(message)) return { role: "user", content: parseMessageContent(message) }; + return { + role: toRole(message.role, "user"), + content: parseMessageContent(message.content), + toolCalls: parseChatToolCalls(message.tool_calls), + toolCallId: typeof message.tool_call_id === "string" ? message.tool_call_id : undefined, + }; +}; + +const parseResponsesInputItem = (item: unknown): ParsedMessage[] => { + if (typeof item === "string") return [{ role: "user", content: item }]; + if (!isRecord(item)) return []; + if (item.type === "function_call") { + return [{ role: "assistant", content: "", toolCalls: [parseResponsesFunctionCall(item)] }]; + } + if (item.type === "function_call_output") { + return [{ role: "tool", content: parseMessageContent(item.output), toolCallId: asString(item.call_id) }]; + } + if (item.type === "reasoning") return []; + if ("role" in item || "content" in item) { + return [{ role: toRole(item.role, "user"), content: parseMessageContent(item.content) }]; + } + return []; +}; + +const isResponsesFunctionCall = (item: unknown): item is UnknownRecord => + isRecord(item) && item.type === "function_call"; + +const parseResponsesFunctionCall = (item: UnknownRecord): ToolCall => ({ + id: asString(item.call_id) || asString(item.id), + name: asString(item.name) || "unknown", + arguments: parseToolArguments(item.arguments), +}); + /** * Parse message content - handle strings and content arrays (for vision, etc.) */ -const parseMessageContent = (content: any): string => { - if (typeof content === "string") { - return content; - } - - if (Array.isArray(content)) { - // Handle content arrays (vision API format) - return content - .map((item) => { - if (typeof item === "string") return item; - if (item.type === "text") return item.text; - if (item.type === "image_url") return "[Image]"; - return JSON.stringify(item); - }) - .join("\n"); - } - - // Fallback to JSON string for complex content +const parseMessageContent = (content: unknown): string => { + if (typeof content === "string") return content; + if (content === null || content === undefined) return ""; + if (Array.isArray(content)) return content.map(parseContentPart).join("\n"); return JSON.stringify(content); }; +const parseContentPart = (part: unknown): string => { + if (typeof part === "string") return part; + if (!isRecord(part)) return JSON.stringify(part); + switch (part.type) { + case "text": + case "input_text": + case "output_text": + return asString(part.text); + case "refusal": + return asString(part.refusal); + case "image_url": + case "input_image": + return "[Image]"; + case "input_file": + return "[File]"; + case "input_audio": + return "[Audio]"; + default: + return JSON.stringify(part); + } +}; + /** * Parse tool calls from response message */ -const parseToolCalls = ( - toolCalls: any[], -): - | Array<{ - id: string; - name: string; - arguments: Record; - }> - | undefined => { - if (!toolCalls || !Array.isArray(toolCalls)) return undefined; - - return toolCalls.map((tc) => ({ - id: tc.id || "", - name: tc.function?.name || "unknown", - arguments: parseToolArguments(tc.function?.arguments), - })); +const parseChatToolCalls = (toolCalls: unknown): ToolCall[] | undefined => { + if (!Array.isArray(toolCalls)) return undefined; + return toolCalls.map((toolCall) => { + const call = isRecord(toolCall) ? toolCall : {}; + const fn = isRecord(call.function) ? call.function : {}; + return { + id: asString(call.id), + name: asString(fn.name) || "unknown", + arguments: parseToolArguments(fn.arguments), + }; + }); }; /** * Parse tool arguments - handle both string and object formats */ -const parseToolArguments = (args: any): Record => { +const parseToolArguments = (args: unknown): Record => { if (!args) return {}; - if (typeof args === "string") { try { - return JSON.parse(args); + const parsed: unknown = JSON.parse(args); + return isRecord(parsed) ? parsed : { raw: args }; } catch { return { raw: args }; } } - - return args; + return isRecord(args) ? args : {}; }; From bd8b377dd7f672cad6354c5d26a2c159a16d726c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 3 Aug 2026 23:30:31 +0000 Subject: [PATCH 055/124] chore(typing): clear basedpyright Any errors in budget reset, access groups, and cache settings Replace Any seams in three proxy modules with real types so the values keep their shape through the call graph: - reset_budget_job: Protocols for the Prisma spend-linked tables, the reset batcher, and each cascade row shape, with the per-table counter/cache key lambdas promoted to typed module functions so the row type is inferred - access_group_endpoints: Protocols for the access group record, the team and key tables, and the transaction handle; record to response conversion now goes through model_validate on the record dict - cache_settings_endpoints: the opaque cache settings blobs are Mapping[str, object] / dict[str, object] instead of Any, keeping Any only on the two returns that feed the dynamic litellm.Cache kwargs bag Whole-tree basedpyright: reportAny 19435 -> 19306, reportExplicitAny 6518 -> 6487, total errors 148372 -> 148117, with no rule above its baseline and no untouched file changed. No behavior changes. --- basedpyright-code-budget.json | 16 +- .../proxy/common_utils/reset_budget_job.py | 143 ++++++++++++++---- .../access_group_endpoints.py | 119 +++++++++++---- .../cache_settings_endpoints.py | 40 ++--- ruff-strict-budget.json | 4 +- type-discipline-budget.json | 4 +- 6 files changed, 240 insertions(+), 86 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index f6dd90077b1..acdf97cb386 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 29813 + "limit": 29684 }, "reportArgumentType": { "limit": 2645 @@ -24,7 +24,7 @@ "limit": 42 }, "reportExplicitAny": { - "limit": 9473 + "limit": 9442 }, "reportFunctionMemberAccess": { "limit": 11 @@ -54,10 +54,10 @@ "limit": 0 }, "reportMissingParameterType": { - "limit": 5855 + "limit": 5848 }, "reportMissingTypeArgument": { - "limit": 15852 + "limit": 15850 }, "reportMissingTypeStubs": { "limit": 41 @@ -99,19 +99,19 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 45324 + "limit": 45297 }, "reportUnknownLambdaType": { "limit": 113 }, "reportUnknownMemberType": { - "limit": 40452 + "limit": 40411 }, "reportUnknownParameterType": { - "limit": 20309 + "limit": 20301 }, "reportUnknownVariableType": { - "limit": 31978 + "limit": 31968 }, "reportUnnecessaryCast": { "limit": 177 diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 3087e356f99..6ec441a0e06 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -1,12 +1,13 @@ import asyncio import json import time -from collections.abc import Callable +from collections.abc import Callable, Mapping, Sequence from datetime import datetime, timezone -from typing import Any, Literal +from typing import Literal, Protocol, TypeVar import litellm from litellm._logging import verbose_proxy_logger +from litellm.caching.dual_cache import DualCache from litellm.constants import GLOBAL_PROXY_SPEND_CACHE_KEY, LITELLM_PROXY_BUDGET_NAME from litellm.proxy._types import ( LiteLLM_BudgetTableFull, @@ -33,6 +34,98 @@ from litellm.repositories.verification_token_repository import ( ) from litellm.types.services import ServiceTypes +_RowT = TypeVar("_RowT") +_RowT_co = TypeVar("_RowT_co", covariant=True) + + +class _PrismaRecord(Protocol): + def dict(self) -> Mapping[str, object]: ... + + +class _BatchTable(Protocol): + def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> None: ... + + +class _ResetBatcher(Protocol): + @property + def litellm_verificationtoken(self) -> _BatchTable: ... + + @property + def litellm_usertable(self) -> _BatchTable: ... + + @property + def litellm_teamtable(self) -> _BatchTable: ... + + async def commit(self) -> None: ... + + +class _EndUserTable(Protocol): + async def find_many(self, where: Mapping[str, object]) -> Sequence[_PrismaRecord]: ... + + +class _SpendLinkedTable(Protocol[_RowT_co]): + async def find_many(self, where: Mapping[str, object]) -> Sequence[_RowT_co]: ... + + async def update_many(self, where: Mapping[str, object], data: Mapping[str, object]) -> int: ... + + +class _TeamMembershipRow(Protocol): + @property + def user_id(self) -> str: ... + + @property + def team_id(self) -> str: ... + + +class _KeyRow(Protocol): + @property + def token(self) -> str: ... + + +class _OrgRow(Protocol): + @property + def organization_id(self) -> str: ... + + +class _TagRow(Protocol): + @property + def tag_name(self) -> str: ... + + +def _team_membership_counter_key(row: _TeamMembershipRow) -> str: + return f"spend:team_member:{row.user_id}:{row.team_id}" + + +def _team_membership_cache_key(row: _TeamMembershipRow) -> str: + return f"{row.team_id}_{row.user_id}" + + +def _key_counter_key(row: _KeyRow) -> str: + return f"spend:key:{row.token}" + + +def _key_cache_key(row: _KeyRow) -> str: + return row.token + + +def _org_counter_key(row: _OrgRow) -> str: + return f"spend:org:{row.organization_id}" + + +def _org_cache_keys(row: _OrgRow) -> Sequence[str]: + return [ + f"org_id:{row.organization_id}", + f"org_id:{row.organization_id}:with_budget", + ] + + +def _tag_counter_key(row: _TagRow) -> str: + return f"spend:tag:{row.tag_name}" + + +def _tag_cache_key(row: _TagRow) -> str: + return f"tag:{row.tag_name}" + class ResetBudgetJob: """ @@ -134,11 +227,11 @@ class ResetBudgetJob: async def _cascade_reset_spend_for_budget_link( self, budgets_to_reset: list[LiteLLM_BudgetTableFull], - table: Any, - counter_key_fn: Callable[[Any], str], + table: "_SpendLinkedTable[_RowT]", + counter_key_fn: Callable[[_RowT], str], log_subject: str, - extra_where: dict | None = None, - cache_key_fn: Callable[[Any], str | list[str]] | None = None, + extra_where: dict[str, object] | None = None, + cache_key_fn: Callable[[_RowT], str | Sequence[str]] | None = None, ): """ Generic cascade: zero spend on rows whose budget_id is in the reset set. @@ -151,14 +244,14 @@ class ResetBudgetJob: if not budget_ids: return - where: dict = {"budget_id": {"in": budget_ids}} + where: dict[str, object] = {"budget_id": {"in": budget_ids}} if extra_where: where.update(extra_where) try: - rows = await table.find_many(where=where) + rows: Sequence[_RowT] = await table.find_many(where=where) except Exception as e: - rows = [] + rows = () verbose_proxy_logger.warning("Failed to fetch %s for counter invalidation: %s", log_subject, e) update_result = await table.update_many(where=where, data={"spend": 0}) @@ -181,9 +274,9 @@ class ResetBudgetJob: return await self._cascade_reset_spend_for_budget_link( budgets_to_reset=budgets_to_reset, table=TeamMembershipRepository(self.prisma_client).table, - counter_key_fn=lambda m: f"spend:team_member:{m.user_id}:{m.team_id}", + counter_key_fn=_team_membership_counter_key, log_subject="team memberships", - cache_key_fn=lambda m: f"{m.team_id}_{m.user_id}", + cache_key_fn=_team_membership_cache_key, ) async def reset_budget_for_keys_linked_to_budgets(self, budgets_to_reset: list[LiteLLM_BudgetTableFull]): @@ -196,10 +289,10 @@ class ResetBudgetJob: return await self._cascade_reset_spend_for_budget_link( budgets_to_reset=budgets_to_reset, table=VerificationTokenRepository(self.prisma_client).table, - counter_key_fn=lambda k: f"spend:key:{k.token}", + counter_key_fn=_key_counter_key, log_subject="keys", extra_where={"budget_duration": None, "spend": {"gt": 0}}, - cache_key_fn=lambda k: k.token, + cache_key_fn=_key_cache_key, ) async def reset_budget_for_orgs_linked_to_budgets(self, budgets_to_reset: list[LiteLLM_BudgetTableFull]): @@ -209,13 +302,10 @@ class ResetBudgetJob: return await self._cascade_reset_spend_for_budget_link( budgets_to_reset=budgets_to_reset, table=OrganizationRepository(self.prisma_client).table, - counter_key_fn=lambda o: f"spend:org:{o.organization_id}", + counter_key_fn=_org_counter_key, log_subject="orgs", extra_where={"spend": {"gt": 0}}, - cache_key_fn=lambda o: [ - f"org_id:{o.organization_id}", - f"org_id:{o.organization_id}:with_budget", - ], + cache_key_fn=_org_cache_keys, ) async def reset_budget_for_tags_linked_to_budgets(self, budgets_to_reset: list[LiteLLM_BudgetTableFull]): @@ -233,10 +323,10 @@ class ResetBudgetJob: return await self._cascade_reset_spend_for_budget_link( budgets_to_reset=budgets_to_reset, table=TagRepository(self.prisma_client).table, - counter_key_fn=lambda t: f"spend:tag:{t.tag_name}", + counter_key_fn=_tag_counter_key, log_subject="tags", extra_where={"spend": {"gt": 0}}, - cache_key_fn=lambda t: f"tag:{t.tag_name}", + cache_key_fn=_tag_cache_key, ) async def reset_budget_for_litellm_budget_table(self): @@ -376,13 +466,14 @@ class ResetBudgetJob: rely on the default budget (litellm.max_end_user_budget_id) applied in-memory during auth checks. """ - rows = await EndUserRepository(self.prisma_client).table.find_many( + table: _EndUserTable = EndUserRepository(self.prisma_client).table + rows = await table.find_many( where={ "budget_id": None, "spend": {"gt": 0}, }, ) - return [LiteLLM_EndUserTable(**row.dict()) for row in rows] + return [LiteLLM_EndUserTable.model_validate(row.dict()) for row in rows] async def _write_key_reset_updates(self, updated_keys: list[LiteLLM_VerificationToken]) -> None: """ @@ -395,7 +486,7 @@ class ResetBudgetJob: aborts the entire batch — silently leaving spend over the cap and budget_reset_at unchanged forever. """ - batcher = self.prisma_client.db.batch_() + batcher: _ResetBatcher = self.prisma_client.db.batch_() for k in updated_keys: token = getattr(k, "token", None) if token is None: @@ -414,7 +505,7 @@ class ResetBudgetJob: that trips Prisma's DataError on rows carrying unrecognised fields (see #27730). """ - batcher = self.prisma_client.db.batch_() + batcher: _ResetBatcher = self.prisma_client.db.batch_() for u in updated_users: user_id = getattr(u, "user_id", None) if user_id is None: @@ -433,7 +524,7 @@ class ResetBudgetJob: that trips Prisma's DataError on rows carrying unrecognised fields (see #27730). """ - batcher = self.prisma_client.db.batch_() + batcher: _ResetBatcher = self.prisma_client.db.batch_() for t in updated_teams: team_id = getattr(t, "team_id", None) if team_id is None: @@ -688,7 +779,7 @@ class ResetBudgetJob: async def _reset_expired_window( window: dict, counter_key: str, - spend_counter_cache: Any, + spend_counter_cache: DualCache, now: datetime, reset_settings: BudgetResetSettings, ) -> bool: diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index 13e45b17090..4c870c3fb1d 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -1,3 +1,6 @@ +from collections.abc import Mapping, Sequence +from typing import Protocol + from fastapi import APIRouter, Depends, HTTPException, status from litellm._logging import verbose_proxy_logger @@ -29,6 +32,74 @@ router = APIRouter( ) +class _AccessGroupRecord(Protocol): + @property + def access_group_id(self) -> str: ... + + @property + def assigned_team_ids(self) -> Sequence[str] | None: ... + + @property + def assigned_key_ids(self) -> Sequence[str] | None: ... + + def dict(self) -> Mapping[str, object]: ... + + +class _TeamRecord(Protocol): + @property + def team_id(self) -> str: ... + + @property + def access_group_ids(self) -> Sequence[str] | None: ... + + +class _KeyRecord(Protocol): + @property + def token(self) -> str: ... + + @property + def access_group_ids(self) -> Sequence[str] | None: ... + + +class _AccessGroupTable(Protocol): + async def find_unique(self, where: Mapping[str, object]) -> _AccessGroupRecord | None: ... + + async def find_many(self, order: Mapping[str, object]) -> Sequence[_AccessGroupRecord]: ... + + async def create(self, data: Mapping[str, object]) -> _AccessGroupRecord: ... + + async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> _AccessGroupRecord: ... + + async def delete(self, where: Mapping[str, object]) -> object: ... + + +class _TeamTable(Protocol): + async def find_unique(self, where: Mapping[str, object]) -> _TeamRecord | None: ... + + async def find_many(self, where: Mapping[str, object]) -> Sequence[_TeamRecord]: ... + + async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> object: ... + + +class _KeyTable(Protocol): + async def find_unique(self, where: Mapping[str, object]) -> _KeyRecord | None: ... + + async def find_many(self, where: Mapping[str, object]) -> Sequence[_KeyRecord]: ... + + async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> object: ... + + +class _AccessGroupTx(Protocol): + @property + def litellm_accessgrouptable(self) -> _AccessGroupTable: ... + + @property + def litellm_teamtable(self) -> _TeamTable: ... + + @property + def litellm_verificationtoken(self) -> _KeyTable: ... + + def _require_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None: if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: raise HTTPException( @@ -48,29 +119,16 @@ def _require_admin_view(user_api_key_dict: UserAPIKeyAuth) -> None: ) -def _record_to_response(record) -> AccessGroupResponse: - return AccessGroupResponse( - access_group_id=record.access_group_id, - access_group_name=record.access_group_name, - description=record.description, - access_model_names=record.access_model_names, - access_mcp_server_ids=record.access_mcp_server_ids, - access_agent_ids=record.access_agent_ids, - assigned_team_ids=record.assigned_team_ids, - assigned_key_ids=record.assigned_key_ids, - created_at=record.created_at, - created_by=record.created_by, - updated_at=record.updated_at, - updated_by=record.updated_by, - ) +def _record_to_response(record: _AccessGroupRecord) -> AccessGroupResponse: + return AccessGroupResponse.model_validate(record.dict()) -def _record_to_access_group_table(record) -> LiteLLM_AccessGroupTable: +def _record_to_access_group_table(record: _AccessGroupRecord) -> LiteLLM_AccessGroupTable: """Convert a Prisma record to a LiteLLM_AccessGroupTable pydantic object for caching.""" - return LiteLLM_AccessGroupTable(**record.dict()) + return LiteLLM_AccessGroupTable.model_validate(record.dict()) -async def _cache_access_group_record(record) -> None: +async def _cache_access_group_record(record: _AccessGroupRecord) -> None: """ Cache an access group Prisma record in the user_api_key_cache. @@ -109,7 +167,7 @@ async def _invalidate_cache_access_group(access_group_id: str) -> None: # --------------------------------------------------------------------------- -async def _sync_add_access_group_to_teams(tx, team_ids: list[str], access_group_id: str) -> None: +async def _sync_add_access_group_to_teams(tx: _AccessGroupTx, team_ids: list[str], access_group_id: str) -> None: """Add access_group_id to each team's access_group_ids (idempotent).""" for team_id in team_ids: team = await tx.litellm_teamtable.find_unique(where={"team_id": team_id}) @@ -120,18 +178,18 @@ async def _sync_add_access_group_to_teams(tx, team_ids: list[str], access_group_ ) -async def _sync_remove_access_group_from_teams(tx, team_ids: list[str], access_group_id: str) -> None: +async def _sync_remove_access_group_from_teams(tx: _AccessGroupTx, team_ids: list[str], access_group_id: str) -> None: """Remove access_group_id from each team's access_group_ids (idempotent).""" for team_id in team_ids: team = await tx.litellm_teamtable.find_unique(where={"team_id": team_id}) if team is not None and access_group_id in (team.access_group_ids or []): await tx.litellm_teamtable.update( where={"team_id": team_id}, - data={"access_group_ids": [ag for ag in team.access_group_ids if ag != access_group_id]}, + data={"access_group_ids": [ag for ag in (team.access_group_ids or ()) if ag != access_group_id]}, ) -async def _sync_add_access_group_to_keys(tx, key_tokens: list[str], access_group_id: str) -> None: +async def _sync_add_access_group_to_keys(tx: _AccessGroupTx, key_tokens: list[str], access_group_id: str) -> None: """Add access_group_id to each key's access_group_ids (idempotent).""" for token in key_tokens: key = await tx.litellm_verificationtoken.find_unique(where={"token": token}) @@ -142,14 +200,14 @@ async def _sync_add_access_group_to_keys(tx, key_tokens: list[str], access_group ) -async def _sync_remove_access_group_from_keys(tx, key_tokens: list[str], access_group_id: str) -> None: +async def _sync_remove_access_group_from_keys(tx: _AccessGroupTx, key_tokens: list[str], access_group_id: str) -> None: """Remove access_group_id from each key's access_group_ids (idempotent).""" for token in key_tokens: key = await tx.litellm_verificationtoken.find_unique(where={"token": token}) if key is not None and access_group_id in (key.access_group_ids or []): await tx.litellm_verificationtoken.update( where={"token": token}, - data={"access_group_ids": [ag for ag in key.access_group_ids if ag != access_group_id]}, + data={"access_group_ids": [ag for ag in (key.access_group_ids or ()) if ag != access_group_id]}, ) @@ -280,6 +338,7 @@ async def create_access_group( prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) try: + tx: _AccessGroupTx async with prisma_client.db.tx() as tx: existing = await tx.litellm_accessgrouptable.find_unique( where={"access_group_name": data.access_group_name} @@ -347,7 +406,8 @@ async def list_access_groups( _require_admin_view(user_api_key_dict) prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) - records = await AccessGroupRepository(prisma_client).table.find_many(order={"created_at": "desc"}) + table: _AccessGroupTable = AccessGroupRepository(prisma_client).table + records = await table.find_many(order={"created_at": "desc"}) return [_record_to_response(r) for r in records] @@ -362,7 +422,8 @@ async def get_access_group( _require_admin_view(user_api_key_dict) prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) - record = await AccessGroupRepository(prisma_client).table.find_unique(where={"access_group_id": access_group_id}) + table: _AccessGroupTable = AccessGroupRepository(prisma_client).table + record = await table.find_unique(where={"access_group_id": access_group_id}) if record is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -408,6 +469,7 @@ async def update_access_group( keys_to_remove: list[str] = [] try: + tx: _AccessGroupTx async with prisma_client.db.tx() as tx: # Read inside the transaction so delta computation is consistent with the write, # avoiding a TOCTOU race where a concurrent update could make deltas stale. @@ -480,6 +542,7 @@ async def delete_access_group( affected_team_ids: list[str] = [] affected_key_tokens: list[str] = [] + tx: _AccessGroupTx async with prisma_client.db.tx() as tx: existing = await tx.litellm_accessgrouptable.find_unique(where={"access_group_id": access_group_id}) if existing is None: @@ -512,7 +575,7 @@ async def delete_access_group( for team in teams_with_group: await tx.litellm_teamtable.update( where={"team_id": team.team_id}, - data={"access_group_ids": [ag for ag in (team.access_group_ids or []) if ag != access_group_id]}, + data={"access_group_ids": [ag for ag in (team.access_group_ids or ()) if ag != access_group_id]}, ) # Use _sync_remove only for out-of-sync teams not found by the hasSome query. out_of_sync_team_ids = set(existing.assigned_team_ids or []) - {t.team_id for t in teams_with_group} @@ -522,7 +585,7 @@ async def delete_access_group( for key in keys_with_group: await tx.litellm_verificationtoken.update( where={"token": key.token}, - data={"access_group_ids": [ag for ag in (key.access_group_ids or []) if ag != access_group_id]}, + data={"access_group_ids": [ag for ag in (key.access_group_ids or ()) if ag != access_group_id]}, ) # Use _sync_remove only for out-of-sync keys not found by the hasSome query. out_of_sync_key_tokens = set(existing.assigned_key_ids or []) - {k.token for k in keys_with_group} diff --git a/litellm/proxy/management_endpoints/cache_settings_endpoints.py b/litellm/proxy/management_endpoints/cache_settings_endpoints.py index e08bc13a14d..439449403c4 100644 --- a/litellm/proxy/management_endpoints/cache_settings_endpoints.py +++ b/litellm/proxy/management_endpoints/cache_settings_endpoints.py @@ -63,7 +63,7 @@ _REDACTED_VALUE = "***REDACTED***" _URL_OVERRIDDEN_CONNECTION_FIELDS: frozenset = frozenset({"host", "port", "db", "password", "username"}) -def _resolve_cache_url_precedence(settings: Mapping[str, Any]) -> dict[str, Any]: +def _resolve_cache_url_precedence(settings: Mapping[str, object]) -> dict[str, Any]: """Return cache settings with the url-vs-discrete-fields ambiguity resolved. When a full ``url`` is supplied it wins: the discrete @@ -80,7 +80,7 @@ def _resolve_cache_url_precedence(settings: Mapping[str, Any]) -> dict[str, Any] return {k: v for k, v in settings.items() if k not in _URL_OVERRIDDEN_CONNECTION_FIELDS} -def _parse_stored_settings(cache_settings_value: object) -> dict[str, Any]: +def _parse_stored_settings(cache_settings_value: object) -> dict[str, object]: """Normalize a stored cache_settings blob to a dict. The prisma column comes back as either a JSON string or an already-parsed @@ -91,7 +91,7 @@ def _parse_stored_settings(cache_settings_value: object) -> dict[str, Any]: return parsed if isinstance(parsed, dict) else {} -def _overlay_environment(stored: Mapping[str, Any]) -> dict[str, Any]: +def _overlay_environment(stored: Mapping[str, object]) -> dict[str, object]: """Fill connection fields from the REDIS_* environment the cache actually reads. A response cache pointed at Redis resolves host/port/password/etc. from the @@ -113,7 +113,7 @@ def _overlay_environment(stored: Mapping[str, Any]) -> dict[str, Any]: return effective -def _redact_credentials(settings: Mapping[str, Any]) -> dict[str, Any]: +def _redact_credentials(settings: Mapping[str, object]) -> dict[str, object]: """Replace credential-bearing values with a fixed marker, keeping the rest. The marker is unambiguous on the way back in: an admin who edits an @@ -164,7 +164,7 @@ def _target_repr(value: object) -> str: return str(value) -def _saved_secret_is_reusable(incoming: Mapping[str, Any], saved: Mapping[str, Any]) -> bool: +def _saved_secret_is_reusable(incoming: Mapping[str, object], saved: Mapping[str, object]) -> bool: """Whether a stored credential may be restored for this request. A stored secret belongs to the stored connection target, so it is reused only @@ -197,7 +197,7 @@ def _saved_secret_is_reusable(incoming: Mapping[str, Any], saved: Mapping[str, A return True -def _merge_over_saved(incoming: Mapping[str, Any], saved: Mapping[str, Any]) -> dict[str, Any]: +def _merge_over_saved(incoming: Mapping[str, object], saved: Mapping[str, object]) -> dict[str, Any]: """Keep the stored secret behind any credential the caller echoed back redacted or omitted. GET returns credentials as the marker and the form never re-prefills a @@ -239,7 +239,7 @@ def _merge_over_saved(incoming: Mapping[str, Any], saved: Mapping[str, Any]) -> return merged -def _redact_settings(settings: Mapping[str, Any] | None) -> dict[str, Any]: +def _redact_settings(settings: Mapping[str, object] | None) -> dict[str, object]: """Replace every value in a settings map with a fixed marker. Cache config carries Redis credentials (passwords, connection strings). @@ -268,8 +268,8 @@ def _log_audit_task_exception(task: "asyncio.Task[None]") -> None: async def _emit_cache_settings_audit_log( *, action: AUDIT_ACTIONS, - before_settings: Mapping[str, Any] | None, - after_settings: Mapping[str, Any] | None, + before_settings: Mapping[str, object] | None, + after_settings: Mapping[str, object] | None, user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: str | None, ) -> None: @@ -313,17 +313,17 @@ class CacheSettingsManager: Tracks last cache params to avoid unnecessary reinitialization. """ - _last_cache_params: dict[str, Any] | None = None + _last_cache_params: dict[str, object] | None = None @staticmethod - def _cache_params_equal(params1: dict[str, Any], params2: dict[str, Any]) -> bool: + def _cache_params_equal(params1: dict[str, object], params2: dict[str, object]) -> bool: """ Compare two cache parameter dictionaries for equality. Normalizes values and filters out UI-only fields. """ # Normalize by removing None values and UI-only fields - def normalize(params: dict[str, Any]) -> dict[str, Any]: + def normalize(params: dict[str, object]) -> dict[str, object]: normalized = {} for k, v in params.items(): if k == "redis_type": # Skip UI-only field @@ -390,7 +390,7 @@ class CacheSettingsManager: ) @staticmethod - def update_cache_params(cache_params: dict[str, Any]): + def update_cache_params(cache_params: dict[str, object]): """ Update the last cache params after initialization. Called after cache settings are updated via the API. @@ -400,12 +400,12 @@ class CacheSettingsManager: class CacheSettingsResponse(BaseModel): fields: list[CacheSettingsField] = Field(description="List of all configurable cache settings with metadata") - current_values: dict[str, Any] = Field(description="Current values of cache settings") + current_values: dict[str, object] = Field(description="Current values of cache settings") redis_type_descriptions: dict[str, str] = Field(description="Descriptions for each Redis type option") class CacheTestRequest(BaseModel): - cache_settings: dict[str, Any] = Field(description="Cache settings to test connection with") + cache_settings: dict[str, object] = Field(description="Cache settings to test connection with") class CacheTestResponse(BaseModel): @@ -415,7 +415,7 @@ class CacheTestResponse(BaseModel): class CacheSettingsUpdateRequest(BaseModel): - cache_settings: dict[str, Any] = Field(description="Cache settings to save") + cache_settings: dict[str, object] = Field(description="Cache settings to save") @router.get( @@ -441,7 +441,7 @@ async def get_cache_settings( cache_fields = [field.model_copy(deep=True) for field in CACHE_SETTINGS_FIELDS] # Read the stored settings (decrypted); an env-only cache has none. - stored: dict[str, Any] = {} + stored: dict[str, object] = {} if prisma_client is not None: cache_config = await CacheConfigRepository(prisma_client).table.find_unique(where={"id": "cache_config"}) if cache_config is not None and cache_config.cache_settings: @@ -507,7 +507,7 @@ async def test_cache_connection( # A credential the form left untouched arrives redacted; resolve it back # to the stored secret so the test connects with the real password. A # lookup failure must not block the test, so fall back to no stored row. - saved_settings: dict[str, Any] = {} + saved_settings: dict[str, object] = {} if prisma_client is not None: try: existing_row = await CacheConfigRepository(prisma_client).table.find_unique( @@ -590,8 +590,8 @@ async def update_cache_settings( # Read the stored row first: its decrypted values back any credential the # caller echoed back redacted, and its key set drives the audit diff. existing_row = await CacheConfigRepository(prisma_client).table.find_unique(where={"id": "cache_config"}) - before_settings: dict[str, Any] | None = None - saved_settings: dict[str, Any] = {} + before_settings: dict[str, object] | None = None + saved_settings: dict[str, object] = {} if existing_row is not None and existing_row.cache_settings: before_settings = _parse_stored_settings(existing_row.cache_settings) saved_settings = proxy_config._decrypt_db_variables(variables_dict=before_settings) diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index d3ef01940bb..386108169bc 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -1,6 +1,6 @@ { "ANN001": { - "limit": 3104 + "limit": 3097 }, "ANN002": { "limit": 69 @@ -24,7 +24,7 @@ "limit": 130 }, "ANN401": { - "limit": 1851 + "limit": 1849 }, "ASYNC230": { "limit": 14 diff --git a/type-discipline-budget.json b/type-discipline-budget.json index f071c381916..a2cf4e351f5 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 23350 + "limit": 23349 }, "LIT002": { - "limit": 27256 + "limit": 27253 }, "LIT003": { "limit": 292 From c98d595359c7f58572c0d7cb37769fc85bfd419e Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 3 Aug 2026 17:07:05 -0700 Subject: [PATCH 056/124] fix(proxy): redact credential headers from request logging copies (#35678) * fix(proxy): redact credential headers from request logging copies clean_headers preserves an Anthropic subscription OAuth token, and other client-supplied provider credentials, so they can be forwarded upstream. The same dict was also stored as proxy_server_request["headers"] and metadata["headers"], so those credentials reached every logging callback and the SpendLogs proxy_server_request column that the Admin UI logs page renders. Build the observability facing copies through redact_credential_headers, and drop the transport-only keys (provider_specific_header, headers, api_key) from the request body snapshot since they have to keep the real values. * fix(proxy): use the redacted header copy in the request debug log The stdout secret filter matches Bearer and sk- shaped values, so an MCP auth token printed by the request-header debug line survived it in cleartext. * fix(proxy): resolve the configured MCP auth header name through the secret manager get_secret_str also consults a configured secret manager, so a deployment that stores the header name there now gets that header masked too. Drops the added comments in favour of a named constant. * perf(proxy): resolve the MCP auth header name once per process get_secret_str issues a blocking secret-manager SDK call when one is configured, and configured_credential_header_names runs on every proxied request. * fix(proxy): read the MCP auth header name live, cache only the secret manager The config reloader rewrites os.environ on an interval and after /config/update, and MCPRequestHandler resolves the same setting per request, so caching the env lookup left a renamed header logged in the clear until the process restarted. Only the blocking secret-manager call stays cached. * refactor(proxy): narrow header redaction to the reported credential set Drops the MCP header-name resolution, its per-request config and secret-manager lookups, and the x-mcp- prefix rule. Those cover a separate credential family than the one this ticket reports and carried their own config-reload staleness surface; they belong in their own change. --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/litellm_pre_call_utils.py | 42 ++++- .../proxy/test_litellm_pre_call_utils.py | 149 +++++++++++++++++- 2 files changed, 185 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 13eb41af751..6864caccea2 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -4,6 +4,7 @@ import json import re import time from collections import OrderedDict +from collections.abc import Mapping from typing import TYPE_CHECKING, Any from fastapi import HTTPException, Request @@ -48,6 +49,12 @@ from litellm.proxy.common_utils.http_parsing_utils import _safe_get_request_head # Cache special headers as a frozenset for O(1) lookup performance _SPECIAL_HEADERS_CACHE = frozenset(v.value.lower() for v in SpecialHeaders._member_map_.values()) +_REDACTED_HEADER_VALUE = "***REDACTED***" +_CREDENTIAL_HEADER_NAMES = SpecialHeaders.litellm_credential_header_names() | frozenset( + {"cookie", "proxy-authorization"} +) +_TRANSPORT_ONLY_CREDENTIAL_KEYS = frozenset({"provider_specific_header", "headers", "api_key"}) + # Matches any header of the form x--session-id (case-insensitive). # Excludes the two explicit litellm headers which are handled with higher priority. _GENERIC_SESSION_ID_HEADER_RE = re.compile(r"^x-.+-session-id$", re.IGNORECASE) @@ -747,6 +754,30 @@ def clean_headers( return clean_headers +def _is_credential_header(header: str) -> bool: + """Whether `header` carries a caller credential rather than request context.""" + return header.lower() in _CREDENTIAL_HEADER_NAMES + + +def redact_credential_headers(headers: Mapping[str, str]) -> Mapping[str, str]: + """Return a copy of `headers` with credential-bearing values masked. + + `clean_headers` deliberately preserves some credential headers so they can be + forwarded to the upstream provider; an Anthropic subscription OAuth token in + `Authorization`, or a client-supplied provider key in `x-api-key`. Those values + must never reach a logging callback or a spend log, so every observability-facing + copy of the header dict is built through this helper while the copy that is + forwarded upstream keeps the real values. + + The returned object is a plain dict; guardrail hooks stamp their own headers onto + the stored copy and the logging callbacks JSON-serialize it. + """ + return { + header: (_REDACTED_HEADER_VALUE if _is_credential_header(header) else value) + for header, value in headers.items() + } + + class LiteLLMProxyRequestSetup: @staticmethod def _get_timeout_from_request(headers: dict) -> float | None: @@ -1443,7 +1474,8 @@ async def add_litellm_data_to_request( _headers, allow_client_message_redaction_opt_out=_allow_client_message_redaction_opt_out, ) - verbose_proxy_logger.debug(f"Request Headers: {_headers}") + _logging_safe_headers = redact_credential_headers(_headers) + verbose_proxy_logger.debug(f"Request Headers: {_logging_safe_headers}") verbose_proxy_logger.debug(f"Raw Headers: {_raw_headers}") if forward_llm_auth and "x-api-key" in _headers: @@ -1464,7 +1496,7 @@ async def add_litellm_data_to_request( data["proxy_server_request"] = { "url": str(request.url), "method": request.method, - "headers": _headers, + "headers": _logging_safe_headers, "body": None, # filled in post-strip; see below "arrival_time": arrival_time, # Track when request arrived at proxy } @@ -1490,7 +1522,7 @@ async def add_litellm_data_to_request( # Expose request headers under the metadata field for guardrails (fixes #17477) if _metadata_variable_name in data and isinstance(data[_metadata_variable_name], dict): - data[_metadata_variable_name]["headers"] = _headers + data[_metadata_variable_name]["headers"] = _logging_safe_headers # check for forwardable headers data = LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group( @@ -1619,7 +1651,7 @@ async def add_litellm_data_to_request( # self-reference — body.proxy_server_request.body would be the same # dict as body, producing an infinite traversal loop for any consumer # that walks the structure. - _body_snapshot_exclude = {"secret_fields", "proxy_server_request"} + _body_snapshot_exclude = frozenset({"secret_fields", "proxy_server_request"}) | _TRANSPORT_ONLY_CREDENTIAL_KEYS _body_snapshot = {k: v for k, v in data.items() if k not in _body_snapshot_exclude} data["proxy_server_request"]["body"] = _body_snapshot @@ -1726,7 +1758,7 @@ async def add_litellm_data_to_request( data[_metadata_variable_name]["user_api_key_team_object_permission_id"] = getattr( user_api_key_dict, "team_object_permission_id", None ) - data[_metadata_variable_name]["headers"] = _headers + data[_metadata_variable_name]["headers"] = _logging_safe_headers data[_metadata_variable_name]["endpoint"] = str(request.url) # Carry the proxy-receive instant via metadata (like `endpoint`) so the # OTel layer can compute pre-request latency, including on the failure diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 37642605088..0e9aac7bf85 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -5559,6 +5559,153 @@ def test_warn_stale_team_alias_once_evicts_oldest_key_beyond_cap(monkeypatch): assert list(pre_call_utils._STALE_TEAM_ALIAS_WARNING_KEYS) == ["key-2", "key-3"] +_OAUTH_TOKEN = "Bearer sk-ant-oat01-regression-token-lit5108" + + +def _all_header_dicts(data: dict, metadata_variable_name: str) -> list[dict]: + metadata = data.get(metadata_variable_name) or {} + proxy_server_request = data["proxy_server_request"] + body = proxy_server_request["body"] + return [ + metadata.get("headers") or {}, + (metadata.get("requester_metadata") or {}).get("headers") or {}, + proxy_server_request["headers"], + (body.get(metadata_variable_name) or {}).get("headers") or {}, + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path, metadata_variable_name", + [ + ("/v1/messages", "litellm_metadata"), + ("/v1/chat/completions", "metadata"), + ], +) +async def test_add_litellm_data_to_request_redacts_oauth_header_from_logging_copies(path, metadata_variable_name): + """The Anthropic subscription token is forwarded upstream but never handed to logging.""" + request_mock = _make_request_mock( + path, + { + "Content-Type": "application/json", + "anthropic-version": "2023-06-01", + "Authorization": _OAUTH_TOKEN, + "x-litellm-api-key": "Bearer sk-virtual-key", + }, + ) + + updated = await add_litellm_data_to_request( + data={"model": "anthropic-claude", "messages": [{"role": "user", "content": "hello"}]}, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"forward_client_headers_to_llm_api": True}, + version="test-version", + ) + + for header_dict in _all_header_dicts(updated, metadata_variable_name): + assert header_dict.get("Authorization") != _OAUTH_TOKEN + assert "sk-ant-oat01" not in json.dumps(header_dict) + + assert "sk-ant-oat01" not in json.dumps(updated["proxy_server_request"], default=repr) + + assert updated["proxy_server_request"]["headers"] is updated[metadata_variable_name]["headers"] + + assert updated["provider_specific_header"]["extra_headers"]["Authorization"] == _OAUTH_TOKEN + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_keeps_every_forwarded_credential_out_of_logging_copies(): + """Credentials kept for transport must not survive anywhere under proxy_server_request.""" + secrets = { + "x-api-key": "sk-byok-provider-key-lit5108", + "cookie": "litellm_jwt=session-token-lit5108", + "proxy-authorization": "Bearer proxy-token-lit5108", + } + request_mock = _make_request_mock( + "/v1/chat/completions", + { + "Content-Type": "application/json", + "x-litellm-api-key": "Bearer sk-virtual-key", + **secrets, + }, + ) + + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": [{"role": "user", "content": "hello"}]}, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={ + "forward_llm_provider_auth_headers": True, + "forward_client_headers_to_llm_api": True, + }, + version="test-version", + ) + + assert updated["api_key"] == secrets["x-api-key"] + assert updated["headers"]["x-api-key"] == secrets["x-api-key"] + + logged = json.dumps(updated["proxy_server_request"], default=repr) + for value in secrets.values(): + assert value not in logged + + + +@pytest.mark.parametrize( + "header, expected_redacted", + [ + ("Authorization", True), + ("X-Api-Key", True), + ("x-goog-api-key", True), + ("Ocp-Apim-Subscription-Key", True), + ("API-Key", True), + ("Cookie", True), + ("Proxy-Authorization", True), + ("anthropic-version", False), + ("user-agent", False), + ], +) +def test_redact_credential_headers_classifies_each_header(header, expected_redacted): + from litellm.proxy.litellm_pre_call_utils import redact_credential_headers + + headers = {header: "secret-value"} + + redacted = redact_credential_headers(headers) + + assert redacted[header] == ("***REDACTED***" if expected_redacted else "secret-value") + assert headers[header] == "secret-value" + + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_debug_log_does_not_print_credentials(): + """The request-header debug line carries values the stdout secret filter does not match.""" + import litellm.proxy.litellm_pre_call_utils as pre_call_utils + + request_mock = _make_request_mock( + "/v1/chat/completions", + { + "Content-Type": "application/json", + "Ocp-Apim-Subscription-Key": "apim-plaintext-token-lit5108", + "x-litellm-api-key": "Bearer sk-virtual-key", + }, + ) + + with patch.object(pre_call_utils.verbose_proxy_logger, "debug") as mock_debug: + await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": [{"role": "user", "content": "hello"}]}, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"forward_llm_provider_auth_headers": True}, + version="test-version", + ) + + logged = " ".join(str(call) for call in mock_debug.call_args_list) + assert "apim-plaintext-token-lit5108" not in logged + + def _callback_credential_request_mock() -> MagicMock: request_mock = MagicMock(spec=Request) request_mock.url = MagicMock() @@ -5722,4 +5869,4 @@ async def test_key_level_callback_vars_survive_the_strip(): ) assert updated[TRUSTED_CALLBACK_VARS_FIELD] == {"dd_api_key": "key-dd-key", "dd_site": "us5.datadoghq.com"} - assert updated["dd_site"] == "us5.datadoghq.com" + assert updated["dd_site"] == "us5.datadoghq.com" \ No newline at end of file From ba1bde70e4b45d4f2bf5e9dd4b49858e7d9ac691 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 4 Aug 2026 00:20:25 +0000 Subject: [PATCH 057/124] feat(guardrails/rubrik): prompt moderation, response-text blocking, streaming buffer, failure logging (#35722) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(guardrails/rubrik): prompt moderation, response-text blocking, streaming buffer, failure logging (#34019) * feat(guardrails/rubrik): add prompt moderation, response-text blocking, streaming buffer, failure logging - Add `pre_call` prompt moderation via `/v1/before_prompt/openai/v1` webhook: structured messages are flattened and sent before the LLM is called; blocked prompts surface a `ModifyResponseException` with the refusal text. - Extend `post_call` response moderation to cover assistant text in addition to tool calls; text blocks (wholesale replacement) are distinguished from tool-block explanations (appended) via `startswith` diffing. - Add `streaming_end_of_stream_only = True` and `streaming_buffer_until_moderated = True` so streamed responses are withheld until end-of-stream moderation passes (requires litellm >= BerriAI/litellm#31389; older versions fall back to detect-only). - Add `_MalformedToolBlockingResponseError` for structurally invalid service responses; `_guarded` logs at CRITICAL so operators notice misconfiguration. - Add `max_queue_size = 10_000`, `_enforce_max_queue_size`, and drop-oldest backpressure so a webhook outage cannot grow the retry queue unboundedly. - Add `flush_queue` override that snapshots once for both send and drain, preventing duplicate delivery on concurrent flush calls. - Make `_log_batch_to_rubrik` re-raise on error so `flush_queue` preserves undelivered events for the next retry. - Add `async_post_call_failure_hook` to log blocked requests (`ModifyResponseException`) with a best-effort fallback payload for prompt blocks (where no `standard_logging_object` exists yet). - Add `_correlation_id` / `_apply_correlation_id` / `_prepend_system_prompt` helpers; `_prepare_log_payload` now applies them for all providers (not just Anthropic) so every log correlates by `litellm_call_id`. - Add `get_supported_event_hooks` classmethod advertising `[pre_call, post_call]`. - Use dedicated `httpx.AsyncClient` (`moderation_client`) for webhook calls with explicit pool limits, separate from the shared logging client. - Drop module-level `rubrik_handler` singleton (inappropriate for a library). - Update `initialize_guardrail` docstring to explain `pre_call` vs `post_call` mode. - Update tests: rename `tool_blocking_client` → `moderation_client`, `tool_blocking_endpoint` → `response_moderation_endpoint`, `_flush_task` → `_periodic_flush_task`; migrate `TestExtractBlockedTools` to `TestExtractResponseBlock` for the new combined text+tool block API; add tests for prompt moderation, text blocking, streaming flags, and failure payload construction. Co-Authored-By: Claude Sonnet 4.6 (1M context) * test(guardrails/rubrik): add tests to reach 100% coverage 50 new tests across 18 classes covering previously-untested paths: - Prompt moderation: passthrough, block, no-messages skip, message flattening (content-list → string), payload construction with tools/user/correlation_key/litellm_call_id fallback, refusal extraction - async_post_call_failure_hook: non-matching exception no-op, missing stash warning, valid stash → enqueue, AttributeError in payload build, flush exception handling - Block payload building: standard_logging_object present vs fallback path, missing start_time - async_log_success_event: _rubrik_blocked=True skip path - aclose: task cancel + moderation_client.aclose() - Edge cases: sampling rate clamp warning, unknown input_type passthrough, empty-inputs early return, model_call_details warning, _stash_block_context, duck-typed tool-call normalization, request_data["tools"] preference over optional_params, system-prompt exception handler, flush-at-batch-size, enqueue exception swallowing, queue empty/lock-None guards, non-dict JSON response TypeError Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): use get_async_httpx_client, ruff format - Replace bare httpx.AsyncClient with get_async_httpx_client (required by ensure_async_clients_test; avoids per-request client creation) - aclose() calls close() (AsyncHTTPHandler interface, not aclose()) - ruff format on rubrik.py and guardrail_hooks/rubrik/__init__.py - Update 3 tests for AsyncHTTPHandler type (isinstance check, close()) osv-scan and documentation CI failures are pre-existing on the base branch and unrelated to this PR. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): fix UP006 strict ruff violation get_supported_event_hooks return type used List[...] (UP006) instead of list[...]. Replace with the built-in generic and remove the now-unused List import from typing. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): fix 3 reportArgumentType basedpyright violations Use `# pyright: ignore[reportArgumentType]` (not `# type: ignore`) to suppress the three errors basedpyright reports in --outputjson mode: - convert_content_list_to_str call (dict vs AllMessageValues) - _apply_correlation_id call (StandardLoggingPayload vs dict[str, Any]) - _prepend_system_prompt call (same) Also tighten _apply_correlation_id and _prepend_system_prompt signatures from bare `dict` to `dict[str, Any]`. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): don't close shared HTTP client in aclose() moderation_client and async_httpx_client both come from LiteLLM's global HTTP-client cache (get_async_httpx_client keys on llm_provider + params). Two RubrikLogger instances with the same parameters share the same underlying AsyncHTTPHandler object. Calling close() in aclose() closed the shared connection pool for all instances, breaking any subsequent moderation request on other loggers. aclose() now only cancels the periodic flush task and lets LiteLLM manage the shared client lifecycle. Tests updated to assert close() is NOT called. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): use Counter for duplicate tool-call ID detection Set-based comparison lost ID multiplicity: two original tool calls with the same ID both appeared "allowed" even when the service returned only one (e.g. one allowed + one prohibited sharing an ID). Replace with Counter so returned_id_counts[id] >= required_id_counts[id] must hold for every ID. Matches the approach in the original _extract_blocked_tools. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): respect default_on=true when omitted from config LitellmParams.__init__ converts an omitted default_on to False before initialize_guardrail receives it, so litellm_params.default_on is always bool and never None. The is-None guard in RubrikLogger.__init__ therefore never fired on the proxy path, leaving prompt/response moderation inactive for any config that omitted default_on. Fix: read the raw guardrail dict (before LitellmParams coercion) to distinguish an explicit `default_on: false` from the absent-means-True default. When the key is absent from the raw config, default_on=True is used; when it is explicitly set (either True or False), that value wins. Co-Authored-By: Claude Sonnet 4.6 (1M context) * style: ruff format rubrik.py after Counter import addition Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): detect ID-less tool call removal; fix UP045 ID-less tool calls (tc.id is falsy) were excluded from required_id_counts, so the Counter comparison never caught their removal. Add a cardinality check (len(returned) < len(original)) that fires on any removal regardless of ID presence, combined with the Counter check for duplicate-ID attacks. Also fix 5 UP045 violations (Optional[X] → X | None) introduced by our new code against the daily-branch baseline. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): filter optional_params through ModelParamHelper in fallback payload _build_fallback_payload forwarded the raw optional_params dict as model_parameters. optional_params can contain extra_headers, api_key, and other upstream provider credentials that must not reach the Rubrik webhook. The normal standard_logging_object path already filters through ModelParamHelper.get_standard_logging_model_parameters(), which allowlists only safe LLM API parameters. Apply the same filter here. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): scope failure hook by guardrail_name; moderate text-completions Guard async_post_call_failure_hook by guardrail_name so multiple Rubrik instances don't cross-log: the failure hook is called for every registered callback; without the check the first instance pops the stash and the originating instance finds None and silently skips logging. Now each instance only handles blocks raised by itself. Also moderate /v1/completions prompts: _moderate_prompt returned early when structured_messages was absent. For text-completion requests litellm supplies inputs["texts"] with no structured_messages. Added a fallback that synthesises a user-message from texts so the before_prompt webhook can evaluate text-completion prompts. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(lint): add reason comments to pyright: ignore suppressions type-discipline budget requires each # pyright: ignore[...] to carry an explanatory comment. Add reasons to the three bare suppressions on lines 483, 651, 652. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): include tool-call arguments in prompt moderation _flatten_messages_for_moderation only sent the content field, silently dropping tool_calls[].function.arguments and function_call.arguments. An attacker could embed prohibited text in tool-call arguments inside assistant history turns and bypass prompt moderation entirely. Now collects all attacker-controlled text per message: text content via convert_content_list_to_str, plus all tool_calls[].function.arguments and the deprecated function_call.arguments, joined with newlines before being sent to the before_prompt webhook. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): tighten append detection to prevent prefix bypass startswith(sent_content) allowed any replacement whose text shares the original as a prefix (e.g. "Hello" → "Hello, blocked.") to be classified as a tool-block append rather than a text block, bypassing detection. Use startswith(f"{sent_content}\n\n") to require the exact two-newline separator the webhook uses between original text and appended tool-block explanations. Also add `returned_content != sent_content` to text_blocked so an unchanged passthrough is never classified as a block. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): default_on=False when omitted (follow existing pattern) Remove the custom raw-dict lookup that was defaulting default_on to True when omitted from the guardrail config. Follow the standard litellm convention: omitted resolves to False (users must explicitly opt in with default_on: true). - initialize_guardrail: pass litellm_params.default_on directly - RubrikLogger.__init__: is-None guard defaults to False not True - Test updated to assert the correct False default Co-Authored-By: Claude Sonnet 4.6 (1M context) --------- Co-authored-by: Claude Sonnet 4.6 (1M context) * chore(rubrik): keep the ported guardrail within staging lint budgets Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore: credit the original author of the rubrik guardrail work Co-authored-by: Joseph Barker <156112794+seph-barker@users.noreply.github.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore: keep this mirror PR's diff limited to the rubrik files Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Joseph Barker <156112794+seph-barker@users.noreply.github.com> Co-authored-by: Claude Sonnet 4.6 (1M context) Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/rubrik.py | 1040 +++++++++++++---- .../guardrail_hooks/rubrik/__init__.py | 12 + .../test_litellm/integrations/test_rubrik.py | 956 +++++++++++++-- 3 files changed, 1691 insertions(+), 317 deletions(-) diff --git a/litellm/integrations/rubrik.py b/litellm/integrations/rubrik.py index 2e49da45ce9..4bcbe8bae37 100644 --- a/litellm/integrations/rubrik.py +++ b/litellm/integrations/rubrik.py @@ -1,12 +1,14 @@ -"""Rubrik LiteLLM Plugin for tool blocking and batch logging.""" +"""Rubrik LiteLLM Plugin for prompt/response moderation and batch logging.""" import asyncio import os import random import time -import urllib.parse import uuid from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Literal, Optional import httpx @@ -18,6 +20,10 @@ from litellm.integrations.custom_guardrail import ( ModifyResponseException, ) from litellm.litellm_core_utils.core_helpers import safe_deep_copy +from litellm.litellm_core_utils.model_param_helper import ModelParamHelper +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + convert_content_list_to_str, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -35,15 +41,16 @@ if TYPE_CHECKING: Logging as LiteLLMLoggingObj, ) -_ENDPOINT_ANTHROPIC_MESSAGES = "/v1/messages" -_WEBHOOK_PATH_TOOL_BLOCKING = "/v1/after_completion/openai/v1" +_WEBHOOK_PATH_RESPONSE_MODERATION = "/v1/after_completion/openai/v1" +_WEBHOOK_PATH_PROMPT_MODERATION = "/v1/before_prompt/openai/v1" _WEBHOOK_PATH_LOGGING_BATCH = "/v1/litellm/batch" _MAX_QUEUE_SIZE = 10_000 _DROP_WARNING_INTERVAL_SECONDS = 60.0 +_EMPTY_MAPPING: Mapping[str, Any] = MappingProxyType({}) class _MalformedToolBlockingResponseError(Exception): - """Raised when the tool blocking service returns a structurally invalid + """Raised when the response moderation service returns a structurally invalid response (e.g. empty ``choices``). Distinct from transient network/HTTP errors so callers can surface a @@ -52,11 +59,15 @@ class _MalformedToolBlockingResponseError(Exception): """ -class RubrikLogger(CustomGuardrail, CustomBatchLogger): - @classmethod - def get_supported_event_hooks(cls) -> list[GuardrailEventHooks]: - return [GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call] +@dataclass +class BlockedResponseResult: + """Returned by _extract_response_block when the response was blocked + (response text replaced, or at least one tool call removed).""" + explanation: str + + +class RubrikLogger(CustomGuardrail, CustomBatchLogger): def __init__( self, api_key: str | None = None, @@ -67,21 +78,82 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): kwargs.setdefault("guardrail_name", "rubrik") # `initialize_guardrail` always passes these kwargs explicitly, with # value `None` when the user omits `mode` / `default_on` from the - # guardrail config. Coerce None (omitted) to the desired default - # while preserving any explicit value the caller did set -- - # in particular `default_on=False` if the user wants the guardrail - # off by default. + # guardrail config. Follow the standard litellm convention: omitted + # resolves to False (off by default, user must opt in explicitly). kwargs["event_hook"] = kwargs.get("event_hook") or GuardrailEventHooks.post_call if kwargs.get("default_on") is None: - kwargs["default_on"] = True - kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks())) + kwargs["default_on"] = False super().__init__( flush_lock=self.flush_lock, + supported_event_hooks=list(self.get_supported_event_hooks()), **kwargs, ) verbose_logger.debug("initializing rubrik logger") + # Defining ``apply_guardrail`` routes streaming responses through + # litellm's ``unified_guardrail.async_post_call_streaming_iterator_hook``. + # By default that hook samples intermediate chunks + # (``streaming_sampling_rate``, default 5) and also moderates at + # end-of-stream, so a streamed response costs ~ceil(N/5)+1 Rubrik + # webhook round-trips. litellm reads this attribute via + # ``getattr(guardrail, "streaming_end_of_stream_only", False)``; when + # True it yields chunks unprocessed and only moderates the fully + # assembled response once at end of stream. + self.streaming_end_of_stream_only = True + + # ``streaming_end_of_stream_only`` is detect-only: it releases every + # chunk to the client *before* moderating, so a block can only append a + # trailing message -- the original content has already been delivered. + # ``streaming_buffer_until_moderated`` (litellm >= BerriAI/litellm#31389) + # withholds all chunks until end-of-stream moderation passes, then + # releases the original response (clean) or only the block message + # (blocked). On older litellm this attribute is ignored and we fall + # back to the detect-only behavior above. + self.streaming_buffer_until_moderated = True + + self._parse_sampling_rate() + + self.key = api_key or os.getenv("RUBRIK_API_KEY") + if not self.key: + verbose_logger.warning("Rubrik: No API key configured. Requests will be unauthenticated.") + + self._parse_batch_size() + + # Cap the in-memory retry queue so a Rubrik webhook outage cannot let + # authenticated traffic accumulate prompt/response payloads until the + # proxy runs out of memory. Once the cap is reached, oldest events are + # dropped to make room for fresh ones (drop-oldest backpressure). + self.max_queue_size = _MAX_QUEUE_SIZE + self._dropped_since_warning = 0 + self._last_drop_warning_time = 0.0 + + _webhook_url = api_base or os.getenv("RUBRIK_WEBHOOK_URL") + if not _webhook_url: + raise ValueError("Rubrik webhook URL not configured. Set RUBRIK_WEBHOOK_URL or pass api_base.") + + _webhook_url = _webhook_url.rstrip("/").removesuffix("/v1") + self._setup_clients(_webhook_url) + + self._headers: Mapping[str, str] = MappingProxyType( + {"Content-Type": "application/json", "Authorization": f"Bearer {self.key}"} + if self.key + else {"Content-Type": "application/json"} + ) + + self._periodic_flush_task: asyncio.Task[Any] | None = self._start_periodic_flush_task() + + @classmethod + def get_supported_event_hooks(cls) -> list[GuardrailEventHooks]: + """Return the guardrail event hooks this integration supports. + + Prompt moderation (``pre_call``) evaluates the user's message before + the LLM is called. Response moderation (``post_call``) evaluates the + assistant's reply and tool calls after the LLM returns. + """ + return [GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call] + + def _parse_sampling_rate(self) -> None: self.sampling_rate = 1.0 rbrk_sampling_rate = os.getenv("RUBRIK_SAMPLING_RATE") if rbrk_sampling_rate is not None: @@ -93,80 +165,54 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): except ValueError: verbose_logger.warning(f"Invalid RUBRIK_SAMPLING_RATE: {rbrk_sampling_rate!r}, using 1.0") - self.key = api_key or os.getenv("RUBRIK_API_KEY") - if not self.key: - verbose_logger.warning("Rubrik: No API key configured. Requests will be unauthenticated.") + def _parse_batch_size(self) -> None: _batch_size = os.getenv("RUBRIK_BATCH_SIZE") - if _batch_size: try: - self.batch_size = int(_batch_size) + parsed_size = int(_batch_size) + if parsed_size <= 0: + verbose_logger.warning(f"RUBRIK_BATCH_SIZE={_batch_size!r} must be > 0, using default") + else: + self.batch_size = parsed_size except ValueError: verbose_logger.warning(f"Invalid RUBRIK_BATCH_SIZE: {_batch_size!r}, using default") - # Cap the in-memory retry queue so a Rubrik webhook outage cannot let - # authenticated traffic accumulate prompt/response payloads until the - # proxy runs out of memory. Once the cap is reached, oldest events are - # dropped to make room for fresh ones (drop-oldest backpressure). - self.max_queue_size = _MAX_QUEUE_SIZE - self._dropped_since_warning = 0 - self._last_drop_warning_time = 0.0 - - _webhook_url = api_base or os.getenv("RUBRIK_WEBHOOK_URL") - - if _webhook_url is None: - raise ValueError("Rubrik webhook URL not configured. Set RUBRIK_WEBHOOK_URL or pass api_base.") - - _webhook_url = _webhook_url.rstrip("/").removesuffix("/v1") - self.tool_blocking_endpoint = f"{_webhook_url}{_WEBHOOK_PATH_TOOL_BLOCKING}" - self.logging_endpoint = f"{_webhook_url}{_WEBHOOK_PATH_LOGGING_BATCH}" + def _setup_clients(self, webhook_url: str) -> None: + self.response_moderation_endpoint = f"{webhook_url}{_WEBHOOK_PATH_RESPONSE_MODERATION}" + self.prompt_moderation_endpoint = f"{webhook_url}{_WEBHOOK_PATH_PROMPT_MODERATION}" + self.logging_endpoint = f"{webhook_url}{_WEBHOOK_PATH_LOGGING_BATCH}" self.async_httpx_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback) - self.tool_blocking_client = get_async_httpx_client( + self.moderation_client = get_async_httpx_client( llm_provider=httpxSpecialProvider.LoggingCallback, params={"timeout": httpx.Timeout(5.0, connect=2.0)}, ) - self._headers: dict[str, str] = {"Content-Type": "application/json"} - if self.key: - self._headers["Authorization"] = f"Bearer {self.key}" - - # Periodic flush is started lazily on the first log event so that - # low-traffic deployments still get their batches drained even when the - # logger is instantiated outside a running event loop (sync init). - self._flush_task: asyncio.Task[Any] | None = self._start_periodic_flush_task() - def _start_periodic_flush_task(self) -> asyncio.Task[Any] | None: """Start the periodic flush task only when an event loop is already running.""" try: loop = asyncio.get_running_loop() except RuntimeError: - verbose_logger.debug( - "Rubrik logger init: no running event loop, periodic flush will start on first log event." - ) return None return loop.create_task(self.periodic_flush()) def _ensure_periodic_flush_task(self) -> None: - # Synchronous helper: in asyncio's cooperative model there is no await - # between the check and assignment, so two callers cannot race here. - if self._flush_task is None or self._flush_task.done(): - self._flush_task = self._start_periodic_flush_task() + if self._periodic_flush_task is None or self._periodic_flush_task.done(): + self._periodic_flush_task = self._start_periodic_flush_task() async def aclose(self): - """Close the dedicated HTTP clients used by this logger.""" - # Cancel the periodic flush task before closing the HTTP clients so - # the loop doesn't wake up and try to POST via a closed client. - if self._flush_task is not None and not self._flush_task.done(): - self._flush_task.cancel() - try: - await self._flush_task - except (asyncio.CancelledError, Exception): - pass - self._flush_task = None - await self.tool_blocking_client.close() - await self.async_httpx_client.close() + """Cancel the periodic flush task. + + ``moderation_client`` and ``async_httpx_client`` are shared objects + from LiteLLM's global HTTP-client cache (``get_async_httpx_client`` + uses the same cache key for all instances with equal parameters). + Closing them here would close the shared connection pool for every + other logger instance; let LiteLLM manage their lifecycle instead. + """ + task = getattr(self, "_periodic_flush_task", None) + if task is not None: + task.cancel() # -- Guardrail hook -------------------------------------------------------- @@ -177,67 +223,104 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): input_type: Literal["request", "response"], logging_obj: Optional["LiteLLMLoggingObj"] = None, ) -> GenericGuardrailAPIInputs: - """Validate tool calls against the blocking service (fail-open).""" - if input_type != "response": - return inputs + """Moderate prompts (request) and responses (response); fail-open. - tool_calls = inputs.get("tool_calls") - if not tool_calls: - return inputs + - ``request``: evaluate the prompt via the before_prompt webhook and + block disallowed prompts before the model is called. + - ``response``: evaluate the assistant's response text and tool calls + via the after_completion webhook and block on a policy violation. + litellm's guardrail-translation layer normalizes Anthropic and OpenAI + requests/responses into ``inputs`` before this runs, so a single code + path covers both wire formats. The configured guardrail ``mode`` + selects which surface(s) run. + """ + if input_type == "request": + return await self._guarded( + self._moderate_prompt(inputs, request_data, logging_obj), + inputs, + "Prompt moderation", + ) + if input_type == "response": + return await self._guarded( + self._moderate_response(inputs, request_data, logging_obj), + inputs, + "Response moderation", + ) + return inputs + + @staticmethod + async def _guarded( + coro: Any, + inputs: GenericGuardrailAPIInputs, + label: str, + ) -> GenericGuardrailAPIInputs: + """Await a moderation coroutine fail-open: re-raise an intentional + block, log at critical for malformed service responses, and swallow + any other error returning ``inputs`` unchanged.""" try: - return await self._check_tool_calls(inputs, tool_calls, request_data, logging_obj) + return await coro except ModifyResponseException: raise except _MalformedToolBlockingResponseError as e: - # Distinct from transient errors: the service responded but the - # payload was structurally invalid, which usually indicates a - # misconfigured webhook or a breaking change in its response - # format. Log loudly so operators notice their tool-blocking - # policy is not actually being enforced. + # The service responded but the payload was structurally invalid, + # which usually indicates a misconfigured webhook or a breaking + # change in its response format. Log loudly so operators notice + # their moderation policy is not actually being enforced. verbose_logger.critical( - "Tool blocking service returned a malformed response: %s. " - "Tool calls are NOT being checked -- verify the webhook " - "configuration. Returning original response unchanged.", + "Response moderation service returned a malformed response: %s. " + "Requests are NOT being checked -- verify the webhook " + "configuration. Returning original inputs unchanged.", e, exc_info=True, ) return inputs except Exception as e: verbose_logger.error( - f"Tool blocking hook failed: {e}. Returning original response unchanged.", + f"{label} hook failed: {e}. Returning original inputs unchanged.", exc_info=True, ) return inputs - async def _check_tool_calls( + async def _moderate_response( self, inputs: GenericGuardrailAPIInputs, - tool_calls: Any, request_data: dict, logging_obj: Optional["LiteLLMLoggingObj"], ) -> GenericGuardrailAPIInputs: - """Send tool calls to blocking service, raise if any are blocked.""" - message_tool_calls = self._normalize_tool_calls(tool_calls) + """Send response text + tool calls to the after_completion webhook and + raise if either the response text or any tool call is blocked.""" + tool_calls = inputs.get("tool_calls") + texts = inputs.get("texts") + if not tool_calls and not texts: + return inputs - call_details = getattr(logging_obj, "model_call_details", {}) if logging_obj else {} - response = request_data.get("response") - request_id = getattr(response, "id", None) if response else None + message_tool_calls = self._normalize_tool_calls(tool_calls or ()) + sent_content = self._join_texts(texts) + + call_details = getattr(logging_obj, "model_call_details", _EMPTY_MAPPING) if logging_obj else _EMPTY_MAPPING if logging_obj and not call_details: verbose_logger.warning( "Rubrik: logging_obj present but model_call_details is empty -- request context will be missing" ) - response_data = self._build_tool_call_payload(message_tool_calls, request_id) - req_data = self._extract_request_data(call_details) + # The moderation payload's ``id`` becomes the tool-blocking log's + # correlation key (the S3 filename), so it must match the failure + # (response) log written for the same blocked request. Both use + # ``litellm_call_id`` -- see ``_correlation_id``. + request_id = self._correlation_id(call_details, request_data) - service_response = await self._post_to_tool_blocking_service(response_data, req_data) - blocked_explanation = self._extract_blocked_tools(service_response, message_tool_calls) + response_data = self._build_response_moderation_payload(message_tool_calls, sent_content, request_id) + req_data = self._extract_request_data(call_details, request_data) - if blocked_explanation is not None: + service_response = await self._post_to_response_moderation_endpoint(response_data, req_data) + blocked = self._extract_response_block(service_response, message_tool_calls, sent_content) + + if blocked: model = self._resolve_model(request_data, call_details) + self._stash_block_context(logging_obj, request_data) raise ModifyResponseException( - message=blocked_explanation, + message=blocked.explanation, model=model, request_data=request_data, guardrail_name=self.guardrail_name, @@ -245,43 +328,125 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): return inputs - @staticmethod - def _normalize_tool_calls(tool_calls: Any) -> list[ChatCompletionMessageToolCall]: - """Convert tool_calls from inputs to ChatCompletionMessageToolCall objects.""" - result = [] - for tc in tool_calls: - if isinstance(tc, ChatCompletionMessageToolCall): - result.append(tc) - elif isinstance(tc, dict): - func = tc.get("function", {}) - result.append( - ChatCompletionMessageToolCall( - id=tc.get("id", ""), - type=tc.get("type", "function"), - function=Function( - name=func.get("name", ""), - arguments=func.get("arguments", ""), - ), - ) - ) - elif hasattr(tc, "id") and hasattr(tc, "function"): - result.append( - ChatCompletionMessageToolCall( - id=tc.id or "", - type=getattr(tc, "type", None) or "function", - function=tc.function, - ) - ) - else: - raise TypeError(f"Cannot normalize tool_call of type {type(tc).__name__}") - return result + async def _moderate_prompt( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + logging_obj: Optional["LiteLLMLoggingObj"], + ) -> GenericGuardrailAPIInputs: + """Send the (normalized) prompt to the before_prompt webhook and raise + if the prompt is blocked.""" + messages = inputs.get("structured_messages") + if not messages: + # For non-chat request types (e.g. /v1/completions), litellm + # supplies the prompt as ``texts`` with no structured_messages. + # Synthesise a user-message so the webhook can evaluate the prompt. + texts = inputs.get("texts") + if texts: + joined = "\n".join(t for t in texts if t) + if joined: + messages = [{"role": "user", "content": joined}] + if not messages: + return inputs + + payload = self._build_prompt_moderation_payload(inputs, request_data) + service_response = await self._post_to_prompt_moderation_endpoint(payload) + refusal = self._extract_prompt_refusal(service_response) + if refusal is None: + return inputs + + model = inputs.get("model") or request_data.get("model") or "unknown" + self._stash_block_context(logging_obj, request_data) + raise ModifyResponseException( + message=refusal, + model=model, + request_data=request_data, + guardrail_name=self.guardrail_name, + ) @staticmethod - def _build_tool_call_payload( - tool_calls: list[ChatCompletionMessageToolCall], + def _stash_block_context( + logging_obj: Optional["LiteLLMLoggingObj"], + request_data: dict, + ) -> None: + """Stash signals so the deferred success-event skips this request and + ``async_post_call_failure_hook`` can build the failure payload. + + - Sets a flag on ``logging_obj.model_call_details`` so the deferred + success-event handler short-circuits. + - Stashes a reference to ``logging_obj`` on ``request_data`` under a + custom key. ``ProxyLogging.post_call_failure_hook`` pops only + ``litellm_logging_obj`` before iterating callbacks, so this key + survives. + + When ``logging_obj`` is ``None`` the success-event has no way to + observe the block (the flag has nowhere to live), so we log an error + instead of silently dropping the signal. + """ + if logging_obj is None: + verbose_logger.error( + "Rubrik: moderation block fired with logging_obj=None for " + f"litellm_call_id={request_data.get('litellm_call_id')}; " + "cannot suppress success event or attach failure payload." + ) + request_data["_rubrik_logging_obj"] = None + return + logging_obj.model_call_details["_rubrik_blocked"] = True + request_data["_rubrik_logging_obj"] = logging_obj + + @staticmethod + def _normalize_tool_calls(tool_calls: Any) -> tuple[ChatCompletionMessageToolCall, ...]: + """Convert tool_calls from inputs to ChatCompletionMessageToolCall objects.""" + return tuple(RubrikLogger._normalize_tool_call(tc) for tc in tool_calls) + + @staticmethod + def _normalize_tool_call(tc: Any) -> ChatCompletionMessageToolCall: + if isinstance(tc, ChatCompletionMessageToolCall): + return tc + if isinstance(tc, dict): + func = tc.get("function") or _EMPTY_MAPPING + return ChatCompletionMessageToolCall( + id=tc.get("id", ""), + type=tc.get("type", "function"), + function=Function( + name=func.get("name", ""), + arguments=func.get("arguments", ""), + ), + ) + if hasattr(tc, "id") and hasattr(tc, "function"): + return ChatCompletionMessageToolCall( + id=tc.id or "", + type=getattr(tc, "type", None) or "function", + function=tc.function, + ) + raise TypeError(f"Cannot normalize tool_call of type {type(tc).__name__}: {tc!r}") + + @staticmethod + def _join_texts(texts: Any) -> str: + """Join response text segments into the single content string the + webhook evaluates. Empty when there is no assistant text.""" + if not texts: + return "" + return "\n".join(t for t in texts if t) + + @staticmethod + def _build_response_moderation_payload( + tool_calls: Sequence[ChatCompletionMessageToolCall], + content: str, request_id: str | None, - ) -> dict[str, Any]: - """Build a full OpenAI ChatCompletion-format dict for the blocking service.""" + ) -> Mapping[str, Any]: + """Build an OpenAI ChatCompletion-format dict (assistant text + tool + calls) for the after_completion webhook. + + ``content`` is sent so the webhook can moderate the response text; + ``None`` when the assistant produced no text (tool-call-only response). + """ + message: dict[str, Any] = { + "role": "assistant", + "content": content or None, + } + if tool_calls: + message["tool_calls"] = tuple(tc.model_dump(exclude_none=True) for tc in tool_calls) return { "id": request_id or f"chatcmpl-{uuid.uuid4()}", "object": "chat.completion", @@ -290,42 +455,133 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): "choices": [ { "index": 0, - "message": { - "role": "assistant", - "content": None, - "tool_calls": [tc.model_dump(exclude_none=True) for tc in tool_calls], - }, - "finish_reason": "tool_calls", + "message": message, + "finish_reason": "tool_calls" if tool_calls else "stop", } ], } @staticmethod - def _extract_request_data(call_details: dict[str, Any]) -> dict[str, Any]: - """Extract original request data from model_call_details.""" - if not call_details: - return {} - litellm_params = call_details.get("litellm_params", {}) or {} + def _flatten_messages_for_moderation(messages: Any) -> tuple[Mapping[str, Any], ...]: + """Collapse each message's content to a plain string for the webhook. + + litellm normalizes Anthropic ``/v1/messages`` requests to OpenAI shape, + but a turn sent as content-parts (``[{"type": "text", ...}]``) stays a + list. The before_prompt webhook reads ``content`` as a string and drops + non-string content, so we flatten text parts here (images skipped, per + ``convert_content_list_to_str``) -- otherwise block-content prompts + would pass through unmoderated. Builds a new list; never mutates the + shared ``structured_messages``. + """ + return tuple( + { + "role": message.get("role"), + "content": "\n".join(p for p in RubrikLogger._moderation_text_parts(message) if p), + } + for message in messages or () + if isinstance(message, dict) + ) + + @staticmethod + def _moderation_text_parts(message: Mapping[str, Any]) -> tuple[str, ...]: + """Every attacker-controlled text segment of a message: its content plus + the arguments of any tool call or deprecated function call.""" + fc = message.get("function_call") + return ( + # Base text content (flattens Anthropic content-part arrays) + convert_content_list_to_str(message), # pyright: ignore[reportArgumentType] # dict[str,Any] is AllMessageValues at runtime + *( + str((tc.get("function") or _EMPTY_MAPPING).get("arguments") or "") + for tc in message.get("tool_calls") or () + if isinstance(tc, dict) + ), + str((fc.get("arguments") if isinstance(fc, dict) else None) or ""), + ) + + @staticmethod + def _build_prompt_moderation_payload( + inputs: GenericGuardrailAPIInputs, + request_data: Mapping[str, Any], + ) -> Mapping[str, Any]: + """Build the bare OpenAI request the before_prompt webhook consumes. + + Unlike the after_completion envelope, this endpoint takes a raw OpenAI + chat-completions request. ``structured_messages`` is litellm's + OpenAI-normalized view of the prompt, so this works for Anthropic + ``/v1/messages`` requests too. Optional fields are sent only when + present so the payload stays clean. + """ + payload: dict[str, Any] = { + "model": inputs.get("model") or request_data.get("model") or "", + "messages": RubrikLogger._flatten_messages_for_moderation(inputs.get("structured_messages")), + } + tools = inputs.get("tools") + if tools is not None: + payload["tools"] = tools + user = request_data.get("user") + if user: + payload["user"] = user + # Fall back to litellm_call_id, the stable cross-provider join key the + # response/tool path uses (see _correlation_id). LiteLLM does not + # populate request_data["correlation_key"]; it carries litellm_call_id. + # The before_prompt webhook skips the *_prompt_moderation.json S3 write + # when correlation_key is empty, so without this the block fires but no + # log is ever written. An explicit correlation_key still wins. + correlation_key = request_data.get("correlation_key") or request_data.get("litellm_call_id") + if correlation_key: + payload["correlation_key"] = correlation_key + return payload + + @staticmethod + def _extract_request_data( + call_details: Mapping[str, Any], + request_data: Mapping[str, Any] | None, + ) -> Mapping[str, Any]: + """Extract original request data from model_call_details for the + response moderation service envelope. + + Includes the agent's declared ``tools`` (OpenAI-format) when available + so the webhook's hallucination evaluator can compare returned tool calls + against the declared tool list. + """ + if not call_details and not request_data: + return _EMPTY_MAPPING + call_details = call_details or _EMPTY_MAPPING + request_data = request_data or _EMPTY_MAPPING + optional_params = call_details.get("optional_params") or _EMPTY_MAPPING + + # Use ``in`` rather than truthy ``or`` so an explicit empty list + # (caller declared the agent has NO tools) is forwarded as-is. + # The response moderation service uses that signal to flag tool-call + # hallucinations -- ``or`` would mask it by falling through to + # optional_params. + if "tools" in request_data: + tools = request_data["tools"] + else: + tools = optional_params.get("tools") + + # The response moderation service consumes only messages/model/tools. + # Don't forward proxy_server_request -- in litellm >=1.83 its ``body`` + # snapshot carries a UserAPIKeyAuth instance that breaks json.dumps, + # silently fail-opening the guardrail. return { "messages": call_details.get("messages"), "model": call_details.get("model"), - "proxy_server_request": RubrikLogger._sanitize_proxy_server_request( - litellm_params.get("proxy_server_request") - ), + "tools": tools, } @staticmethod def _sanitize_proxy_server_request(proxy_server_request: Any) -> Any: """Allowlist only routing fields (``url``, ``method``) when forwarding - ``proxy_server_request`` to the external Rubrik webhook, dropping - inbound ``headers`` (Authorization, Cookie, x-api-key, ...) and the raw + ``proxy_server_request`` to an external webhook, dropping inbound + ``headers`` (Authorization, Cookie, x-api-key, ...) and the raw request ``body`` so proxy credentials are not exfiltrated.""" if not isinstance(proxy_server_request, dict): return proxy_server_request return {key: proxy_server_request[key] for key in ("url", "method") if key in proxy_server_request} @staticmethod - def _resolve_model(request_data: dict[str, Any], call_details: dict[str, Any]) -> str: + def _resolve_model(request_data: Mapping[str, Any], call_details: Mapping[str, Any]) -> str: """Get the model name for the ModifyResponseException.""" response = request_data.get("response") if response and hasattr(response, "model"): @@ -334,8 +590,70 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): # -- Logging hooks --------------------------------------------------------- - async def _prepare_log_payload(self, kwargs: dict, event_type: str) -> StandardLoggingPayload | None: - """Shared logic for success and failure logging.""" + @staticmethod + def _correlation_id(call_details: Mapping[str, Any], request_data: Mapping[str, Any] | None = None) -> str | None: + """The id that joins a blocked request's two S3 logs by filename: the + moderation (``_blocking``) log and the failure (response) log. + + Always ``litellm_call_id``. It is assigned at request start and is + present identically in both the guardrail path (``model_call_details`` + / ``request_data``) and the failure-hook path. Unlike ``response.id`` + or ``standard_logging_object["id"]`` it is immune to the race where a + block fires before the response/logging object is populated, so the + two logs correlate for every provider (OpenAI and Anthropic alike). + """ + return call_details.get("litellm_call_id") or (request_data or _EMPTY_MAPPING).get("litellm_call_id") + + @classmethod + def _apply_correlation_id(cls, payload: dict[str, Any], source: Mapping[str, Any]) -> None: + """Pin ``payload["id"]`` to ``litellm_call_id`` in place so this log + shares its S3 filename id with the moderation (``_blocking``) and + failure logs for the same request -- for every provider. + + ``standard_logging_object["id"]`` is the provider response id + (``response_obj.get("id", litellm_call_id)``), a ``chatcmpl-*`` value + for OpenAI, which would not correlate. ``litellm_call_id`` is assigned + at request start and is identical across all log paths. Falls back to + the existing id when ``litellm_call_id`` is somehow absent rather than + writing a null filename key. + + ``source`` may be ``model_call_details`` directly or a ``kwargs`` dict + that aliases it -- same shape either way. + """ + correlated = cls._correlation_id(source) + if correlated: + payload["id"] = correlated + + @staticmethod + def _prepend_system_prompt(payload: dict[str, Any], source: Mapping[str, Any]) -> None: + """Prepend ``source["system"]`` onto ``payload["messages"]``. + + Builds a NEW messages list rather than mutating ``payload["messages"]`` + in place. The fallback branch of ``_prepare_block_failure_payload`` + aliases ``call_details["messages"]`` directly, so an in-place + ``list.insert(0, ...)`` would mutate the shared source dict. + + No-op if no system prompt is present. Tolerates list/dict/str + message shapes; on unexpected shape, leaves payload alone. + """ + system_prompt = source.get("system") + if not system_prompt: + return + try: + system_scaffold = {"role": "system", "content": system_prompt} + messages = payload.get("messages") + if isinstance(messages, list): + payload["messages"] = (system_scaffold, *messages) + elif isinstance(messages, (dict, str)): + payload["messages"] = (system_scaffold, messages) + except Exception as e: + verbose_logger.warning( + f"Rubrik: failed to prepend system prompt: {e}", + exc_info=True, + ) + + async def _prepare_log_payload(self, kwargs: Mapping[str, Any], event_type: str) -> StandardLoggingPayload | None: + """Shared logic for success logging (sampled).""" if random.random() > self.sampling_rate: verbose_logger.debug(f"Skipping Rubrik {event_type} logging (sampling_rate={self.sampling_rate})") return None @@ -343,59 +661,17 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): # Deep-copy so mutations don't affect other callbacks sharing this object standard_logging_payload: StandardLoggingPayload = safe_deep_copy(kwargs["standard_logging_object"]) - # For Anthropic /v1/messages requests, LiteLLM creates a separate - # ModelResponse (with a generated chatcmpl-* id) for logging, which - # differs from the original Anthropic msg-* id on the response dict. - # Normalize to litellm_call_id so that the logging and tool-blocking - # endpoints see the same request identifier. - litellm_params = kwargs.get("litellm_params", {}) or {} - proxy_request = litellm_params.get("proxy_server_request", {}) or {} - url_path = urllib.parse.urlparse(proxy_request.get("url", "")).path - if url_path.endswith(_ENDPOINT_ANTHROPIC_MESSAGES): - _litellm_call_id = kwargs.get("litellm_call_id") - if _litellm_call_id: - standard_logging_payload["id"] = _litellm_call_id # type: ignore[literal-required] - - if "system" in kwargs: - system_prompt_msg_list = kwargs["system"] - try: - if system_prompt_msg_list: - system_scaffold = { - "role": "system", - "content": system_prompt_msg_list, - } - if isinstance(standard_logging_payload["messages"], list): - standard_logging_payload["messages"].insert(0, system_scaffold) - elif isinstance(standard_logging_payload["messages"], (dict, str)): - standard_logging_payload["messages"] = [ - system_scaffold, - standard_logging_payload["messages"], - ] - except Exception as e: - verbose_logger.warning( - f"Rubrik: failed to prepend system prompt: {e}", - exc_info=True, - ) + self._apply_correlation_id(standard_logging_payload, kwargs) # pyright: ignore[reportArgumentType] # StandardLoggingPayload is dict[str,Any] at runtime + self._prepend_system_prompt(standard_logging_payload, kwargs) # pyright: ignore[reportArgumentType] # StandardLoggingPayload is dict[str,Any] at runtime return standard_logging_payload - async def _enqueue_log_event(self, kwargs: dict, event_type: str): - try: - self._ensure_periodic_flush_task() - payload = await self._prepare_log_payload(kwargs, event_type) - if payload is None: - return - - self.log_queue.append(payload) - self._enforce_max_queue_size() - - if len(self.log_queue) >= self.batch_size: - await self.flush_queue() - except Exception as e: - verbose_logger.error( - f"Rubrik {event_type} logging hook failed: {e}. Skipping logging for this event.", - exc_info=True, - ) + async def _append_and_maybe_flush(self, payload) -> None: + self._ensure_periodic_flush_task() + self.log_queue.append(payload) + self._enforce_max_queue_size() + if len(self.log_queue) >= self.batch_size: + await self.flush_queue() def _enforce_max_queue_size(self) -> None: overflow = len(self.log_queue) - self.max_queue_size @@ -415,18 +691,213 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): self._dropped_since_warning = 0 self._last_drop_warning_time = now + async def _enqueue_log_event(self, kwargs: Mapping[str, Any], event_type: str): + try: + payload = await self._prepare_log_payload(kwargs, event_type) + if payload is None: + return + await self._append_and_maybe_flush(payload) + except Exception as e: + verbose_logger.error( + f"Rubrik {event_type} logging hook failed: {e}. Skipping logging for this event.", + exc_info=True, + ) + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + # Blocked requests are logged via async_post_call_failure_hook; + # skip here to avoid double-logging the pre-block response. + if kwargs.get("_rubrik_blocked"): + verbose_logger.debug( + f"Rubrik: skipping success event for blocked request litellm_call_id={kwargs.get('litellm_call_id')}" + ) + return await self._enqueue_log_event(kwargs, "success") async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + # Log regular LLM failures (timeouts, upstream errors, etc.) to Rubrik. + # NOTE: ``ModifyResponseException`` blocks are NOT routed here; they + # bypass ``Logging.async_failure_handler`` entirely and reach + # ``async_post_call_failure_hook`` instead. So there is no risk of + # double-logging a block through this path. await self._enqueue_log_event(kwargs, "failure") + async def async_post_call_failure_hook( + self, + request_data: dict, + original_exception: Exception, + user_api_key_dict: Any, + traceback_str: str | None = None, + ) -> None: + """Log blocked requests signalled via ``ModifyResponseException`` + (prompt blocks, response/tool blocks, streaming blocks). + + Carries the stashed ``_rubrik_logging_obj``. For every other + exception we no-op; LiteLLM's standard failure plumbing handles those. + """ + if not isinstance(original_exception, ModifyResponseException): + return + + # Guard by guardrail_name so that when multiple Rubrik instances are + # registered, only the instance that raised the block handles it. + # The failure hook is called for every registered callback; without + # this check the first instance pops the stash and the originating + # instance finds None and silently skips logging. + if getattr(original_exception, "guardrail_name", None) != self.guardrail_name: + return + + logging_obj = request_data.pop("_rubrik_logging_obj", None) + if logging_obj is None: + # Legitimate when a non-Rubrik guardrail raised the block; + # problematic if Rubrik did and the stash was lost (e.g. + # ``_stash_block_context`` ran with ``logging_obj=None``). Either + # way we cannot build the payload. + verbose_logger.warning( + "Rubrik: block exception without stashed logging_obj. " + f"litellm_call_id={request_data.get('litellm_call_id')}, " + f"model={request_data.get('model')}, " + f"user_id={getattr(user_api_key_dict, 'user_id', None)}, " + f"raising_guardrail=" + f"{getattr(original_exception, 'guardrail_name', None)}" + ) + return + + call_id: str | None = None + await self._build_and_enqueue_block_event(logging_obj, original_exception, call_id) + + async def _build_and_enqueue_block_event( + self, + logging_obj: "LiteLLMLoggingObj", + exception: "ModifyResponseException", + call_id: str | None, + ) -> None: + try: + call_details = logging_obj.model_call_details + # Do NOT pop "_rubrik_blocked" here. The deferred success-handler + # task may still be iterating callbacks, and popping mid-iteration + # (between two awaited callback invocations) would cause this + # plugin's success-event callback to read the flag as absent and + # log the pre-block response -- the exact bug this hook exists to + # prevent. The flag dies with model_call_details when the request + # completes; there's nothing to clean up. + call_id = call_details.get("litellm_call_id") + payload = self._prepare_block_failure_payload(logging_obj, exception) + except (AttributeError, KeyError, TypeError) as e: + verbose_logger.error( + f"Rubrik: failed to build blocked-tool payload for " + f"litellm_call_id={call_id}: {e}. Event will NOT be logged.", + exc_info=True, + ) + return + + try: + await self._append_and_maybe_flush(payload) + except Exception as e: + verbose_logger.error( + f"Rubrik: failed to enqueue blocked-tool event for litellm_call_id={call_id}: {e}.", + exc_info=True, + ) + + def _prepare_block_failure_payload( + self, + logging_obj: "LiteLLMLoggingObj", + exception: "ModifyResponseException", + ) -> StandardLoggingPayload: + """Build a failure-style payload using the exception text as response. + + Blocked-tool events are security-relevant and **bypass sampling**: + every block is logged. + + The deferred success-handler runs as a separately-scheduled task and + races with this hook, so ``standard_logging_object`` on + ``model_call_details`` may not yet be populated. If present we reuse + it; otherwise we fall back to a best-effort payload built from the + fields available at block time. + + For prompt blocks the LLM is never called, so ``standard_logging_object`` + is never populated. The fallback therefore must carry enough fields to + pass the log processor's ``LogEntry`` schema (``BaseLogEntry`` requires + ``metadata``, ``model_id``, ``model_group``, ``model_parameters``, + ``startTime``, ``endTime``, and ``completionStartTime``). Without a + parseable payload the log processor discards the entry with a parse + error and no session is created, so prompt-moderation violations are + silently dropped even though the ``_prompt_moderation.json`` forensic + log is written correctly. + + Field sourcing for the fallback path: + - ``model`` / ``model_group``: ``call_details["model"]`` -- this is the + model-group name (e.g. "gpt-4o") set by the proxy before the guardrail + fires. The router writes ``metadata["model_group"]`` only inside + ``acompletion()``, which hasn't run yet for a prompt block. + - ``model_id``: not available before the LLM returns hidden_params; + defaults to empty string. + - ``user_api_key_hash``: ``call_details["metadata"]["user_api_key"]`` -- + the hashed token written by ``add_user_information_to_request_data`` + before ``pre_call_hook`` fires. + - time fields: ``call_details["start_time"]`` reused for all three; + end/completion times are meaningless for a prompt block. + """ + call_details = logging_obj.model_call_details + exception_text = f"{type(exception).__name__}: {exception.message}" + + base = call_details.get("standard_logging_object") + if base is not None: + payload: dict = safe_deep_copy(base) + else: + verbose_logger.debug( + "Rubrik: standard_logging_object not yet on model_call_details " + f"for litellm_call_id={call_details.get('litellm_call_id')}; " + "using best-effort fallback payload." + ) + payload = self._build_fallback_payload(call_details) + + payload["response"] = exception_text + + # Pin the correlation key to litellm_call_id so this failure log shares + # its S3 filename id with the moderation (``_blocking``) log for the + # same request. The copied ``standard_logging_object["id"]`` is + # ``response_obj.get("id", litellm_call_id)`` -- a provider ``chatcmpl-*`` + # value for OpenAI -- which would not correlate; overwrite it. + payload["id"] = self._correlation_id(call_details) or f"chatcmpl-{uuid.uuid4()}" + self._prepend_system_prompt(payload, call_details) + + return payload # type: ignore[return-value] + + @staticmethod + def _build_fallback_payload(call_details: Mapping[str, Any]) -> dict[str, Any]: + _metadata: Mapping[str, Any] = call_details.get("metadata") or _EMPTY_MAPPING + # Convert datetime to a Unix float so json.dumps can serialize it. + # httpx's json= parameter uses stdlib json.dumps with no custom encoder. + _raw_start = call_details.get("start_time") + _start = _raw_start.timestamp() if _raw_start is not None else None + return { + "id": call_details.get("litellm_call_id"), + "model": call_details.get("model") or "", + # model_group is set by the router inside acompletion(), which + # hasn't run for a prompt block; use the model name instead. + "model_group": call_details.get("model") or "", + # model_id comes from response.hidden_params -- unavailable here. + "model_id": "", + "model_parameters": ModelParamHelper.get_standard_logging_model_parameters( + call_details.get("optional_params") or _EMPTY_MAPPING # pyright: ignore[reportArgumentType] # helper only reads the mapping + ), + "startTime": _start, + "endTime": _start, + "completionStartTime": _start, + "messages": call_details.get("messages") or (), + "metadata": { + # "user_api_key" is the hashed token written by + # add_user_information_to_request_data before guardrails fire. + "user_api_key_hash": _metadata.get("user_api_key_hash") or _metadata.get("user_api_key") or "", + }, + "status": "failure", + } + # -- Batch logging --------------------------------------------------------- async def _log_batch_to_rubrik(self, data): - # NOTE: this method intentionally re-raises on failure so the parent - # CustomBatchLogger.flush_queue keeps the unsent events in the queue - # for the next flush attempt instead of silently dropping them. + # NOTE: this method intentionally re-raises on failure so flush_queue + # can preserve the unsent events for the next flush attempt instead of + # silently dropping them. try: response = await self.async_httpx_client.post( url=self.logging_endpoint, @@ -452,10 +923,8 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): if not self.log_queue: return - log_queue_snapshot = list(self.log_queue) - verbose_logger.debug("Rubrik: Flushing batch of %s events", len(log_queue_snapshot)) await self._log_batch_to_rubrik( - data=log_queue_snapshot, + data=self.log_queue, ) async def flush_queue(self): @@ -463,8 +932,8 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): Overrides the base implementation so the same snapshot drives both the HTTP send and the queue truncation. This avoids the subtle - coupling where the base class captures `len(self.log_queue)` - separately from the snapshot taken inside `async_send_batch`, + coupling where the base class captures ``len(self.log_queue)`` + separately from the snapshot taken inside ``async_send_batch``, which could otherwise drift in a future refactor and cause duplicate deliveries to Rubrik. """ @@ -485,70 +954,141 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): del self.log_queue[: len(snapshot)] self.last_flush_time = time.time() - # -- Tool blocking service ------------------------------------------------- + # -- Webhook services ------------------------------------------------------ - async def _post_to_tool_blocking_service( + async def _post_json(self, endpoint: str, payload: Mapping[str, Any], service_name: str) -> Mapping[str, Any]: + """POST ``payload`` to a Rubrik webhook and return its dict response. + + Raises: + Exception: If the service is unavailable or returns an error. + TypeError: If the response JSON is not a dict. + """ + verbose_logger.debug(f"Sending request to {service_name}: {endpoint}") + http_response = await self.moderation_client.post( + endpoint, + json=payload, + headers=self._headers, + ) + http_response.raise_for_status() + result = http_response.json() + if not isinstance(result, dict): + raise TypeError( + f"{service_name} returned non-dict JSON " + f"({type(result).__name__}); expected OpenAI chat completion " + "shape or empty object." + ) + return result + + async def _post_to_response_moderation_endpoint( self, - response_data: dict[str, Any], - request_data: dict[str, Any], - ) -> dict[str, Any]: - """Post a payload to the tool blocking service and return the response. + response_data: Mapping[str, Any], + request_data: Mapping[str, Any], + ) -> Mapping[str, Any]: + """Post the ``{request, response}`` envelope to the after_completion + webhook and return its (possibly rewritten) response. Args: response_data: The OpenAI-formatted response payload to send. request_data: Original LLM request data to include alongside the response for additional context. Empty dict if unavailable. - - Raises: - Exception: If the service is unavailable or returns an error. """ - envelope = { - "request": request_data, - "response": response_data, - } - verbose_logger.debug(f"Sending request to tool blocking service: {self.tool_blocking_endpoint}") - http_response = await self.tool_blocking_client.post( - self.tool_blocking_endpoint, - json=envelope, - headers=self._headers, + envelope = {"request": request_data, "response": response_data} + return await self._post_json( + self.response_moderation_endpoint, + envelope, + "Response moderation service", ) - http_response.raise_for_status() - result: dict[str, Any] = http_response.json() - return result + + async def _post_to_prompt_moderation_endpoint(self, payload: Mapping[str, Any]) -> Mapping[str, Any]: + """Post a bare OpenAI request to the before_prompt webhook. + + Returns ``{}`` (passthrough) or a synthetic chat.completion (block). + """ + return await self._post_json(self.prompt_moderation_endpoint, payload, "Prompt moderation service") @staticmethod - def _extract_blocked_tools( - service_response: dict[str, Any], - all_tool_calls: list[ChatCompletionMessageToolCall], - ) -> str | None: - """Return the blocking explanation if any tool calls were blocked. + def _extract_prompt_refusal(service_response: Mapping[str, Any]) -> str | None: + """Return the refusal text when the prompt was blocked, else None. - Compares the service response (which contains only allowed tools) against - the full set of tool calls. Returns ``None`` if all tools are allowed, or - the explanation string (prefixed with newlines) otherwise. + The before_prompt webhook returns ``{}`` (passthrough) or a synthetic + chat.completion whose ``choices[0].message.content`` is the refusal + explanation. + """ + choices = service_response.get("choices") + if not choices: + return None + message = choices[0].get("message") or _EMPTY_MAPPING + content = message.get("content") + return content or "Request blocked by policy." + + @staticmethod + def _extract_response_block( + service_response: Mapping[str, Any], + all_tool_calls: Sequence[ChatCompletionMessageToolCall], + sent_content: str, + ) -> BlockedResponseResult | None: + """Detect whether the webhook moderated the response text or tool calls. + + The after_completion webhook rewrites the response in place with no + explicit "blocked" flag, so we infer a block by diffing what we sent + against what came back: + + - Tool block: a tool call we sent is absent from the returned (allowed) + set. + - Text block: the returned content was REPLACED wholesale (a text + violation), as opposed to having a tool-block explanation APPENDED to + the original content. We tell them apart with ``startswith``, which + mirrors the webhook's own append-vs-replace behavior. + + Returns None when nothing was moderated. A text block supersedes a tool + block (mirroring the webhook, which drops tool calls on a text block). Expects service_response in OpenAI chat completion format: {"choices": [{"message": {"tool_calls": [...], "content": "..."}}]} """ - choices = service_response.get("choices", []) + choices = service_response.get("choices") or () if not choices: - raise _MalformedToolBlockingResponseError("Tool blocking service returned empty response") + raise _MalformedToolBlockingResponseError("Response moderation service returned empty response") - message = choices[0].get("message", {}) - returned_tool_calls = message.get("tool_calls") or [] - blocking_explanation = message.get("content", "") + message = choices[0].get("message") or _EMPTY_MAPPING + returned_tool_calls = message.get("tool_calls") or () + returned_content = message.get("content") or "" - allowed_id_counts: Counter = Counter( - tc["id"] for tc in returned_tool_calls if isinstance(tc, dict) and tc.get("id") - ) - required_id_counts: Counter = Counter(tc.id for tc in all_tool_calls if tc.id) - - all_allowed = len(returned_tool_calls) >= len(all_tool_calls) and all( - allowed_id_counts.get(tc_id, 0) >= count for tc_id, count in required_id_counts.items() + # Use Counter so duplicate IDs are handled correctly: if the model + # emits two calls with the same ID (one allowed, one prohibited) and + # the service returns only the allowed one, a set-based check would + # miss the block. Counter preserves multiplicity. + returned_id_counts: Counter[str] = Counter(tc["id"] for tc in returned_tool_calls if tc.get("id")) + required_id_counts: Counter[str] = Counter(tc.id for tc in all_tool_calls if tc.id) + # Cardinality check catches ID-less tool calls (not counted in + # required_id_counts because tc.id is falsy); Counter check catches + # duplicate-ID attacks where one occurrence is silently removed. + tools_blocked = len(returned_tool_calls) < len(all_tool_calls) or not all( + returned_id_counts.get(tc_id, 0) >= count for tc_id, count in required_id_counts.items() ) - if all_allowed: - return None + # The webhook either replaces content wholesale (text block) or appends + # a tool-block explanation to the original text. ``appended`` tells the + # two apart, and is reused below to recover just the explanation. A text + # block requires there to have been assistant text to block. + # Use the documented ``\n\n`` separator to distinguish a tool-block + # append from a text replacement that shares the original as a prefix. + # Without the separator, a replacement like "Hello, blocked." where the + # original was "Hello" would be classified as an append (not a text + # block) and silently pass through to the client. + appended = bool(sent_content) and returned_content.startswith(f"{sent_content}\n\n") + text_blocked = bool(sent_content) and returned_content != sent_content and not appended - explanation = blocking_explanation or "Tool call blocked by policy." - return f"\n\n{explanation}" + if text_blocked: + return BlockedResponseResult(explanation=returned_content or "Response blocked by policy.") + + if tools_blocked: + if appended: + # Recover just the appended explanation: drop the original text + # and the leading separator the webhook inserted before it. + explanation = returned_content[len(sent_content) :].lstrip("\n") + else: + explanation = returned_content + return BlockedResponseResult(explanation=explanation or "Tool call blocked by policy.") + + return None diff --git a/litellm/proxy/guardrails/guardrail_hooks/rubrik/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/rubrik/__init__.py index 4ad29bbeae8..2f2228ae312 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/rubrik/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/rubrik/__init__.py @@ -10,6 +10,18 @@ if TYPE_CHECKING: def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail") -> RubrikLogger: + """Create and register a RubrikLogger instance. + + The ``mode`` field in the guardrail config controls which surfaces are + moderated: + - ``pre_call`` (or a mode that includes it): prompt moderation via the + ``/v1/before_prompt/openai/v1`` webhook. + - ``post_call`` (the default when ``mode`` is omitted): response and tool + call moderation via the ``/v1/after_completion/openai/v1`` webhook. + + Both hooks are active when ``mode`` covers both ``pre_call`` and + ``post_call``. + """ import litellm rubrik_callback = RubrikLogger( diff --git a/tests/test_litellm/integrations/test_rubrik.py b/tests/test_litellm/integrations/test_rubrik.py index 922d2fe8a15..7f589dc15bf 100644 --- a/tests/test_litellm/integrations/test_rubrik.py +++ b/tests/test_litellm/integrations/test_rubrik.py @@ -1,8 +1,8 @@ """ Tests for the Rubrik LiteLLM plugin. -Covers initialization, apply_guardrail tool blocking (all allowed, all blocked, -partial blocking, fail-open), batch logging, and Anthropic format handling. +Covers initialization, apply_guardrail (prompt moderation + response/tool +blocking), batch logging, and Anthropic format handling. """ import os @@ -13,8 +13,10 @@ import httpx import pytest from litellm.integrations.custom_guardrail import ModifyResponseException -from litellm.integrations.rubrik import RubrikLogger -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.integrations.rubrik import ( + RubrikLogger, + _MalformedToolBlockingResponseError, +) from tests.test_litellm.integrations.rubrik_test_helpers import ( make_inputs_with_tools, @@ -50,19 +52,19 @@ class TestInitialization: with patch("asyncio.create_task", Mock()): handler = RubrikLogger() assert ( - handler.tool_blocking_endpoint + handler.response_moderation_endpoint == "http://localhost:8080/v1/after_completion/openai/v1" ) assert handler.logging_endpoint == "http://localhost:8080/v1/litellm/batch" assert handler.key == "test-api-key" - assert isinstance(handler.tool_blocking_client, AsyncHTTPHandler) + assert handler.moderation_client is not None def test_init_with_constructor_params(self): with patch("asyncio.create_task", Mock()): handler = RubrikLogger(api_key="ctor-key", api_base="http://ctor-host:9090") assert handler.key == "ctor-key" assert ( - handler.tool_blocking_endpoint + handler.response_moderation_endpoint == "http://ctor-host:9090/v1/after_completion/openai/v1" ) @@ -82,7 +84,7 @@ class TestInitialization: with patch.dict(os.environ, {"RUBRIK_WEBHOOK_URL": "http://localhost:8080/"}): with patch("asyncio.create_task", Mock()): assert ( - RubrikLogger().tool_blocking_endpoint + RubrikLogger().response_moderation_endpoint == "http://localhost:8080/v1/after_completion/openai/v1" ) @@ -90,13 +92,13 @@ class TestInitialization: with patch("asyncio.create_task", Mock()): with patch.dict(os.environ, {"RUBRIK_WEBHOOK_URL": "http://host/v1"}): assert ( - RubrikLogger().tool_blocking_endpoint + RubrikLogger().response_moderation_endpoint == "http://host/v1/after_completion/openai/v1" ) with patch.dict(os.environ, {"RUBRIK_WEBHOOK_URL": "http://host/v11"}): assert ( - RubrikLogger().tool_blocking_endpoint + RubrikLogger().response_moderation_endpoint == "http://host/v11/v1/after_completion/openai/v1" ) @@ -155,10 +157,10 @@ class TestInitialization: # Do NOT patch asyncio.create_task — the real call should be # guarded and fall back gracefully when there is no event loop. handler = RubrikLogger() - assert handler.tool_blocking_endpoint.startswith("http://localhost:8080") + assert handler.response_moderation_endpoint.startswith("http://localhost:8080") # Without a running loop at init, the periodic flush task should be # deferred so batches still get drained once a log event arrives. - assert handler._flush_task is None + assert handler._periodic_flush_task is None @pytest.mark.asyncio async def test_periodic_flush_task_started_lazily_on_first_log(self, mock_env): @@ -170,7 +172,7 @@ class TestInitialization: side_effect=RuntimeError("no running loop"), ): handler = RubrikLogger() - assert handler._flush_task is None + assert handler._periodic_flush_task is None kwargs = { "standard_logging_object": { @@ -183,8 +185,8 @@ class TestInitialization: with patch.object(handler, "_log_batch_to_rubrik", AsyncMock()): await handler.async_log_success_event(kwargs, None, None, None) - assert handler._flush_task is not None - handler._flush_task.cancel() + assert handler._periodic_flush_task is not None + handler._periodic_flush_task.cancel() def test_event_hook_defaults_to_post_call_when_none_passed(self, mock_env): """`initialize_guardrail` always passes ``event_hook=litellm_params.mode`` @@ -204,14 +206,13 @@ class TestInitialization: handler = RubrikLogger(event_hook=GuardrailEventHooks.pre_call) assert handler.event_hook == GuardrailEventHooks.pre_call - def test_default_on_defaults_to_true_when_none_passed(self, mock_env): - """`initialize_guardrail` always passes ``default_on=litellm_params.default_on`` - (which is ``None`` when the user omits ``default_on``). The logger must - coerce a None ``default_on`` to True, otherwise ``should_run_guardrail`` - (which checks ``self.default_on is True``) silently skips the guardrail.""" + def test_default_on_defaults_to_false_when_none_passed(self, mock_env): + """Follows the standard litellm pattern: omitted ``default_on`` resolves + to ``False`` (off by default). Users must explicitly set + ``default_on: true`` to enable the guardrail for all requests.""" with patch("asyncio.create_task", Mock()): handler = RubrikLogger(default_on=None) - assert handler.default_on is True + assert handler.default_on is False def test_explicit_default_on_false_preserved(self, mock_env): """A user explicitly setting ``default_on: false`` in their guardrail @@ -421,7 +422,7 @@ class TestBatchLogging: ) assert len(handler.log_queue) == 1 msgs = handler.log_queue[0]["messages"] - assert isinstance(msgs, list) + assert isinstance(msgs, tuple) assert msgs[0]["role"] == "system" assert msgs[1] == {"role": "user", "content": "hi"} @@ -444,7 +445,10 @@ class TestBatchLogging: ) assert handler.log_queue[0]["id"] == "litellm-call-123" - async def test_non_anthropic_id_unchanged(self, handler): + async def test_litellm_call_id_always_used_as_correlation_key(self, handler): + """The merged plugin always uses litellm_call_id as the log ID for all + providers (not just Anthropic) so that logs correlate with the + moderation (_blocking) and failure logs for the same request.""" kwargs = { "standard_logging_object": { "id": "chatcmpl-original", @@ -461,7 +465,7 @@ class TestBatchLogging: await handler.async_log_success_event( kwargs=kwargs, response_obj=None, start_time=None, end_time=None ) - assert handler.log_queue[0]["id"] == "chatcmpl-original" + assert handler.log_queue[0]["id"] == "litellm-call-123" async def test_payload_deep_copied_not_mutated(self, handler): """Verify the shared standard_logging_object is not mutated.""" @@ -536,7 +540,7 @@ class TestApplyGuardrail: tc2 = make_tool_call_dict("call_2", "get_time") inputs = make_inputs_with_tools([tc1, tc2]) - handler.tool_blocking_client = _echo_service() + handler.moderation_client = _echo_service() result = await handler.apply_guardrail( inputs=inputs, request_data={}, input_type="response" @@ -548,7 +552,7 @@ class TestApplyGuardrail: tc2 = make_tool_call_dict("call_2", "drop_database") inputs = make_inputs_with_tools([tc1, tc2]) - handler.tool_blocking_client = _mock_service_response( + handler.moderation_client = _mock_service_response( { "choices": [ { @@ -594,7 +598,7 @@ class TestApplyGuardrail: mock_client = AsyncMock() mock_client.post = mock_post - handler.tool_blocking_client = mock_client + handler.moderation_client = mock_client with pytest.raises(ModifyResponseException): await handler.apply_guardrail( @@ -607,7 +611,7 @@ class TestApplyGuardrail: mock_client = AsyncMock() mock_client.post = AsyncMock(side_effect=httpx.TimeoutException("Timeout")) - handler.tool_blocking_client = mock_client + handler.moderation_client = mock_client result = await handler.apply_guardrail( inputs=inputs, request_data={}, input_type="response" @@ -618,7 +622,7 @@ class TestApplyGuardrail: tc1 = make_tool_call_dict("call_1", "test_tool") inputs = make_inputs_with_tools([tc1]) - handler.tool_blocking_client = _mock_service_response({"choices": []}) + handler.moderation_client = _mock_service_response({"choices": []}) result = await handler.apply_guardrail( inputs=inputs, request_data={}, input_type="response" @@ -641,7 +645,7 @@ class TestApplyGuardrail: mock_client = AsyncMock() mock_client.post = mock_post - handler.tool_blocking_client = mock_client + handler.moderation_client = mock_client await handler.apply_guardrail( inputs=inputs, request_data={}, input_type="response" @@ -675,7 +679,7 @@ class TestApplyGuardrail: mock_client = AsyncMock() mock_client.post = mock_post - handler.tool_blocking_client = mock_client + handler.moderation_client = mock_client logging_obj = Mock() logging_obj.model_call_details = { @@ -697,7 +701,10 @@ class TestApplyGuardrail: assert req["model"] == "gpt-4" assert req["messages"] == [{"role": "user", "content": "hi"}] - async def test_proxy_server_request_headers_stripped(self, handler): + async def test_proxy_server_request_not_forwarded(self, handler): + """proxy_server_request is intentionally NOT included in the request + envelope: in litellm >=1.83 its ``body`` carries a UserAPIKeyAuth + instance that breaks json.dumps, silently fail-opening the guardrail.""" tc = make_tool_call_dict("call_1", "test_tool") inputs = make_inputs_with_tools([tc]) @@ -712,7 +719,7 @@ class TestApplyGuardrail: mock_client = AsyncMock() mock_client.post = mock_post - handler.tool_blocking_client = mock_client + handler.moderation_client = mock_client logging_obj = Mock() logging_obj.model_call_details = { @@ -739,8 +746,8 @@ class TestApplyGuardrail: logging_obj=logging_obj, ) - forwarded = captured_payload["request"]["proxy_server_request"] - assert forwarded == {"url": "/chat/completions", "method": "POST"} + # proxy_server_request is deliberately excluded from the forwarded envelope + assert "proxy_server_request" not in captured_payload["request"] # -- Anthropic format ---------------------------------------------------------- @@ -760,7 +767,7 @@ class TestApplyGuardrailAnthropicFormat: ) inputs = make_inputs_with_tools([tc], texts=["I'll check the weather."]) - handler.tool_blocking_client = _echo_service() + handler.moderation_client = _echo_service() result = await handler.apply_guardrail( inputs=inputs, request_data={}, input_type="response" @@ -771,7 +778,7 @@ class TestApplyGuardrailAnthropicFormat: tc = make_tool_call_dict("toolu_123", "dangerous_tool", '{"arg": "value"}') inputs = make_inputs_with_tools([tc]) - handler.tool_blocking_client = _mock_service_response( + handler.moderation_client = _mock_service_response( { "choices": [ { @@ -790,21 +797,21 @@ class TestApplyGuardrailAnthropicFormat: inputs=inputs, request_data={}, input_type="response" ) - async def test_text_only_response_no_blocking(self, handler): + async def test_text_only_response_sent_to_moderation(self, handler): + """Text-only responses (no tool calls) are sent to the response + moderation service to check the assistant's text content.""" from litellm.types.utils import GenericGuardrailAPIInputs inputs = GenericGuardrailAPIInputs(texts=["Hello! I'm Claude."]) - mock_client = AsyncMock() - mock_client.post = AsyncMock() - handler.tool_blocking_client = mock_client + # Service allows the response (returns the content unchanged) + handler.moderation_client = _echo_service() result = await handler.apply_guardrail( inputs=inputs, request_data={}, input_type="response" ) assert result is inputs - mock_client.post.assert_not_called() async def test_service_failure_preserves_tools(self, handler): tc = make_tool_call_dict("toolu_123", "get_weather", '{"location": "SF"}') @@ -812,7 +819,7 @@ class TestApplyGuardrailAnthropicFormat: mock_client = AsyncMock() mock_client.post = AsyncMock(side_effect=httpx.TimeoutException("Timeout")) - handler.tool_blocking_client = mock_client + handler.moderation_client = mock_client result = await handler.apply_guardrail( inputs=inputs, request_data={}, input_type="response" @@ -850,10 +857,13 @@ class TestNormalizeToolCalls: RubrikLogger._normalize_tool_calls(["not_a_tool_call"]) -# -- Extract blocked tools ----------------------------------------------------- +# -- Extract response block ---------------------------------------------------- -class TestExtractBlockedTools: +class TestExtractResponseBlock: + """Tests for _extract_response_block, which replaces the upstream + _extract_blocked_tools and handles both text blocks and tool blocks.""" + def test_all_allowed_returns_none(self): from litellm.types.utils import ChatCompletionMessageToolCall, Function @@ -870,7 +880,7 @@ class TestExtractBlockedTools: } ] } - result = RubrikLogger._extract_blocked_tools(service_resp, [tc]) + result = RubrikLogger._extract_response_block(service_resp, [tc], "") assert result is None def test_some_blocked_returns_explanation(self): @@ -896,13 +906,13 @@ class TestExtractBlockedTools: } ] } - result = RubrikLogger._extract_blocked_tools(service_resp, [tc1, tc2]) + result = RubrikLogger._extract_response_block(service_resp, [tc1, tc2], "") assert result is not None - assert "blocked fn2" in result + assert "blocked fn2" in result.explanation def test_empty_choices_raises(self): - with pytest.raises(Exception, match="empty response"): - RubrikLogger._extract_blocked_tools({"choices": []}, []) + with pytest.raises(_MalformedToolBlockingResponseError): + RubrikLogger._extract_response_block({"choices": []}, [], "") def test_null_tool_calls_treated_as_all_blocked(self): from litellm.types.utils import ChatCompletionMessageToolCall, Function @@ -920,36 +930,55 @@ class TestExtractBlockedTools: } ] } - result = RubrikLogger._extract_blocked_tools(service_resp, [tc]) + result = RubrikLogger._extract_response_block(service_resp, [tc], "") assert result is not None - assert "blocked everything" in result + assert "blocked everything" in result.explanation - def test_duplicate_ids_block_when_only_one_returned(self): + def test_text_block_detected(self): + """When the service replaces the response text wholesale, it's a text block.""" from litellm.types.utils import ChatCompletionMessageToolCall, Function - tc1 = ChatCompletionMessageToolCall( - id="call_dup", - type="function", - function=Function(name="fn", arguments="{}"), - ) - tc2 = ChatCompletionMessageToolCall( - id="call_dup", - type="function", - function=Function(name="fn", arguments="{}"), - ) service_resp = { "choices": [ { "message": { - "tool_calls": [{"id": "call_dup"}], - "content": "blocked duplicate", + "tool_calls": [], + "content": "This content violates policy.", } } ] } - result = RubrikLogger._extract_blocked_tools(service_resp, [tc1, tc2]) + result = RubrikLogger._extract_response_block( + service_resp, [], "Original assistant text." + ) assert result is not None - assert "blocked duplicate" in result + assert "violates policy" in result.explanation + + def test_tool_block_with_appended_explanation(self): + """When the service appends an explanation to the original text, only the + appended part is returned as the explanation.""" + from litellm.types.utils import ChatCompletionMessageToolCall, Function + + tc = ChatCompletionMessageToolCall( + id="call_1", type="function", function=Function(name="fn", arguments="{}") + ) + original_text = "Here is my response." + appended_explanation = "Tool call was blocked." + service_resp = { + "choices": [ + { + "message": { + "tool_calls": [], + "content": original_text + "\n\n" + appended_explanation, + } + } + ] + } + result = RubrikLogger._extract_response_block( + service_resp, [tc], original_text + ) + assert result is not None + assert appended_explanation in result.explanation # -- Sanitize proxy server request ------------------------------------------- @@ -1010,3 +1039,796 @@ class TestResolveModel: {"response": response}, {"model": "fallback"} ) assert result == "unknown" + + +# -- Additional Initialization edge cases ------------------------------------ + + +class TestInitializationEdgeCases: + def test_batch_size_zero_uses_default(self): + """RUBRIK_BATCH_SIZE=0 must warn and fall back to the default.""" + with patch("asyncio.create_task", Mock()): + with patch.dict( + os.environ, + {"RUBRIK_WEBHOOK_URL": "http://host", "RUBRIK_BATCH_SIZE": "0"}, + ): + h = RubrikLogger() + # Should use default, not 0 + assert h.batch_size > 0 + + def test_batch_size_negative_uses_default(self): + """RUBRIK_BATCH_SIZE=-1 must warn and fall back to the default.""" + with patch("asyncio.create_task", Mock()): + with patch.dict( + os.environ, + {"RUBRIK_WEBHOOK_URL": "http://host", "RUBRIK_BATCH_SIZE": "-5"}, + ): + h = RubrikLogger() + assert h.batch_size > 0 + + +# -- aclose() ----------------------------------------------------------------- + + +@pytest.mark.asyncio +class TestAclose: + async def test_aclose_cancels_task_does_not_close_shared_client(self, mock_env): + """aclose() cancels the periodic flush task but does NOT close the shared + moderation_client — closing a shared cached client would break other + RubrikLogger instances that share the same connection pool.""" + with patch("asyncio.create_task", Mock()): + handler = RubrikLogger() + + mock_task = Mock() + mock_task.cancel = Mock() + handler._periodic_flush_task = mock_task + + handler.moderation_client = AsyncMock() + handler.moderation_client.close = AsyncMock() + + await handler.aclose() + + mock_task.cancel.assert_called_once() + handler.moderation_client.close.assert_not_awaited() + + async def test_aclose_with_none_task_does_not_close_client(self, mock_env): + """aclose() with no flush task still does not close the shared client.""" + with patch("asyncio.create_task", Mock()): + handler = RubrikLogger() + + handler._periodic_flush_task = None + handler.moderation_client = AsyncMock() + handler.moderation_client.close = AsyncMock() + + await handler.aclose() + + handler.moderation_client.close.assert_not_awaited() + + +# -- apply_guardrail edge cases ----------------------------------------------- + + +@pytest.mark.asyncio +class TestApplyGuardrailEdgeCases: + async def test_unknown_input_type_returns_inputs_unchanged(self, handler): + """When input_type is not 'request' or 'response', inputs are returned as-is.""" + inputs = make_inputs_with_tools([make_tool_call_dict("call_1", "tool")]) + result = await handler.apply_guardrail( + inputs=inputs, request_data={}, input_type="unknown" + ) + assert result is inputs + + async def test_response_with_no_texts_and_no_tool_calls_returns_inputs(self, handler): + """_moderate_response early-returns when both texts and tool_calls are empty.""" + from litellm.types.utils import GenericGuardrailAPIInputs + + inputs = GenericGuardrailAPIInputs() + result = await handler.apply_guardrail( + inputs=inputs, request_data={}, input_type="response" + ) + assert result is inputs + + async def test_moderate_response_empty_call_details_emits_warning(self, handler): + """When logging_obj is present but model_call_details is empty, a warning is + logged and moderation proceeds (fail-open on HTTP error).""" + tc = make_tool_call_dict("call_1", "test_tool") + inputs = make_inputs_with_tools([tc]) + + logging_obj = Mock() + logging_obj.model_call_details = {} + + handler.moderation_client = _echo_service() + + result = await handler.apply_guardrail( + inputs=inputs, + request_data={}, + input_type="response", + logging_obj=logging_obj, + ) + assert result is inputs + + +# -- Prompt moderation -------------------------------------------------------- + + +@pytest.mark.asyncio +class TestPromptModeration: + async def test_prompt_moderation_passthrough(self, handler): + """Webhook returns {} (empty dict) → inputs returned unchanged.""" + inputs = {"structured_messages": [{"role": "user", "content": "Hello"}]} + + handler.moderation_client = _mock_service_response({}) + + result = await handler.apply_guardrail( + inputs=inputs, request_data={}, input_type="request" + ) + assert result is inputs + + async def test_prompt_moderation_blocked_raises(self, handler): + """Webhook returns synthetic chat.completion → raises ModifyResponseException.""" + inputs = { + "structured_messages": [{"role": "user", "content": "Harmful prompt"}], + "model": "gpt-4", + } + + handler.moderation_client = _mock_service_response( + { + "choices": [ + { + "message": { + "role": "assistant", + "content": "This request violates our policy.", + } + } + ] + } + ) + + with pytest.raises(ModifyResponseException) as exc_info: + await handler.apply_guardrail( + inputs=inputs, request_data={"model": "gpt-4"}, input_type="request" + ) + assert "violates our policy" in exc_info.value.message + + async def test_prompt_moderation_no_messages_skips_moderation(self, handler): + """When structured_messages is absent/empty, moderation is skipped.""" + inputs = {"model": "gpt-4"} + + result = await handler.apply_guardrail( + inputs=inputs, request_data={}, input_type="request" + ) + assert result is inputs + + async def test_prompt_moderation_stashes_logging_obj_on_block(self, handler): + """On a prompt block, _stash_block_context must set the blocked flag.""" + inputs = { + "structured_messages": [{"role": "user", "content": "bad prompt"}], + } + + handler.moderation_client = _mock_service_response( + { + "choices": [ + {"message": {"role": "assistant", "content": "Blocked."}} + ] + } + ) + + logging_obj = Mock() + logging_obj.model_call_details = {} + request_data: dict = {} + + with pytest.raises(ModifyResponseException): + await handler.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="request", + logging_obj=logging_obj, + ) + + assert logging_obj.model_call_details.get("_rubrik_blocked") is True + assert request_data.get("_rubrik_logging_obj") is logging_obj + + +# -- _stash_block_context ----------------------------------------------------- + + +class TestStashBlockContext: + def test_with_non_none_logging_obj_sets_flag_and_stashes(self): + """Sets _rubrik_blocked flag and stores logging_obj on request_data.""" + logging_obj = Mock() + logging_obj.model_call_details = {} + request_data: dict = {} + + RubrikLogger._stash_block_context(logging_obj, request_data) + + assert logging_obj.model_call_details["_rubrik_blocked"] is True + assert request_data["_rubrik_logging_obj"] is logging_obj + + def test_with_none_logging_obj_stores_none_on_request_data(self): + """When logging_obj is None, stores None on request_data (logged as error).""" + request_data: dict = {"litellm_call_id": "test-id"} + + RubrikLogger._stash_block_context(None, request_data) + + assert request_data["_rubrik_logging_obj"] is None + + +# -- _normalize_tool_calls duck-typed ----------------------------------------- + + +class TestNormalizeToolCallsDuckTyped: + def test_duck_typed_object_with_id_and_function_attrs(self): + """Objects that have .id and .function attrs but are not + ChatCompletionMessageToolCall are handled by the third branch.""" + from litellm.types.utils import Function + + tc = Mock() + tc.id = "call_duck" + tc.type = "function" + tc.function = Function(name="duck_tool", arguments='{"x": 1}') + # Make isinstance(..., ChatCompletionMessageToolCall) return False + # by using a plain Mock (not a ChatCompletionMessageToolCall subclass) + + result = RubrikLogger._normalize_tool_calls([tc]) + assert len(result) == 1 + assert result[0].id == "call_duck" + assert result[0].function.name == "duck_tool" + + def test_duck_typed_without_type_defaults_to_function(self): + """getattr(tc, "type", None) falls back to "function" when absent.""" + from litellm.types.utils import Function + + tc = Mock(spec=["id", "function"]) # no .type attr + tc.id = "call_no_type" + tc.function = Function(name="fn", arguments="{}") + + result = RubrikLogger._normalize_tool_calls([tc]) + assert result[0].type == "function" + + +# -- _flatten_messages_for_moderation ----------------------------------------- + + +class TestFlattenMessagesForModeration: + def test_plain_string_content_preserved(self): + messages = [{"role": "user", "content": "Hello world"}] + result = RubrikLogger._flatten_messages_for_moderation(messages) + assert len(result) == 1 + assert result[0]["role"] == "user" + assert result[0]["content"] == "Hello world" + + def test_content_list_flattened_to_string(self): + """Content as a list of parts (e.g. Anthropic multi-part) is flattened.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello from parts"}, + ], + } + ] + result = RubrikLogger._flatten_messages_for_moderation(messages) + assert len(result) == 1 + assert result[0]["role"] == "user" + assert "Hello from parts" in result[0]["content"] + + def test_non_dict_messages_skipped(self): + """Non-dict entries in the messages list are silently skipped.""" + messages = [ + "raw string message", + {"role": "user", "content": "valid"}, + ] + result = RubrikLogger._flatten_messages_for_moderation(messages) + assert len(result) == 1 + assert result[0]["content"] == "valid" + + def test_none_messages_returns_empty(self): + result = RubrikLogger._flatten_messages_for_moderation(None) + assert result == () + + def test_multiple_messages_preserved_in_order(self): + messages = [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Question?"}, + ] + result = RubrikLogger._flatten_messages_for_moderation(messages) + assert len(result) == 2 + assert result[0]["role"] == "system" + assert result[1]["role"] == "user" + + +# -- _build_prompt_moderation_payload ----------------------------------------- + + +class TestBuildPromptModerationPayload: + def test_payload_includes_tools_when_present(self): + inputs = { + "model": "gpt-4", + "structured_messages": [{"role": "user", "content": "hi"}], + "tools": [{"type": "function", "function": {"name": "fn"}}], + } + payload = RubrikLogger._build_prompt_moderation_payload(inputs, {}) + assert payload["tools"] == [{"type": "function", "function": {"name": "fn"}}] + + def test_payload_includes_user_when_present(self): + inputs = { + "structured_messages": [{"role": "user", "content": "hi"}], + } + request_data = {"user": "alice"} + payload = RubrikLogger._build_prompt_moderation_payload(inputs, request_data) + assert payload["user"] == "alice" + + def test_payload_uses_explicit_correlation_key(self): + inputs = {"structured_messages": [{"role": "user", "content": "hi"}]} + request_data = { + "correlation_key": "corr-123", + "litellm_call_id": "litellm-456", + } + payload = RubrikLogger._build_prompt_moderation_payload(inputs, request_data) + assert payload["correlation_key"] == "corr-123" + + def test_payload_falls_back_to_litellm_call_id(self): + """When correlation_key is absent, litellm_call_id is used.""" + inputs = {"structured_messages": [{"role": "user", "content": "hi"}]} + request_data = {"litellm_call_id": "litellm-789"} + payload = RubrikLogger._build_prompt_moderation_payload(inputs, request_data) + assert payload["correlation_key"] == "litellm-789" + + def test_payload_omits_optional_fields_when_absent(self): + inputs = {"structured_messages": [{"role": "user", "content": "hi"}]} + payload = RubrikLogger._build_prompt_moderation_payload(inputs, {}) + assert "tools" not in payload + assert "user" not in payload + assert "correlation_key" not in payload + + +# -- _extract_request_data tools preference ----------------------------------- + + +class TestExtractRequestDataToolsPreference: + def test_prefers_tools_from_request_data_over_optional_params(self): + """When 'tools' key exists in request_data, it wins over optional_params.""" + call_details = { + "messages": [{"role": "user", "content": "hi"}], + "model": "gpt-4", + "optional_params": { + "tools": [{"type": "function", "function": {"name": "from_optional"}}] + }, + } + request_data = { + "tools": [{"type": "function", "function": {"name": "from_request"}}] + } + result = RubrikLogger._extract_request_data(call_details, request_data) + assert result["tools"] == [ + {"type": "function", "function": {"name": "from_request"}} + ] + + def test_falls_back_to_optional_params_when_not_in_request_data(self): + call_details = { + "optional_params": { + "tools": [{"type": "function", "function": {"name": "from_optional"}}] + } + } + result = RubrikLogger._extract_request_data(call_details, {}) + assert result["tools"] == [ + {"type": "function", "function": {"name": "from_optional"}} + ] + + def test_explicit_empty_list_in_request_data_is_forwarded(self): + """An explicit empty tools list signals 'no tools' to the moderation service.""" + call_details = { + "optional_params": { + "tools": [{"type": "function", "function": {"name": "from_optional"}}] + } + } + request_data = {"tools": []} + result = RubrikLogger._extract_request_data(call_details, request_data) + assert result["tools"] == [] + + +# -- _extract_prompt_refusal -------------------------------------------------- + + +class TestExtractPromptRefusal: + def test_passthrough_response_returns_none(self): + """Empty dict (passthrough) → None.""" + assert RubrikLogger._extract_prompt_refusal({}) is None + + def test_no_choices_returns_none(self): + assert RubrikLogger._extract_prompt_refusal({"choices": []}) is None + + def test_block_response_returns_content(self): + service_response = { + "choices": [{"message": {"content": "Request blocked by Rubrik."}}] + } + result = RubrikLogger._extract_prompt_refusal(service_response) + assert result == "Request blocked by Rubrik." + + def test_empty_content_falls_back_to_default_message(self): + """When content is empty string or falsy, falls back to default refusal.""" + service_response = {"choices": [{"message": {"content": ""}}]} + result = RubrikLogger._extract_prompt_refusal(service_response) + assert result == "Request blocked by policy." + + def test_none_content_falls_back_to_default_message(self): + service_response = {"choices": [{"message": {"content": None}}]} + result = RubrikLogger._extract_prompt_refusal(service_response) + assert result == "Request blocked by policy." + + +# -- _prepend_system_prompt exception path ------------------------------------ + + +class TestPrependSystemPromptException: + def test_exception_during_unpack_is_caught_and_logged(self): + """When an exception is raised inside _prepend_system_prompt, it is swallowed.""" + + class ExplodingList(list): + def __iter__(self): + raise RuntimeError("iteration error!") + + payload = {"messages": ExplodingList()} + source = {"system": "You are an assistant."} + + # Must not raise + RubrikLogger._prepend_system_prompt(payload, source) + + +# -- _append_and_maybe_flush batch trigger ------------------------------------ + + +@pytest.mark.asyncio +class TestAppendAndMaybeFlush: + async def test_flush_triggered_when_queue_reaches_batch_size(self, handler): + """flush_queue is called when the queue length reaches batch_size.""" + handler.batch_size = 2 + handler.flush_queue = AsyncMock() + + await handler._append_and_maybe_flush({"msg": "a"}) + handler.flush_queue.assert_not_called() + + await handler._append_and_maybe_flush({"msg": "b"}) + handler.flush_queue.assert_called_once() + + async def test_no_flush_before_batch_size(self, handler): + handler.batch_size = 5 + handler.flush_queue = AsyncMock() + + for i in range(4): + await handler._append_and_maybe_flush({"msg": str(i)}) + + handler.flush_queue.assert_not_called() + + +# -- _enqueue_log_event exception handling ------------------------------------ + + +@pytest.mark.asyncio +class TestEnqueueLogEventExceptions: + async def test_exception_from_prepare_log_payload_is_caught(self, handler): + """Exceptions raised by _prepare_log_payload are caught and logged.""" + handler._prepare_log_payload = AsyncMock( + side_effect=RuntimeError("payload error") + ) + + # Must not raise + await handler._enqueue_log_event( + {"standard_logging_object": {"messages": [], "response": ""}}, "test" + ) + assert len(handler.log_queue) == 0 + + +# -- async_log_success_event skip when _rubrik_blocked ------------------------ + + +@pytest.mark.asyncio +class TestSuccessEventBlockedSkip: + async def test_skips_enqueue_when_rubrik_blocked_flag_set(self, handler): + """When kwargs['_rubrik_blocked'] is True, the event is not enqueued.""" + kwargs = { + "_rubrik_blocked": True, + "litellm_call_id": "blocked-call-123", + "standard_logging_object": { + "messages": [{"role": "user", "content": "hi"}], + "response": "hello", + }, + } + await handler.async_log_success_event( + kwargs=kwargs, response_obj=None, start_time=None, end_time=None + ) + assert len(handler.log_queue) == 0 + + +# -- async_post_call_failure_hook --------------------------------------------- + + +@pytest.mark.asyncio +class TestPostCallFailureHook: + async def test_non_modify_exception_returns_immediately(self, handler): + """Non-ModifyResponseException causes a no-op.""" + await handler.async_post_call_failure_hook( + request_data={"litellm_call_id": "test"}, + original_exception=ValueError("unrelated error"), + user_api_key_dict=None, + ) + assert len(handler.log_queue) == 0 + + async def test_modify_exception_without_stashed_logging_obj_emits_warning( + self, handler + ): + """ModifyResponseException with no _rubrik_logging_obj → warning, no enqueue.""" + request_data = {"litellm_call_id": "test-123", "model": "gpt-4"} + exc = ModifyResponseException( + message="blocked", + model="gpt-4", + request_data=request_data, + guardrail_name="rubrik", + ) + + await handler.async_post_call_failure_hook( + request_data=request_data, + original_exception=exc, + user_api_key_dict=None, + ) + assert len(handler.log_queue) == 0 + + async def test_modify_exception_with_valid_logging_obj_enqueues_payload( + self, handler + ): + """ModifyResponseException + stashed logging_obj → builds and enqueues.""" + logging_obj = Mock() + logging_obj.model_call_details = { + "litellm_call_id": "call-abc", + "model": "gpt-4", + "messages": [{"role": "user", "content": "hi"}], + "standard_logging_object": { + "id": "chatcmpl-original", + "model": "gpt-4", + "response": "original", + "messages": [{"role": "user", "content": "hi"}], + }, + "metadata": {}, + } + + request_data = {"_rubrik_logging_obj": logging_obj} + exc = ModifyResponseException( + message="blocked by policy", + model="gpt-4", + request_data=request_data, + guardrail_name="rubrik", + ) + + handler.batch_size = 10**6 # disable auto-flush + await handler.async_post_call_failure_hook( + request_data=request_data, + original_exception=exc, + user_api_key_dict=None, + ) + assert len(handler.log_queue) == 1 + assert "ModifyResponseException" in handler.log_queue[0]["response"] + + async def test_logging_obj_popped_from_request_data(self, handler): + """_rubrik_logging_obj must be popped from request_data so it is not + forwarded downstream.""" + logging_obj = Mock() + logging_obj.model_call_details = { + "litellm_call_id": "call-pop", + "model": "gpt-4", + "messages": [], + "standard_logging_object": { + "id": "chatcmpl-pop", + "model": "gpt-4", + "response": "text", + "messages": [], + }, + "metadata": {}, + } + + request_data = {"_rubrik_logging_obj": logging_obj} + exc = ModifyResponseException( + message="popped", + model="gpt-4", + request_data=request_data, + guardrail_name="rubrik", + ) + + handler.batch_size = 10**6 + await handler.async_post_call_failure_hook( + request_data=request_data, + original_exception=exc, + user_api_key_dict=None, + ) + assert "_rubrik_logging_obj" not in request_data + + async def test_build_and_enqueue_swallows_attribute_error_from_prepare_payload( + self, handler + ): + """When _prepare_block_failure_payload raises AttributeError/KeyError/TypeError, + the error is logged and the event is silently dropped (lines 806-812).""" + logging_obj = Mock() + # Make model_call_details.get() raise TypeError + logging_obj.model_call_details = None # .get() will raise AttributeError + + exc = ModifyResponseException( + message="blocked", + model="gpt-4", + request_data={}, + guardrail_name="rubrik", + ) + + # Must not raise + await handler._build_and_enqueue_block_event(logging_obj, exc, None) + assert len(handler.log_queue) == 0 + + async def test_build_and_enqueue_swallows_flush_exception(self, handler): + """When _append_and_maybe_flush raises, the error is logged (lines 816-817).""" + logging_obj = Mock() + logging_obj.model_call_details = { + "litellm_call_id": "call-flush-err", + "model": "gpt-4", + "messages": [], + "standard_logging_object": { + "id": "id-flush-err", + "model": "gpt-4", + "response": "text", + "messages": [], + }, + "metadata": {}, + } + + exc = ModifyResponseException( + message="blocked", + model="gpt-4", + request_data={}, + guardrail_name="rubrik", + ) + + handler._append_and_maybe_flush = AsyncMock( + side_effect=RuntimeError("flush failed") + ) + + # Must not raise + await handler._build_and_enqueue_block_event(logging_obj, exc, None) + + +# -- _prepare_block_failure_payload and _build_fallback_payload --------------- + + +class TestPrepareBlockFailurePayload: + def test_uses_standard_logging_object_when_present(self, handler): + """When standard_logging_object is on model_call_details, it is used as base.""" + logging_obj = Mock() + logging_obj.model_call_details = { + "litellm_call_id": "call-slo", + "model": "gpt-4", + "standard_logging_object": { + "id": "chatcmpl-original", + "model": "gpt-4", + "response": "original response", + "messages": [{"role": "user", "content": "hi"}], + }, + "metadata": {}, + } + exc = ModifyResponseException( + message="blocked", + model="gpt-4", + request_data={}, + guardrail_name="rubrik", + ) + + payload = handler._prepare_block_failure_payload(logging_obj, exc) + + assert "ModifyResponseException: blocked" in payload["response"] + assert payload["id"] == "call-slo" + + def test_uses_fallback_when_standard_logging_object_absent(self, handler): + """When standard_logging_object is absent, _build_fallback_payload is used.""" + from datetime import datetime + + logging_obj = Mock() + logging_obj.model_call_details = { + "litellm_call_id": "call-fallback", + "model": "claude-3", + "messages": [{"role": "user", "content": "question"}], + "optional_params": {"temperature": 0.5}, + "metadata": {"user_api_key_hash": "hash-abc"}, + "start_time": datetime(2024, 6, 1), + } + exc = ModifyResponseException( + message="prompt blocked", + model="claude-3", + request_data={}, + guardrail_name="rubrik", + ) + + payload = handler._prepare_block_failure_payload(logging_obj, exc) + + assert payload["id"] == "call-fallback" + assert payload["model"] == "claude-3" + assert payload["model_group"] == "claude-3" + assert "ModifyResponseException: prompt blocked" in payload["response"] + assert payload["metadata"]["user_api_key_hash"] == "hash-abc" + assert payload["status"] == "failure" + + def test_fallback_payload_without_start_time(self, handler): + """_build_fallback_payload handles missing start_time gracefully.""" + logging_obj = Mock() + logging_obj.model_call_details = { + "litellm_call_id": "call-notime", + "model": "gpt-4", + "messages": [], + "optional_params": {}, + "metadata": {}, + } + exc = ModifyResponseException( + message="blocked", + model="gpt-4", + request_data={}, + guardrail_name="rubrik", + ) + + payload = handler._prepare_block_failure_payload(logging_obj, exc) + assert payload["startTime"] is None + + +# -- async_send_batch empty queue and flush_queue edge cases ------------------ + + +@pytest.mark.asyncio +class TestQueueEdgeCases: + async def test_async_send_batch_returns_early_on_empty_queue(self, handler): + """async_send_batch is a no-op when the queue is empty.""" + handler.async_httpx_client = AsyncMock() + await handler.async_send_batch() + handler.async_httpx_client.post.assert_not_called() + + async def test_flush_queue_returns_early_when_flush_lock_is_none(self, handler): + """flush_queue is a no-op when flush_lock is None.""" + handler.flush_lock = None + handler.log_queue = [{"msg": "a"}] + handler.async_httpx_client = AsyncMock() + + await handler.flush_queue() + handler.async_httpx_client.post.assert_not_called() + + async def test_flush_queue_returns_early_when_queue_empty_inside_lock(self, handler): + """flush_queue acquires the lock then no-ops when the queue is empty.""" + handler.log_queue = [] + handler.async_httpx_client = AsyncMock() + + await handler.flush_queue() + handler.async_httpx_client.post.assert_not_called() + + +# -- _post_json non-dict response --------------------------------------------- + + +@pytest.mark.asyncio +class TestPostJson: + async def test_raises_type_error_for_list_response(self, handler): + """When the service returns a JSON array instead of a dict, TypeError is raised.""" + mock_client = AsyncMock() + mock_resp = Mock() + mock_resp.json.return_value = ["not", "a", "dict"] + mock_resp.raise_for_status = Mock() + mock_client.post = AsyncMock(return_value=mock_resp) + handler.moderation_client = mock_client + + with pytest.raises(TypeError, match="non-dict JSON"): + await handler._post_json( + handler.prompt_moderation_endpoint, {}, "Test service" + ) + + async def test_raises_type_error_for_string_response(self, handler): + """A bare string response also raises TypeError.""" + mock_client = AsyncMock() + mock_resp = Mock() + mock_resp.json.return_value = "blocked" + mock_resp.raise_for_status = Mock() + mock_client.post = AsyncMock(return_value=mock_resp) + handler.moderation_client = mock_client + + with pytest.raises(TypeError, match="non-dict JSON"): + await handler._post_json( + handler.response_moderation_endpoint, {}, "Test service" + ) From 722d9ffa4f6c5ae15702ab9ab2c5f6bf1688308b Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 3 Aug 2026 17:22:11 -0700 Subject: [PATCH 058/124] feat(spend): add caller-scoped key/user/team/organization spend report endpoints --- litellm/proxy/_types.py | 4 + .../spend_management_endpoints.py | 351 +++++++++++++ .../test_spend_management_endpoints.py | 461 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 249 ++++++++++ 4 files changed, 1065 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 3ccf3ea9952..a81bbd8aff3 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -638,6 +638,10 @@ class LiteLLMRoutes(enum.Enum): "/spend/logs/v2", "/spend/logs/ui", "/spend/logs/session/ui", + "/key/spend/report", + "/user/spend/report", + "/team/spend/report", + "/organization/spend/report", # Reads end users out of spend logs, scoped to the caller's own rows and # permitted teams exactly like /spend/logs/ui — it belongs to the same # access tier, not to customer management. diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 0bcc2b9994b..4b202a5054d 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -6,6 +6,7 @@ from collections.abc import Mapping, Sequence from datetime import datetime, timedelta, timezone from typing import ( TYPE_CHECKING, + Annotated, Any, Literal, NamedTuple, @@ -1455,6 +1456,356 @@ async def get_global_spend_report( ) +_SPEND_REPORT_SCOPE_COLUMNS = frozenset({"api_key", "user", "team_id"}) + + +def _scoped_spend_report_sql(scope_column: str) -> str: + """Spend grouped by api_key with a per-model breakdown, cut to one scope column. + + ``scope_column`` is interpolated into the SQL, so it must come from + ``_SPEND_REPORT_SCOPE_COLUMNS`` — never from caller input. Scope values are + always bound as ``$3``. + """ + if scope_column not in _SPEND_REPORT_SCOPE_COLUMNS: + raise ValueError(f"Unsupported spend report scope column: {scope_column!r}") + return f""" + WITH SpendByModelApiKey AS ( + SELECT + sl.api_key, + sl.model, + SUM(sl.spend) AS model_cost, + SUM(sl.prompt_tokens) AS model_input_tokens, + SUM(sl.completion_tokens) AS model_output_tokens + FROM + "LiteLLM_SpendLogs" sl + WHERE + sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC') + AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') + AND sl.{scope_column} = $3 + GROUP BY + sl.api_key, + sl.model + ) + SELECT + api_key, + SUM(model_cost) AS total_cost, + SUM(model_input_tokens) AS total_input_tokens, + SUM(model_output_tokens) AS total_output_tokens, + jsonb_agg(jsonb_build_object( + 'model', model, + 'total_cost', model_cost, + 'total_input_tokens', model_input_tokens, + 'total_output_tokens', model_output_tokens + )) AS model_details + FROM + SpendByModelApiKey + GROUP BY + api_key + ORDER BY + total_cost DESC; + """ + + +_ORG_SPEND_REPORT_SQL = """ + WITH SpendByModelApiKey AS ( + SELECT + sl.api_key, + sl.team_id, + sl.model, + SUM(sl.spend) AS model_cost, + SUM(sl.prompt_tokens) AS model_input_tokens, + SUM(sl.completion_tokens) AS model_output_tokens + FROM + "LiteLLM_SpendLogs" sl + WHERE + sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC') + AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') + AND (sl.organization_id = $3 OR sl.team_id = ANY($4::text[])) + GROUP BY + sl.api_key, + sl.team_id, + sl.model + ) + SELECT + api_key, + SUM(model_cost) AS total_cost, + SUM(model_input_tokens) AS total_input_tokens, + SUM(model_output_tokens) AS total_output_tokens, + jsonb_agg(jsonb_build_object( + 'team_id', team_id, + 'model', model, + 'total_cost', model_cost, + 'total_input_tokens', model_input_tokens, + 'total_output_tokens', model_output_tokens + )) AS model_details + FROM + SpendByModelApiKey + GROUP BY + api_key + ORDER BY + total_cost DESC; +""" + + +def _spend_report_prereqs() -> PrismaClient: + from litellm.proxy.proxy_server import premium_user, prisma_client + + if prisma_client is None: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=CommonProxyErrors.db_not_connected_error.value, + ) + if premium_user is not True: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="/spend/report endpoint " + CommonProxyErrors.not_premium_user.value, + ) + return prisma_client + + +def _parse_spend_report_date_range(start_date: str | None, end_date: str | None) -> tuple[datetime, datetime]: + if start_date is None or end_date is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Please provide start_date and end_date", + ) + try: + parsed = ( + datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=timezone.utc), + datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=timezone.utc), + ) + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="start_date and end_date must be in YYYY-MM-DD format", + ) + return parsed + + +def _resolve_spend_report_scope( + user_api_key_dict: UserAPIKeyAuth, + requested: str | None, + caller_value: str | None, + scope_name: str, +) -> str: + """Return the scope value the caller may query spend for. + + Non-admin callers are clamped to their own identity: a ``requested`` value + that differs from ``caller_value`` is a 403. Proxy admins (and admin + viewers) may request any scope. + """ + if requested: + if requested != caller_value and not _is_admin_view_safe(user_api_key_dict=user_api_key_dict): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=f"Not authorized to view spend for a {scope_name} other than your own", + ) + return requested + if caller_value is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"No {scope_name} associated with this API key; pass a {scope_name} query param", + ) + return caller_value + + +async def _resolve_org_spend_report_scope( + user_api_key_dict: UserAPIKeyAuth, + organization_id: str | None, + prisma_client: PrismaClient, +) -> tuple[str, tuple[str, ...]]: + """Return the organization to report on and the team_ids belonging to it. + + Callable by proxy admins (any organization) and org admins of the target + organization; every other caller is a 403 from ``_verify_org_access``. + """ + from litellm.proxy.management_endpoints.organization_endpoints import _verify_org_access + + target_org = organization_id or user_api_key_dict.org_id + if target_org is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="No organization_id associated with this API key; pass an organization_id query param", + ) + await _verify_org_access( + organization_id=target_org, + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + ) + teams = await TeamRepository(prisma_client).find_by_organization_id(organization_id=target_org) + return target_org, tuple(team.team_id for team in teams) + + +@router.get( + "/key/spend/report", + tags=("Budget & Spend Tracking",), +) +async def get_key_spend_report( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + start_date: Annotated[ + str | None, fastapi.Query(description="Time from which to start viewing spend (YYYY-MM-DD)") + ] = None, + end_date: Annotated[str | None, fastapi.Query(description="Time till which to view spend (YYYY-MM-DD)")] = None, + api_key: Annotated[ + str | None, + fastapi.Query( + description="View spend for a specific api_key. Proxy admin only; other callers are scoped to their own key." + ), + ] = None, +) -> Sequence[Mapping[str, object]]: + """ + Get spend for the calling api_key over a date range, with a per-model breakdown. + + Same row shape as `/global/spend/report?api_key=...`, but callable by any key: + non-admin callers are always scoped to their own api_key, while proxy admins + may pass `?api_key=` to view any key. + """ + prisma_client = _spend_report_prereqs() + start_date_obj, end_date_obj = _parse_spend_report_date_range(start_date=start_date, end_date=end_date) + requested = hash_token(token=api_key) if api_key is not None and api_key.startswith("sk-") else api_key + scoped_api_key = _resolve_spend_report_scope( + user_api_key_dict=user_api_key_dict, + requested=requested, + caller_value=user_api_key_dict.api_key, + scope_name="api_key", + ) + db_response: Sequence[Mapping[str, object]] | None = await _query_raw_or_none( + prisma_client, + _scoped_spend_report_sql(scope_column="api_key"), + start_date_obj, + end_date_obj, + scoped_api_key, + ) + return db_response or () + + +@router.get( + "/user/spend/report", + tags=("Budget & Spend Tracking",), +) +async def get_user_spend_report( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + start_date: Annotated[ + str | None, fastapi.Query(description="Time from which to start viewing spend (YYYY-MM-DD)") + ] = None, + end_date: Annotated[str | None, fastapi.Query(description="Time till which to view spend (YYYY-MM-DD)")] = None, + internal_user_id: Annotated[ + str | None, + fastapi.Query( + description="View spend for a specific internal_user_id. Proxy admin only; other callers are scoped to their own user_id." + ), + ] = None, +) -> Sequence[Mapping[str, object]]: + """ + Get spend for the calling user over a date range, grouped by api_key with a per-model breakdown. + + Same row shape as `/global/spend/report?internal_user_id=...`, but callable by + any key with a user: non-admin callers are always scoped to their own user_id, + while proxy admins may pass `?internal_user_id=` to view any user. + """ + prisma_client = _spend_report_prereqs() + start_date_obj, end_date_obj = _parse_spend_report_date_range(start_date=start_date, end_date=end_date) + scoped_user_id = _resolve_spend_report_scope( + user_api_key_dict=user_api_key_dict, + requested=internal_user_id, + caller_value=user_api_key_dict.user_id, + scope_name="internal_user_id", + ) + db_response: Sequence[Mapping[str, object]] | None = await _query_raw_or_none( + prisma_client, + _scoped_spend_report_sql(scope_column="user"), + start_date_obj, + end_date_obj, + scoped_user_id, + ) + return db_response or () + + +@router.get( + "/team/spend/report", + tags=("Budget & Spend Tracking",), +) +async def get_team_spend_report( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + start_date: Annotated[ + str | None, fastapi.Query(description="Time from which to start viewing spend (YYYY-MM-DD)") + ] = None, + end_date: Annotated[str | None, fastapi.Query(description="Time till which to view spend (YYYY-MM-DD)")] = None, + team_id: Annotated[ + str | None, + fastapi.Query( + description="View spend for a specific team_id. Proxy admin only; other callers are scoped to their key's team." + ), + ] = None, +) -> Sequence[Mapping[str, object]]: + """ + Get spend for the calling key's team over a date range, grouped by api_key with a per-model breakdown. + + Callable by any key that belongs to a team: non-admin callers are always + scoped to their key's team_id, while proxy admins may pass `?team_id=` to + view any team. + """ + prisma_client = _spend_report_prereqs() + start_date_obj, end_date_obj = _parse_spend_report_date_range(start_date=start_date, end_date=end_date) + scoped_team_id = _resolve_spend_report_scope( + user_api_key_dict=user_api_key_dict, + requested=team_id, + caller_value=user_api_key_dict.team_id, + scope_name="team_id", + ) + db_response: Sequence[Mapping[str, object]] | None = await _query_raw_or_none( + prisma_client, + _scoped_spend_report_sql(scope_column="team_id"), + start_date_obj, + end_date_obj, + scoped_team_id, + ) + return db_response or () + + +@router.get( + "/organization/spend/report", + tags=("Budget & Spend Tracking",), +) +async def get_organization_spend_report( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + start_date: Annotated[ + str | None, fastapi.Query(description="Time from which to start viewing spend (YYYY-MM-DD)") + ] = None, + end_date: Annotated[str | None, fastapi.Query(description="Time till which to view spend (YYYY-MM-DD)")] = None, + organization_id: Annotated[ + str | None, + fastapi.Query( + description="View spend for a specific organization_id. Proxy admins may pass any organization; org admins are scoped to organizations they administer." + ), + ] = None, +) -> Sequence[Mapping[str, object]]: + """ + Get spend for an organization over a date range, grouped by api_key with a per-model and per-team breakdown. + + Covers spend logged against the organization directly and against any of its + teams. Callable by proxy admins (any organization) and org admins (their own + organizations). Defaults to the calling key's organization_id when + `?organization_id=` is omitted. + """ + prisma_client = _spend_report_prereqs() + start_date_obj, end_date_obj = _parse_spend_report_date_range(start_date=start_date, end_date=end_date) + target_org, team_ids = await _resolve_org_spend_report_scope( + user_api_key_dict=user_api_key_dict, + organization_id=organization_id, + prisma_client=prisma_client, + ) + db_response: Sequence[Mapping[str, object]] | None = await _query_raw_or_none( + prisma_client, + _ORG_SPEND_REPORT_SQL, + start_date_obj, + end_date_obj, + target_org, + team_ids, + ) + return db_response or () + + @router.get( "/global/spend/all_tag_names", tags=["Budget & Spend Tracking"], diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index f6216d1646e..e0f7ba7a9b3 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -4777,3 +4777,464 @@ def test_ui_view_request_response_reads_from_cold_storage(client, monkeypatch): assert cold_logger.requested_object_keys == ["k/cold.json"] finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +from litellm.proxy._types import ( + LiteLLM_OrganizationMembershipTable, + LiteLLM_UserTable, + LiteLLMRoutes, + hash_token, +) + +_SCOPED_SPEND_REPORT_PATHS = [ + "/key/spend/report", + "/user/spend/report", + "/team/spend/report", + "/organization/spend/report", +] + + +def _spend_report_mock_prisma(query_raw_returns=None, team_rows=None, user_row=None): + pc = MagicMock() + pc.db.query_raw = AsyncMock( + return_value=query_raw_returns if query_raw_returns is not None else [] + ) + pc.db.litellm_teamtable.find_many = AsyncMock( + return_value=team_rows if team_rows is not None else [] + ) + pc.db.litellm_usertable.find_unique = AsyncMock(return_value=user_row) + return pc + + +def _org_member_user_row(user_id, organization_id, membership_role): + now = datetime.datetime.now(timezone.utc) + return LiteLLM_UserTable( + user_id=user_id, + user_email=f"{user_id}@example.com", + organization_memberships=[ + LiteLLM_OrganizationMembershipTable( + user_id=user_id, + organization_id=organization_id, + user_role=membership_role, + created_at=now, + updated_at=now, + ) + ], + ) + + +def test_scoped_spend_report_routes_reachable_by_non_admin_roles(): + """ + The whole point of the scoped report endpoints is that non-admin callers can + reach them. If they fall out of spend_tracking_routes (and with it the + internal-user route allowlists), user_api_key_auth rejects every non-admin + caller before the endpoint runs. + """ + for path in _SCOPED_SPEND_REPORT_PATHS: + assert path in LiteLLMRoutes.spend_tracking_routes.value + assert path in LiteLLMRoutes.internal_user_routes.value + assert path in LiteLLMRoutes.internal_user_view_only_routes.value + assert path in LiteLLMRoutes.org_admin_allowed_routes.value + + +def test_resolve_spend_report_scope_defaults_to_caller(): + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="alice") + resolved = spend_management_endpoints._resolve_spend_report_scope( + user_api_key_dict=auth, + requested=None, + caller_value="team-blue", + scope_name="team_id", + ) + assert resolved == "team-blue" + + +def test_resolve_spend_report_scope_non_admin_override_forbidden(): + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="alice") + with pytest.raises(HTTPException) as exc_info: + spend_management_endpoints._resolve_spend_report_scope( + user_api_key_dict=auth, + requested="team-red", + caller_value="team-blue", + scope_name="team_id", + ) + assert exc_info.value.status_code == 403 + + +def test_resolve_spend_report_scope_non_admin_matching_override_allowed(): + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="alice") + resolved = spend_management_endpoints._resolve_spend_report_scope( + user_api_key_dict=auth, + requested="team-blue", + caller_value="team-blue", + scope_name="team_id", + ) + assert resolved == "team-blue" + + +@pytest.mark.parametrize( + "role", + [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY], +) +def test_resolve_spend_report_scope_admin_override_allowed(role): + auth = UserAPIKeyAuth(user_role=role, user_id="admin") + resolved = spend_management_endpoints._resolve_spend_report_scope( + user_api_key_dict=auth, + requested="team-red", + caller_value="team-blue", + scope_name="team_id", + ) + assert resolved == "team-red" + + +def test_resolve_spend_report_scope_missing_caller_value_400(): + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="alice") + with pytest.raises(HTTPException) as exc_info: + spend_management_endpoints._resolve_spend_report_scope( + user_api_key_dict=auth, + requested=None, + caller_value=None, + scope_name="team_id", + ) + assert exc_info.value.status_code == 400 + + +@pytest.mark.parametrize("bad_column", ["metadata", "end_user", "evil; DROP TABLE", ""]) +def test_scoped_spend_report_sql_rejects_unknown_column(bad_column): + with pytest.raises(ValueError): + spend_management_endpoints._scoped_spend_report_sql(scope_column=bad_column) + + +def test_key_spend_report_scopes_to_caller_key(client, monkeypatch): + mock_prisma = _spend_report_mock_prisma( + query_raw_returns=[{"api_key": "hashed-caller-key", "total_cost": 1.5}] + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="alice", + api_key="hashed-caller-key", + ) + try: + response = client.get( + "/key/spend/report", + params={"start_date": "2026-07-01", "end_date": "2026-07-31"}, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200 + assert response.json() == [{"api_key": "hashed-caller-key", "total_cost": 1.5}] + args, _ = mock_prisma.db.query_raw.await_args + sql, start_param, end_param, scope_param = args + assert "sl.api_key = $3" in sql + assert scope_param == "hashed-caller-key" + assert start_param == datetime.datetime(2026, 7, 1, tzinfo=timezone.utc) + assert end_param == datetime.datetime(2026, 7, 31, tzinfo=timezone.utc) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +def test_key_spend_report_non_admin_override_403(client, monkeypatch): + mock_prisma = _spend_report_mock_prisma() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="alice", + api_key="hashed-caller-key", + ) + try: + response = client.get( + "/key/spend/report", + params={ + "start_date": "2026-07-01", + "end_date": "2026-07-31", + "api_key": "hashed-someone-elses-key", + }, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 403 + mock_prisma.db.query_raw.assert_not_awaited() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +def test_key_spend_report_admin_override_sk_key_gets_hashed(client, monkeypatch): + mock_prisma = _spend_report_mock_prisma() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin", api_key="hashed-admin-key" + ) + try: + response = client.get( + "/key/spend/report", + params={ + "start_date": "2026-07-01", + "end_date": "2026-07-31", + "api_key": "sk-target-key", + }, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200 + args, _ = mock_prisma.db.query_raw.await_args + scope_param = args[3] + assert scope_param == hash_token(token="sk-target-key") + assert "sk-target-key" not in args[0] + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +def test_user_spend_report_scopes_to_caller_user_id(client, monkeypatch): + mock_prisma = _spend_report_mock_prisma(query_raw_returns=[{"api_key": "k1"}]) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="alice", api_key="hashed-k" + ) + try: + response = client.get( + "/user/spend/report", + params={"start_date": "2026-07-01", "end_date": "2026-07-31"}, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200 + args, _ = mock_prisma.db.query_raw.await_args + sql, _, _, scope_param = args + assert "sl.user = $3" in sql + assert scope_param == "alice" + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +def test_user_spend_report_non_admin_override_403(client, monkeypatch): + mock_prisma = _spend_report_mock_prisma() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="alice", api_key="hashed-k" + ) + try: + response = client.get( + "/user/spend/report", + params={ + "start_date": "2026-07-01", + "end_date": "2026-07-31", + "internal_user_id": "bob", + }, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 403 + mock_prisma.db.query_raw.assert_not_awaited() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +def test_team_spend_report_scopes_to_key_team(client, monkeypatch): + mock_prisma = _spend_report_mock_prisma(query_raw_returns=[{"api_key": "k1"}]) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="alice", + api_key="hashed-k", + team_id="team-blue", + ) + try: + response = client.get( + "/team/spend/report", + params={"start_date": "2026-07-01", "end_date": "2026-07-31"}, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200 + args, _ = mock_prisma.db.query_raw.await_args + sql, _, _, scope_param = args + assert "sl.team_id = $3" in sql + assert scope_param == "team-blue" + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +def test_team_spend_report_no_team_400(client, monkeypatch): + mock_prisma = _spend_report_mock_prisma() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="alice", api_key="hashed-k" + ) + try: + response = client.get( + "/team/spend/report", + params={"start_date": "2026-07-01", "end_date": "2026-07-31"}, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 400 + mock_prisma.db.query_raw.assert_not_awaited() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +def test_org_spend_report_proxy_admin_override(client, monkeypatch): + mock_prisma = _spend_report_mock_prisma( + query_raw_returns=[{"api_key": "k1"}], + team_rows=[{"team_id": "team-a"}, {"team_id": "team-b"}], + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin", api_key="hashed-admin" + ) + try: + response = client.get( + "/organization/spend/report", + params={ + "start_date": "2026-07-01", + "end_date": "2026-07-31", + "organization_id": "org-x", + }, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200 + args, _ = mock_prisma.db.query_raw.await_args + sql, _, _, org_param, team_ids_param = args + assert "(sl.organization_id = $3 OR sl.team_id = ANY($4::text[]))" in sql + assert org_param == "org-x" + assert team_ids_param == ("team-a", "team-b") + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +def test_org_spend_report_org_admin_auto_scopes_to_own_org(client, monkeypatch): + user_id = "org-admin-auto-scope" + mock_prisma = _spend_report_mock_prisma( + query_raw_returns=[{"api_key": "k1"}], + team_rows=[{"team_id": "team-a"}], + user_row=_org_member_user_row( + user_id=user_id, + organization_id="org-acme", + membership_role=LitellmUserRoles.ORG_ADMIN.value, + ), + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + user_id=user_id, + api_key="hashed-org-admin-key", + org_id="org-acme", + ) + try: + response = client.get( + "/organization/spend/report", + params={"start_date": "2026-07-01", "end_date": "2026-07-31"}, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200 + args, _ = mock_prisma.db.query_raw.await_args + org_param, team_ids_param = args[3], args[4] + assert org_param == "org-acme" + assert team_ids_param == ("team-a",) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +def test_org_spend_report_non_org_admin_403(client, monkeypatch): + user_id = "org-plain-member" + mock_prisma = _spend_report_mock_prisma( + user_row=_org_member_user_row( + user_id=user_id, + organization_id="org-acme", + membership_role=LitellmUserRoles.INTERNAL_USER.value, + ), + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + user_id=user_id, + api_key="hashed-member-key", + org_id="org-acme", + ) + try: + response = client.get( + "/organization/spend/report", + params={"start_date": "2026-07-01", "end_date": "2026-07-31"}, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 403 + mock_prisma.db.query_raw.assert_not_awaited() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +def test_org_spend_report_no_org_400(client, monkeypatch): + mock_prisma = _spend_report_mock_prisma() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="alice", api_key="hashed-k" + ) + try: + response = client.get( + "/organization/spend/report", + params={"start_date": "2026-07-01", "end_date": "2026-07-31"}, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 400 + mock_prisma.db.query_raw.assert_not_awaited() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.parametrize("path", _SCOPED_SPEND_REPORT_PATHS) +def test_scoped_spend_report_not_premium_403(client, monkeypatch, path): + mock_prisma = _spend_report_mock_prisma() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin", api_key="hashed-admin" + ) + try: + response = client.get( + path, + params={"start_date": "2026-07-01", "end_date": "2026-07-31"}, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 403 + mock_prisma.db.query_raw.assert_not_awaited() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.parametrize("path", _SCOPED_SPEND_REPORT_PATHS) +def test_scoped_spend_report_missing_dates_400(client, monkeypatch, path): + mock_prisma = _spend_report_mock_prisma() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin", api_key="hashed-admin" + ) + try: + response = client.get(path, headers={"Authorization": "Bearer sk-test"}) + assert response.status_code == 400 + mock_prisma.db.query_raw.assert_not_awaited() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +def test_scoped_spend_report_invalid_date_format_400(client, monkeypatch): + mock_prisma = _spend_report_mock_prisma() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin", api_key="hashed-admin" + ) + try: + response = client.get( + "/key/spend/report", + params={"start_date": "07/01/2026", "end_date": "07/31/2026"}, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 400 + mock_prisma.db.query_raw.assert_not_awaited() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 0d2ebe64eb4..1f4d1ebd645 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -6873,6 +6873,30 @@ export interface paths { patch?: never; trace?: never; }; + "/key/spend/report": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Key Spend Report + * @description Get spend for the calling api_key over a date range, with a per-model breakdown. + * + * Same row shape as `/global/spend/report?api_key=...`, but callable by any key: + * non-admin callers are always scoped to their own api_key, while proxy admins + * may pass `?api_key=` to view any key. + */ + get: operations["get_key_spend_report_key_spend_report_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/key/unblock": { parameters: { query?: never; @@ -9134,6 +9158,31 @@ export interface paths { patch?: never; trace?: never; }; + "/organization/spend/report": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Organization Spend Report + * @description Get spend for an organization over a date range, grouped by api_key with a per-model and per-team breakdown. + * + * Covers spend logged against the organization directly and against any of its + * teams. Callable by proxy admins (any organization) and org admins (their own + * organizations). Defaults to the calling key's organization_id when + * `?organization_id=` is omitted. + */ + get: operations["get_organization_spend_report_organization_spend_report_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/organization/update": { parameters: { query?: never; @@ -13859,6 +13908,30 @@ export interface paths { patch?: never; trace?: never; }; + "/team/spend/report": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Team Spend Report + * @description Get spend for the calling key's team over a date range, grouped by api_key with a per-model breakdown. + * + * Callable by any key that belongs to a team: non-admin callers are always + * scoped to their key's team_id, while proxy admins may pass `?team_id=` to + * view any team. + */ + get: operations["get_team_spend_report_team_spend_report_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/team/unblock": { parameters: { query?: never; @@ -14885,6 +14958,30 @@ export interface paths { patch?: never; trace?: never; }; + "/user/spend/report": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get User Spend Report + * @description Get spend for the calling user over a date range, grouped by api_key with a per-model breakdown. + * + * Same row shape as `/global/spend/report?internal_user_id=...`, but callable by + * any key with a user: non-admin callers are always scoped to their own user_id, + * while proxy admins may pass `?internal_user_id=` to view any user. + */ + get: operations["get_user_spend_report_user_spend_report_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/user/update": { parameters: { query?: never; @@ -43226,6 +43323,44 @@ export interface operations { }; }; }; + get_key_spend_report_key_spend_report_get: { + parameters: { + query?: { + /** @description Time from which to start viewing spend (YYYY-MM-DD) */ + start_date?: string | null; + /** @description Time till which to view spend (YYYY-MM-DD) */ + end_date?: string | null; + /** @description View spend for a specific api_key. Proxy admin only; other callers are scoped to their own key. */ + api_key?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + [key: string]: unknown; + }[]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; unblock_key_key_unblock_post: { parameters: { query?: never; @@ -46232,6 +46367,44 @@ export interface operations { }; }; }; + get_organization_spend_report_organization_spend_report_get: { + parameters: { + query?: { + /** @description Time from which to start viewing spend (YYYY-MM-DD) */ + start_date?: string | null; + /** @description Time till which to view spend (YYYY-MM-DD) */ + end_date?: string | null; + /** @description View spend for a specific organization_id. Proxy admins may pass any organization; org admins are scoped to organizations they administer. */ + organization_id?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + [key: string]: unknown; + }[]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; update_organization_organization_update_patch: { parameters: { query?: never; @@ -51269,6 +51442,44 @@ export interface operations { }; }; }; + get_team_spend_report_team_spend_report_get: { + parameters: { + query?: { + /** @description Time from which to start viewing spend (YYYY-MM-DD) */ + start_date?: string | null; + /** @description Time till which to view spend (YYYY-MM-DD) */ + end_date?: string | null; + /** @description View spend for a specific team_id. Proxy admin only; other callers are scoped to their key's team. */ + team_id?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + [key: string]: unknown; + }[]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; unblock_team_team_unblock_post: { parameters: { query?: never; @@ -52512,6 +52723,44 @@ export interface operations { }; }; }; + get_user_spend_report_user_spend_report_get: { + parameters: { + query?: { + /** @description Time from which to start viewing spend (YYYY-MM-DD) */ + start_date?: string | null; + /** @description Time till which to view spend (YYYY-MM-DD) */ + end_date?: string | null; + /** @description View spend for a specific internal_user_id. Proxy admin only; other callers are scoped to their own user_id. */ + internal_user_id?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + [key: string]: unknown; + }[]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; user_update_user_update_post: { parameters: { query?: never; From 26ffb5d04ea693195aa0479a9e93d5addaffdbef Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 3 Aug 2026 17:41:49 -0700 Subject: [PATCH 059/124] fix(spend): scope org report team fallback to unstamped rows and bound report date ranges --- .../spend_management_endpoints.py | 21 ++++++- .../test_spend_management_endpoints.py | 63 ++++++++++++++++++- 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 4b202a5054d..799e8d33b6e 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1458,6 +1458,8 @@ async def get_global_spend_report( _SPEND_REPORT_SCOPE_COLUMNS = frozenset({"api_key", "user", "team_id"}) +_SPEND_REPORT_MAX_RANGE_DAYS = 366 + def _scoped_spend_report_sql(scope_column: str) -> str: """Spend grouped by api_key with a per-model breakdown, cut to one scope column. @@ -1520,7 +1522,13 @@ _ORG_SPEND_REPORT_SQL = """ WHERE sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC') AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') - AND (sl.organization_id = $3 OR sl.team_id = ANY($4::text[])) + AND ( + sl.organization_id = $3 + OR ( + (sl.organization_id IS NULL OR sl.organization_id = '') + AND sl.team_id = ANY($4::text[]) + ) + ) GROUP BY sl.api_key, sl.team_id, @@ -1579,6 +1587,17 @@ def _parse_spend_report_date_range(start_date: str | None, end_date: str | None) status_code=status.HTTP_400_BAD_REQUEST, detail="start_date and end_date must be in YYYY-MM-DD format", ) + start_date_obj, end_date_obj = parsed + if end_date_obj < start_date_obj: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="start_date must be on or before end_date", + ) + if end_date_obj - start_date_obj > timedelta(days=_SPEND_REPORT_MAX_RANGE_DAYS): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Date range too large; maximum is {_SPEND_REPORT_MAX_RANGE_DAYS} days", + ) return parsed diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index e0f7ba7a9b3..057193a69db 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -5096,7 +5096,11 @@ def test_org_spend_report_proxy_admin_override(client, monkeypatch): assert response.status_code == 200 args, _ = mock_prisma.db.query_raw.await_args sql, _, _, org_param, team_ids_param = args - assert "(sl.organization_id = $3 OR sl.team_id = ANY($4::text[]))" in sql + normalized_sql = " ".join(sql.split()) + assert ( + "AND ( sl.organization_id = $3 OR ( (sl.organization_id IS NULL OR sl.organization_id = '') " + "AND sl.team_id = ANY($4::text[]) ) )" + ) in normalized_sql assert org_param == "org-x" assert team_ids_param == ("team-a", "team-b") finally: @@ -5238,3 +5242,60 @@ def test_scoped_spend_report_invalid_date_format_400(client, monkeypatch): mock_prisma.db.query_raw.assert_not_awaited() finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +def test_scoped_spend_report_reversed_range_400(client, monkeypatch): + mock_prisma = _spend_report_mock_prisma() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="alice", api_key="hashed-k" + ) + try: + response = client.get( + "/key/spend/report", + params={"start_date": "2026-08-04", "end_date": "2026-08-01"}, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 400 + mock_prisma.db.query_raw.assert_not_awaited() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +def test_scoped_spend_report_range_over_max_400(client, monkeypatch): + mock_prisma = _spend_report_mock_prisma() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="alice", api_key="hashed-k" + ) + try: + response = client.get( + "/key/spend/report", + params={"start_date": "0001-01-01", "end_date": "9999-12-31"}, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 400 + mock_prisma.db.query_raw.assert_not_awaited() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +def test_scoped_spend_report_range_at_max_allowed(client, monkeypatch): + mock_prisma = _spend_report_mock_prisma(query_raw_returns=[]) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="alice", api_key="hashed-k" + ) + try: + response = client.get( + "/key/spend/report", + params={"start_date": "2025-08-03", "end_date": "2026-08-04"}, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200 + mock_prisma.db.query_raw.assert_awaited_once() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) From d4d0bf0acc078f081e7c0f7628bae3696a48ae10 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 3 Aug 2026 18:09:49 -0700 Subject: [PATCH 060/124] fix(ui): hide guardrail review buttons from non-admin users (#27535) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ui): hide guardrail review buttons from non-admin users The team guardrail submissions list rendered Approve/Reject buttons for non-admin users even though the backend correctly rejected the calls. Thread userRole from the page through GuardrailsPanel into TeamGuardrailsTab and gate the row-card and detail-panel review buttons on isAdmin so the UI matches the backend authorization. Defense in depth only — the backend remains the source of truth and is double-gated at both the route admin check and the explicit endpoint role check. Refs LIT-2494 * refactor(ui): read userRole from useAuthorized hook instead of prop drilling Drop the userRole prop chain through GuardrailsPage → GuardrailsPanel → TeamGuardrailsTab. Each component reads userRole directly from the useAuthorized hook, matching the pattern used elsewhere in the dashboard. Tests now mock useAuthorized per case (the same pattern as top_key_view.test.tsx) instead of passing userRole as a prop. Refs LIT-2494 * fix(ui): drop userRole prop on GuardrailsPanel call site in src/app/page.tsx Missed in the earlier refactor — GuardrailsPanel no longer accepts userRole as a prop (reads from useAuthorized hook), so callers must not pass it. The build was failing in production type-check. Refs LIT-2494 * fix(ui): gate guardrail forward-key toggle and header editors on proxy admin * refactor(ui): remove dead app_admin case from user role formatting --- .../_components/TeamGuardrailsTab.test.tsx | 142 +++++++++++ .../_components/TeamGuardrailsTab.tsx | 221 ++++++++++-------- .../(dashboard)/hooks/useAuthorized.test.ts | 6 +- .../src/components/user_dashboard.tsx | 2 - ui/litellm-dashboard/src/utils/roles.ts | 2 - 5 files changed, 269 insertions(+), 104 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.test.tsx new file mode 100644 index 00000000000..603cddb7b89 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.test.tsx @@ -0,0 +1,142 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { renderWithProviders } from "@/../tests/test-utils"; +import { screen, fireEvent } from "@testing-library/react"; +import { TeamGuardrailsTab } from "./TeamGuardrailsTab"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; + +vi.mock("@/components/networking", () => ({ + listGuardrailSubmissions: vi.fn(), + approveGuardrailSubmission: vi.fn(), + rejectGuardrailSubmission: vi.fn(), + updateGuardrailCall: vi.fn(), +})); + +vi.mock("@/app/(dashboard)/hooks/guardrails/useRegisterGuardrail", () => ({ + useRegisterGuardrail: () => ({ + mutateAsync: vi.fn(), + isPending: false, + }), +})); + +vi.mock("@/components/common_components/team_dropdown", () => ({ + default: () => null, +})); + +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: vi.fn(), +})); + +import { listGuardrailSubmissions } from "@/components/networking"; + +const pendingSubmission = { + guardrail_id: "guard-1", + guardrail_name: "test-pending-guardrail", + status: "pending_review", + team_id: "team-1", + team_guardrail: true, + litellm_params: { + guardrail: "generic_guardrail_api", + mode: "pre_call", + api_base: "https://example.com/guard", + headers: { "X-API-Key": "secret" }, + extra_headers: ["x-request-id"], + }, + guardrail_info: {}, + submitted_at: "2026-05-09T00:00:00Z", +}; + +const baseAuth = { + token: "test-token", + accessToken: "test-token", + userId: "user-1", + userEmail: "user@example.com", + premiumUser: false, + disabledPersonalKeyCreation: null, + showSSOBanner: false, +}; + +describe("TeamGuardrailsTab — approve/reject role gate", () => { + const mockUseAuthorized = vi.mocked(useAuthorized); + + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(listGuardrailSubmissions).mockResolvedValue({ + submissions: [pendingSubmission], + summary: { total: 1, pending_review: 1, active: 0, rejected: 0 }, + }); + }); + + it("hides Approve and Reject buttons for an internal user on a pending submission", async () => { + mockUseAuthorized.mockReturnValue({ ...baseAuth, userRole: "Internal User" }); + renderWithProviders(); + + await screen.findByText("test-pending-guardrail"); + + expect(screen.queryByRole("button", { name: /approve/i })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /reject/i })).not.toBeInTheDocument(); + }); + + it("hides Approve and Reject buttons for an Admin Viewer, whom the backend rejects with 403", async () => { + mockUseAuthorized.mockReturnValue({ ...baseAuth, userRole: "Admin Viewer" }); + renderWithProviders(); + + await screen.findByText("test-pending-guardrail"); + + expect(screen.queryByRole("button", { name: /approve/i })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /reject/i })).not.toBeInTheDocument(); + }); + + it("shows Approve and Reject buttons for an admin on a pending submission", async () => { + mockUseAuthorized.mockReturnValue({ ...baseAuth, userRole: "Admin" }); + renderWithProviders(); + + await screen.findByText("test-pending-guardrail"); + + expect(screen.getByRole("button", { name: /approve/i })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /reject/i })).toBeInTheDocument(); + }); + + it("hides Approve and Reject buttons when userRole is undefined (defaults to non-admin)", async () => { + mockUseAuthorized.mockReturnValue({ ...baseAuth, userRole: undefined }); + renderWithProviders(); + + await screen.findByText("test-pending-guardrail"); + + expect(screen.queryByRole("button", { name: /approve/i })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /reject/i })).not.toBeInTheDocument(); + }); + + it("disables all admin-only write controls for a non-admin, including the detail panel", async () => { + mockUseAuthorized.mockReturnValue({ ...baseAuth, userRole: "Internal User" }); + renderWithProviders(); + + await screen.findByText("test-pending-guardrail"); + expect(screen.getByRole("switch")).toBeDisabled(); + + fireEvent.click(screen.getByRole("button", { name: "Review" })); + await screen.findByText("Forward LiteLLM API Key"); + + expect(screen.queryByRole("button", { name: /approve/i })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /reject/i })).not.toBeInTheDocument(); + screen.getAllByRole("switch").forEach((toggle) => expect(toggle).toBeDisabled()); + expect(screen.queryByRole("button", { name: "Add" })).not.toBeInTheDocument(); + expect(screen.queryByLabelText(/^Remove/)).not.toBeInTheDocument(); + expect(screen.queryByPlaceholderText("e.g. x-request-id")).not.toBeInTheDocument(); + }); + + it("keeps all write controls enabled for an admin in the detail panel", async () => { + mockUseAuthorized.mockReturnValue({ ...baseAuth, userRole: "Admin" }); + renderWithProviders(); + + await screen.findByText("test-pending-guardrail"); + + fireEvent.click(screen.getByRole("button", { name: "Review" })); + await screen.findByText("Forward LiteLLM API Key"); + + expect(screen.getAllByRole("button", { name: /approve/i }).length).toBeGreaterThanOrEqual(2); + screen.getAllByRole("switch").forEach((toggle) => expect(toggle).toBeEnabled()); + expect(screen.getAllByRole("button", { name: "Add" })).toHaveLength(2); + expect(screen.getByLabelText("Remove X-API-Key")).toBeInTheDocument(); + expect(screen.getByLabelText("Remove x-request-id")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx index 4217a765732..496e1129371 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx @@ -27,6 +27,8 @@ import { import NotificationsManager from "@/components/molecules/notifications_manager"; import TeamDropdown from "@/components/common_components/team_dropdown"; import { useRegisterGuardrail } from "@/app/(dashboard)/hooks/guardrails/useRegisterGuardrail"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { isProxyAdminRole } from "@/utils/roles"; type GuardrailStatus = "active" | "pending" | "rejected"; @@ -188,16 +190,25 @@ function StatCard({ label, value, color }: { label: string; value: number; color ); } -function Toggle({ enabled, onToggle }: { enabled: boolean; onToggle: () => void }) { +function Toggle({ + enabled, + onToggle, + disabled = false, +}: { + enabled: boolean; + onToggle: () => void; + disabled?: boolean; +}) { return ( - {g.status === "pending" && ( + {isAdmin && g.status === "pending" && ( <>
- +

When enabled, the caller's LiteLLM API key is forwarded as an{" "} @@ -456,28 +471,63 @@ function DetailPanel({ {h.key}: {h.value} - + {isAdmin && ( + + )} ))} )} -

- setNewStaticHeaderKey(e.target.value)} - placeholder="Header name (e.g. X-API-Key)" - className="flex-1 min-w-0 text-xs font-mono border border-gray-200 rounded-sm px-2 py-1.5 text-gray-700 placeholder-gray-400 focus:outline-hidden focus:ring-1 focus:ring-blue-500" - onKeyDown={(e) => { - if (e.key === "Enter") { - e.preventDefault(); + {isAdmin && ( +
+ setNewStaticHeaderKey(e.target.value)} + placeholder="Header name (e.g. X-API-Key)" + className="flex-1 min-w-0 text-xs font-mono border border-gray-200 rounded-sm px-2 py-1.5 text-gray-700 placeholder-gray-400 focus:outline-hidden focus:ring-1 focus:ring-blue-500" + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + const key = newStaticHeaderKey.trim(); + const value = newStaticHeaderValue.trim(); + if (key && !g.customHeaders.some((h) => h.key.toLowerCase() === key.toLowerCase())) { + onUpdateCustomHeaders([...g.customHeaders, { key, value }]); + setNewStaticHeaderKey(""); + setNewStaticHeaderValue(""); + } + } + }} + /> + setNewStaticHeaderValue(e.target.value)} + placeholder="Value" + className="flex-1 min-w-0 text-xs font-mono border border-gray-200 rounded-sm px-2 py-1.5 text-gray-700 placeholder-gray-400 focus:outline-hidden focus:ring-1 focus:ring-blue-500" + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + const key = newStaticHeaderKey.trim(); + const value = newStaticHeaderValue.trim(); + if (key && !g.customHeaders.some((h) => h.key.toLowerCase() === key.toLowerCase())) { + onUpdateCustomHeaders([...g.customHeaders, { key, value }]); + setNewStaticHeaderKey(""); + setNewStaticHeaderValue(""); + } + } + }} + /> + -
+ }} + className="text-xs font-medium text-blue-600 hover:text-blue-700 border border-blue-200 bg-blue-50 hover:bg-blue-100 px-2 py-1.5 rounded-sm transition-colors shrink-0" + > + Add + +
+ )}
@@ -546,50 +565,54 @@ function DetailPanel({ className="flex items-center justify-between gap-2 text-xs font-mono bg-gray-50 border border-gray-200 rounded-sm px-2 py-1.5" > {name} - + {isAdmin && ( + + )} ))} )} -
- setNewExtraHeader(e.target.value)} - placeholder="e.g. x-request-id" - className="flex-1 min-w-0 text-xs font-mono border border-gray-200 rounded-sm px-2 py-1.5 text-gray-700 placeholder-gray-400 focus:outline-hidden focus:ring-1 focus:ring-blue-500" - onKeyDown={(e) => { - if (e.key === "Enter") { - e.preventDefault(); + {isAdmin && ( +
+ setNewExtraHeader(e.target.value)} + placeholder="e.g. x-request-id" + className="flex-1 min-w-0 text-xs font-mono border border-gray-200 rounded-sm px-2 py-1.5 text-gray-700 placeholder-gray-400 focus:outline-hidden focus:ring-1 focus:ring-blue-500" + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + const name = newExtraHeader.trim().toLowerCase(); + if (name && !g.extraHeaders.map((h) => h.toLowerCase()).includes(name)) { + onUpdateExtraHeaders([...g.extraHeaders, name]); + setNewExtraHeader(""); + } + } + }} + /> + -
+ }} + className="text-xs font-medium text-blue-600 hover:text-blue-700 border border-blue-200 bg-blue-50 hover:bg-blue-100 px-2 py-1.5 rounded-sm transition-colors" + > + Add + +
+ )}
- {g.status === "pending" && ( + {isAdmin && g.status === "pending" && (
+ + ); +}; + +describe("MetadataKeyValueFields", () => { + it("renders one row per existing pair", () => { + render( + , + ); + + const keyInputs = screen.getAllByPlaceholderText("Key"); + const valueInputs = screen.getAllByPlaceholderText("Value"); + expect(keyInputs.map((input) => (input as HTMLInputElement).value)).toEqual(["department", "tier"]); + expect(valueInputs.map((input) => (input as HTMLInputElement).value)).toEqual(["research", "3"]); + }); + + it("adds a row and submits the entered pair", async () => { + const user = userEvent.setup(); + const onFinish = vi.fn(); + render(); + + await user.click(screen.getByRole("button", { name: /add key-value pair/i })); + await user.type(screen.getByPlaceholderText("Key"), "cost_center"); + await user.type(screen.getByPlaceholderText("Value"), "eng-1"); + await user.click(screen.getByRole("button", { name: "Save" })); + + await waitFor(() => { + expect(onFinish).toHaveBeenCalledWith({ metadata: [{ key: "cost_center", value: "eng-1" }] }); + }); + }); + + it("removes a row when its remove icon is clicked", async () => { + const user = userEvent.setup(); + const onFinish = vi.fn(); + render( + , + ); + + await user.click(screen.getAllByLabelText("Remove key-value pair")[0]); + await user.click(screen.getByRole("button", { name: "Save" })); + + await waitFor(() => { + expect(onFinish).toHaveBeenCalledWith({ metadata: [{ key: "tier", value: "3" }] }); + }); + }); + + it("blocks submission on duplicate keys", async () => { + const user = userEvent.setup(); + const onFinish = vi.fn(); + render( + , + ); + + await user.click(screen.getByRole("button", { name: "Save" })); + + await waitFor(() => { + expect(screen.getAllByText("Duplicate key").length).toBeGreaterThan(0); + }); + expect(onFinish).not.toHaveBeenCalled(); + }); + + it("blocks submission when a row is missing its key", async () => { + const user = userEvent.setup(); + const onFinish = vi.fn(); + render(); + + await user.click(screen.getByRole("button", { name: /add key-value pair/i })); + await user.type(screen.getByPlaceholderText("Value"), "orphan"); + await user.click(screen.getByRole("button", { name: "Save" })); + + await waitFor(() => { + expect(screen.getByText("Missing key")).toBeInTheDocument(); + }); + expect(onFinish).not.toHaveBeenCalled(); + }); +}); + +describe("MetadataKeyValueFields with a declared schema", () => { + const schema: TeamMetadataField[] = [ + { key: "cost_center", label: "Cost Center" }, + { key: "app_name", label: "Application Name" }, + ]; + + it("should prepopulate one ordinary editable pair row per declared key", async () => { + render(); + + await waitFor(() => { + expect(screen.getAllByPlaceholderText("Key").map((input) => (input as HTMLInputElement).value)).toEqual([ + "cost_center", + "app_name", + ]); + }); + screen.getAllByPlaceholderText("Key").forEach((input) => expect(input).toBeEnabled()); + expect(screen.getAllByLabelText("Remove key-value pair")).toHaveLength(2); + }); + + it("should submit a prepopulated key with its typed value", async () => { + const user = userEvent.setup(); + const onFinish = vi.fn(); + render(); + + await user.type(await screen.findByPlaceholderText("Value"), "CC-1001"); + await user.click(screen.getByRole("button", { name: "Save" })); + + await waitFor(() => { + expect(onFinish).toHaveBeenCalledWith({ metadata: [{ key: "cost_center", value: "CC-1001" }] }); + }); + }); + + it("should not add a second row for keys already present in the form", async () => { + render( + , + ); + + await waitFor(() => { + expect(screen.getAllByPlaceholderText("Key").map((input) => (input as HTMLInputElement).value)).toEqual([ + "cost_center", + "app_name", + ]); + }); + expect(screen.getAllByPlaceholderText("Value").map((input) => (input as HTMLInputElement).value)).toEqual([ + "CC-1001", + "", + ]); + }); + + it("should let the user remove a prepopulated row", async () => { + const user = userEvent.setup(); + render(); + + await screen.findAllByPlaceholderText("Key"); + await user.click(screen.getAllByLabelText("Remove key-value pair")[0]); + + await waitFor(() => { + expect(screen.getAllByPlaceholderText("Key").map((input) => (input as HTMLInputElement).value)).toEqual([ + "app_name", + ]); + }); + }); + + it("should show a skeleton instead of the editor while the schema is loading", () => { + render(); + + expect(screen.getByTestId("metadata-schema-skeleton")).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /add key-value pair/i })).not.toBeInTheDocument(); + }); + + it("should seed rows when the schema arrives after an initial loading state", async () => { + const onFinish = vi.fn(); + const { rerender } = render(); + + rerender(); + + await waitFor(() => { + expect(screen.getAllByPlaceholderText("Key").map((input) => (input as HTMLInputElement).value)).toEqual([ + "cost_center", + "app_name", + ]); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/common_components/MetadataKeyValueFields.tsx b/ui/litellm-dashboard/src/components/common_components/MetadataKeyValueFields.tsx new file mode 100644 index 00000000000..da085f95ad8 --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/MetadataKeyValueFields.tsx @@ -0,0 +1,135 @@ +import { MinusCircleOutlined, PlusOutlined } from "@ant-design/icons"; +import { Button, Form, FormInstance, Input, Skeleton, Space } from "antd"; +import React, { useEffect, useRef } from "react"; + +import { TeamMetadataField } from "@/app/(dashboard)/hooks/teams/useTeamMetadataSchema"; + +export interface MetadataPair { + key: string; + value: string; +} + +function formatMetadataValue(value: unknown): string { + if (typeof value !== "string") { + return JSON.stringify(value) ?? ""; + } + try { + JSON.parse(value); + return JSON.stringify(value); + } catch { + return value; + } +} + +function parseMetadataValue(raw: string): unknown { + try { + return JSON.parse(raw); + } catch { + return raw; + } +} + +export function metadataObjectToPairs( + metadata: Record | null | undefined, + excludedKeys: ReadonlySet = new Set(), +): MetadataPair[] { + return Object.entries(metadata ?? {}) + .filter(([key]) => !excludedKeys.has(key)) + .map(([key, value]) => ({ key, value: formatMetadataValue(value) })); +} + +export function metadataPairsToObject( + pairs: readonly (Partial | undefined)[] | undefined, +): Record { + return Object.fromEntries( + (pairs ?? []) + .filter((pair): pair is Partial & { key: string } => Boolean(pair?.key)) + .map((pair) => [pair.key, parseMetadataValue(pair.value ?? "")]), + ); +} + +interface MetadataKeyValueFieldsProps { + form: FormInstance; + name?: string; + schemaFields?: readonly TeamMetadataField[]; + schemaLoading?: boolean; +} + +const MetadataKeyValueFields: React.FC = ({ + form, + name = "metadata", + schemaFields = [], + schemaLoading = false, +}) => { + const seededRef = useRef(false); + + useEffect(() => { + if (seededRef.current || schemaLoading || schemaFields.length === 0) return; + seededRef.current = true; + const pairs: (Partial | undefined)[] = form.getFieldValue(name) ?? []; + if (!Array.isArray(pairs)) return; + const existingKeys = new Set(pairs.map((pair) => pair?.key).filter(Boolean)); + const seeded = schemaFields + .filter((field) => !existingKeys.has(field.key)) + .map((field) => ({ key: field.key, value: "" })); + if (seeded.length > 0) { + form.setFieldValue(name, [...pairs, ...seeded]); + } + }, [form, name, schemaFields, schemaLoading]); + + if (schemaLoading) { + return ( +
+ +
+ ); + } + + return ( + + {(fields, { add, remove }) => ( + <> + {fields.map(({ key, name: fieldName, ...restField }) => ( + + { + if (!value) return Promise.resolve(); + const all: (Partial | undefined)[] = form.getFieldValue(name) ?? []; + const dupes = all.filter((entry) => entry?.key === value); + if (dupes.length > 1) { + return Promise.reject(new Error("Duplicate key")); + } + return Promise.resolve(); + }, + }, + ]} + > + + + + + + remove(fieldName)} + style={{ color: "#ef4444" }} + /> + + ))} + + + + + )} + + ); +}; + +export default MetadataKeyValueFields; diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 03cf0e9583c..5a2d33ee4bd 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -38,7 +38,7 @@ import type { CoordinationRedisTestResponse, } from "@/app/(dashboard)/caching/_components/coordination_redis_settings/types"; import { MCP_TOOLS_PREVIEW_FORBIDDEN_MESSAGE } from "./mcp_tools/constants"; -import { createApiClient, deriveErrorMessage } from "@/lib/http/client"; +import { createApiClient, deriveErrorMessage, unwrapProxyErrorMessage } from "@/lib/http/client"; import { resolveApiBase } from "@/lib/http/resolveApiBase"; import { registerAuthHeaderNameGetter, @@ -2643,7 +2643,7 @@ export const teamUpdateCall = async ( const errorData = await response.text(); handleError(errorData); console.error("Error response from the server:", errorData); - NotificationsManager.fromBackend("Failed to update team settings: " + errorData); + NotificationsManager.fromBackend("Failed to update team settings: " + unwrapProxyErrorMessage(errorData)); throw new Error(errorData); } const data = (await response.json()) as { data: Team; team_id: string }; diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx index 712cff80649..513719a2ad9 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx @@ -1,3 +1,4 @@ +import { useTeamMetadataSchema } from "@/app/(dashboard)/hooks/teams/useTeamMetadataSchema"; import * as networking from "@/components/networking"; import { screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; @@ -26,6 +27,10 @@ vi.mock("@/components/utils/dataUtils", () => ({ formatNumberWithCommas: vi.fn((value: number) => value.toLocaleString()), })); +vi.mock("@/app/(dashboard)/hooks/teams/useTeamMetadataSchema", () => ({ + useTeamMetadataSchema: vi.fn(() => ({ data: [], isLoading: false })), +})); + vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({ useAllProxyModels: vi.fn(), })); @@ -220,6 +225,7 @@ describe("TeamInfoView", () => { isFetching: false, refetch: vi.fn(), } as any); + vi.mocked(useTeamMetadataSchema).mockReturnValue({ data: [], isLoading: false } as any); vi.mocked(networking.getGuardrailsList).mockResolvedValue({ guardrails: [] }); vi.mocked(networking.getPoliciesList).mockResolvedValue({ policies: [] }); @@ -893,6 +899,137 @@ describe("TeamInfoView", () => { }); }); + describe("metadata key-value editing", () => { + const openSettingsEditor = async (user: ReturnType) => { + await waitFor(() => { + const teamNameElements = screen.queryAllByText("Test Team"); + expect(teamNameElements.length).toBeGreaterThan(0); + }); + + await user.click(screen.getByRole("tab", { name: "Settings" })); + + await waitFor(() => { + expect(screen.getByRole("button", { name: /edit settings/i })).toBeInTheDocument(); + }); + + await user.click(screen.getByRole("button", { name: /edit settings/i })); + + await waitFor(() => { + expect(screen.getByLabelText("Team Name")).toBeInTheDocument(); + }); + }; + + it("prefills pairs from team metadata, hides UI-managed keys, and round-trips typed values on save", async () => { + const user = userEvent.setup({ delay: null }); + vi.mocked(networking.teamInfoCall).mockResolvedValue( + createMockTeamData({ + metadata: { + department: "research", + tier: 3, + beta: true, + config: { region: "us" }, + logging: [{ callback_name: "langfuse", callback_type: "success", callback_vars: {} }], + guardrails: ["g1"], + disable_global_guardrails: false, + model_tpm_limit: { "gpt-4": 100 }, + }, + models: ["gpt-4"], + }), + ); + vi.mocked(networking.teamUpdateCall).mockResolvedValue({ data: {}, team_id: "123" } as any); + + renderWithProviders(); + await openSettingsEditor(user); + + const keyValues = screen.getAllByPlaceholderText("Key").map((input) => (input as HTMLInputElement).value); + expect(keyValues).toEqual(["department", "tier", "beta", "config"]); + const valueValues = screen.getAllByPlaceholderText("Value").map((input) => (input as HTMLInputElement).value); + expect(valueValues).toEqual(["research", "3", "true", '{"region":"us"}']); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => { + expect(networking.teamUpdateCall).toHaveBeenCalled(); + }); + + const updateArg = vi.mocked(networking.teamUpdateCall).mock.calls[0][1]; + expect(updateArg.metadata).toMatchObject({ + department: "research", + tier: 3, + beta: true, + config: { region: "us" }, + logging: [{ callback_name: "langfuse", callback_type: "success", callback_vars: {} }], + }); + expect(updateArg.metadata).not.toHaveProperty("model_tpm_limit"); + expect(updateArg.model_tpm_limit).toEqual({ "gpt-4": 100 }); + }); + + it("includes a newly added pair in the team update", async () => { + const user = userEvent.setup({ delay: null }); + vi.mocked(networking.teamInfoCall).mockResolvedValue(createMockTeamData({ models: ["gpt-4"] })); + vi.mocked(networking.teamUpdateCall).mockResolvedValue({ data: {}, team_id: "123" } as any); + + renderWithProviders(); + await openSettingsEditor(user); + + await user.click(screen.getByRole("button", { name: /add key-value pair/i })); + await user.type(screen.getByPlaceholderText("Key"), "cost_center"); + await user.type(screen.getByPlaceholderText("Value"), "eng-1"); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => { + expect(networking.teamUpdateCall).toHaveBeenCalled(); + }); + + expect(vi.mocked(networking.teamUpdateCall).mock.calls[0][1].metadata).toMatchObject({ cost_center: "eng-1" }); + }); + + it("should keep declared keys as ordinary prefilled rows and submit the edited value", async () => { + const user = userEvent.setup({ delay: null }); + vi.mocked(useTeamMetadataSchema).mockReturnValue({ + data: [ + { key: "cost_center", label: "Cost Center" }, + { key: "app_name", label: "Application Name" }, + ], + isLoading: false, + } as any); + vi.mocked(networking.teamInfoCall).mockResolvedValue( + createMockTeamData({ + metadata: { cost_center: "CC-OLD", department: "research" }, + models: ["gpt-4"], + }), + ); + vi.mocked(networking.teamUpdateCall).mockResolvedValue({ data: {}, team_id: "123" } as any); + + renderWithProviders(); + await openSettingsEditor(user); + + await waitFor(() => { + expect(screen.getAllByPlaceholderText("Key").map((input) => (input as HTMLInputElement).value)).toEqual([ + "cost_center", + "department", + "app_name", + ]); + }); + expect(screen.getAllByPlaceholderText("Value")[0]).toHaveValue("CC-OLD"); + + await user.clear(screen.getAllByPlaceholderText("Value")[0]); + await user.type(screen.getAllByPlaceholderText("Value")[0], "CC-NEW"); + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => { + expect(networking.teamUpdateCall).toHaveBeenCalled(); + }); + + expect(vi.mocked(networking.teamUpdateCall).mock.calls[0][1].metadata).toMatchObject({ + cost_center: "CC-NEW", + department: "research", + app_name: "", + }); + }); + }); + describe("model aliases", () => { const openSettingsEditor = async (user: ReturnType) => { await waitFor(() => { diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index 34570043f52..bbe5dc05a88 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -35,6 +35,11 @@ import { CheckIcon, CopyIcon } from "lucide-react"; import React, { useEffect, useMemo, useState } from "react"; import { copyToClipboard as utilCopyToClipboard } from "../../utils/dataUtils"; import AccessGroupSelector from "../common_components/AccessGroupSelector"; +import MetadataKeyValueFields, { + metadataObjectToPairs, + metadataPairsToObject, +} from "../common_components/MetadataKeyValueFields"; +import { useTeamMetadataSchema } from "@/app/(dashboard)/hooks/teams/useTeamMetadataSchema"; import ModelAliasManager from "../common_components/ModelAliasManager"; import AgentSelector from "../agent_management/AgentSelector"; import DeleteResourceModal from "../common_components/DeleteResourceModal"; @@ -66,6 +71,18 @@ import { import TeamMembersComponent from "./TeamMemberTab"; import { TeamVirtualKeysTable } from "./TeamVirtualKeysTable"; +const UI_MANAGED_METADATA_KEYS: ReadonlySet = new Set([ + "logging", + "secret_manager_settings", + "soft_budget_alerting_emails", + "model_tpm_limit", + "model_rpm_limit", + "allowed_passthrough_routes", + "guardrails", + "opted_out_global_guardrails", + "disable_global_guardrails", +]); + export interface TeamMembership { user_id: string; team_id: string; @@ -203,6 +220,7 @@ const TeamInfoView: React.FC = ({ const [organization, setOrganization] = useState(null); const { userRole, userId } = useAuthorized(); const { data: userOrganizations = [] } = useOrganizations(); + const { data: teamMetadataSchemaFields = [], isLoading: isTeamMetadataSchemaLoading } = useTeamMetadataSchema(); const queryClient = useQueryClient(); // Check if user is org admin for this team's organization @@ -461,16 +479,7 @@ const TeamInfoView: React.FC = ({ if (!accessToken) return; setIsTeamSaving(true); - let parsedMetadata = {}; - try { - const rawMetadata = values.metadata ? JSON.parse(values.metadata) : {}; - // Exclude soft_budget_alerting_emails from parsed metadata since it's handled separately - const { soft_budget_alerting_emails, ...rest } = rawMetadata; - parsedMetadata = rest; - } catch (e) { - NotificationsManager.fromBackend("Invalid JSON in metadata field"); - return; - } + const parsedMetadata = metadataPairsToObject(values.metadata); let secretManagerSettings: Record | undefined; if (typeof values.secret_manager_settings === "string") { @@ -980,21 +989,7 @@ const TeamInfoView: React.FC = ({ soft_budget_alerting_emails: Array.isArray(info.metadata?.soft_budget_alerting_emails) ? info.metadata.soft_budget_alerting_emails.join(", ") : "", - metadata: info.metadata - ? JSON.stringify( - (({ - logging, - secret_manager_settings, - soft_budget_alerting_emails, - model_tpm_limit, - model_rpm_limit, - allowed_passthrough_routes, - ...rest - }) => rest)(info.metadata), - null, - 2, - ) - : "", + metadata: metadataObjectToPairs(info.metadata, UI_MANAGED_METADATA_KEYS), logging_settings: info.metadata?.logging || [], secret_manager_settings: info.metadata?.secret_manager_settings ? JSON.stringify(info.metadata.secret_manager_settings, null, 2) @@ -1170,6 +1165,17 @@ const TeamInfoView: React.FC = ({ + + + + = ({ /> - - - -
diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index f7cc9a1deae..8879844c24a 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -1,4 +1,4 @@ -import { renderWithProviders, screen, waitFor } from "../../../tests/test-utils"; +import { renderWithProviders, screen, waitFor, within } from "../../../tests/test-utils"; import userEvent from "@testing-library/user-event"; import { vi } from "vitest"; import AddAutoRouterTab from "./add_auto_router_tab"; @@ -52,14 +52,22 @@ describe("AddAutoRouterTab", () => { vi.clearAllMocks(); }); - it("flags every mandatory field when Add Auto Router is clicked with nothing filled", async () => { + // Nothing is filled in, so there is nothing to submit. The button reports that itself instead of + // accepting a click and answering with a toast. + it("offers no submit at all until every tier has a model", async () => { + renderWithProviders(); + + expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled(); + }); + + it("still flags the router name once the config no longer blocks the submit", async () => { const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); renderWithProviders(); await user.click(screen.getByRole("button", { name: /add auto router/i })); expect(await screen.findByText("Auto router name is required")).toBeInTheDocument(); - expect(screen.getAllByText("This tier is required")).toHaveLength(4); expect(NotificationManager.fromBackend).toHaveBeenCalledWith("Please enter an Auto Router Name"); }); @@ -97,6 +105,83 @@ describe("AddAutoRouterTab", () => { expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0]).toMatchObject({ team_id: "team-1" }); }); + // LIT-5133: "Add keyword rule" seeds a row with no keywords, and the semantic toggle that used + // to be the only thing checking them is off by default. The row was dropped on the way to the + // payload, so the create succeeded and the caller's rule was gone with nothing said about it. + it("takes the submit away while a keyword rule is left empty", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router"); + await user.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + await user.click(screen.getByRole("button", { name: /add keyword rule/i })); + + expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled(); + // The row says so on its own; there is no failed submit left to surface it. + expect(await screen.findByText("At least one keyword is required")).toBeInTheDocument(); + expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled(); + }); + + it("gives the submit back once that keyword rule is filled", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router"); + await user.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + await user.click(screen.getByRole("button", { name: /add keyword rule/i })); + expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled(); + + await user.type( + within(screen.getByText("Keywords 1").closest("div") as HTMLElement).getByRole("combobox"), + "invoice{enter}", + ); + + expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled(); + expect(screen.queryByText("At least one keyword is required")).not.toBeInTheDocument(); + }); + + it("marks only the offending keyword row, leaving a filled one alone", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router"); + await user.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + await user.click(screen.getByRole("button", { name: /add keyword rule/i })); + await user.type( + within(screen.getByText("Keywords 1").closest("div") as HTMLElement).getByRole("combobox"), + "invoice{enter}", + ); + await user.click(screen.getByRole("button", { name: /add keyword rule/i })); + + expect(await screen.findAllByText("At least one keyword is required")).toHaveLength(1); + expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled(); + }); + + it("creates the router once that keyword rule is filled in", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router"); + await user.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + await user.click(screen.getByRole("button", { name: /add keyword rule/i })); + const keywordsField = screen.getByText("Keywords 1").closest("div") as HTMLElement; + await user.type(within(keywordsField).getByRole("combobox"), "invoice{enter}"); + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0]).toMatchObject({ + complexity_router_config: { keyword_tier_rules: [{ keywords: ["invoice"], tier: "COMPLEX" }] }, + }); + }); + it("blocks the submit when a team admin has not picked a team", async () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index ea75bd8e283..ae90d42ba8a 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -18,6 +18,7 @@ import { DEFAULT_ESCALATION_KEYWORDS } from "./EscalationKeywords"; import { DEFAULT_MATCH_THRESHOLD } from "./SemanticKeywordMatching"; import { buildComplexityRouterConfig, + getKeywordTierRulesError, getMissingTiersError, getSemanticConfigError, } from "./build_complexity_router_config"; @@ -95,6 +96,11 @@ const AddAutoRouterTab: React.FC = ({ label: model_group, })); + // Why the submit is unavailable, or null when it is available. The button reads this to disable + // itself and to say what is missing, so the two can never give different answers. + const submitBlockedReason = + getMissingTiersError(complexityRouterConfig.tiers) ?? getKeywordTierRulesError(keywordTierRules); + const submitRecommendedRouter = (name: string) => { const { tiers, @@ -124,6 +130,13 @@ const AddAutoRouterTab: React.FC = ({ return; } + const keywordRulesError = getKeywordTierRulesError(keywordTierRules); + if (keywordRulesError) { + setShowValidationErrors(true); + NotificationManager.fromBackend(keywordRulesError); + return; + } + const semanticError = getSemanticConfigError({ semanticMatchingEnabled, embeddingModel, keywordTierRules }); if (semanticError) { setShowValidationErrors(true); @@ -310,14 +323,17 @@ const AddAutoRouterTab: React.FC = ({ Test Connection } - + + +
diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 9d784b57903..4cbe54ad4a6 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -1,5 +1,6 @@ import { buildComplexityRouterConfig, + getKeywordTierRulesError, getMissingTiersError, getSemanticConfigError, BuildComplexityRouterConfigParams, @@ -183,7 +184,7 @@ describe("buildComplexityRouterConfig", () => { expect(config.keyword_tier_rules).toBeUndefined(); }); - it("trims keywords and drops rules left empty, so unfilled rows never 400 the backend", () => { + it("trims keywords but keeps rules left empty, so a dropped row can never pass for a saved one", () => { const params: BuildComplexityRouterConfigParams = { ...baseParams, keywordTierRules: [ @@ -193,17 +194,13 @@ describe("buildComplexityRouterConfig", () => { ], }; const config = buildComplexityRouterConfig(params); - // r1 keeps only its real keyword (trimmed); r2 and r3 are dropped entirely. - expect(config.keyword_tier_rules).toEqual([{ keywords: ["deploy to k8s"], tier: "REASONING" }]); - }); - - it("omits keyword_tier_rules entirely when every rule is empty", () => { - const params: BuildComplexityRouterConfigParams = { - ...baseParams, - keywordTierRules: [{ id: "r1", keywords: ["", " "], tier: "COMPLEX" }], - }; - const config = buildComplexityRouterConfig(params); - expect(config.keyword_tier_rules).toBeUndefined(); + // getKeywordTierRulesError blocks this submit; r2 and r3 survive here so the backend rejects + // them loudly rather than the caller's rows vanishing on a successful save. + expect(config.keyword_tier_rules).toEqual([ + { keywords: ["deploy to k8s"], tier: "REASONING" }, + { keywords: [], tier: "COMPLEX" }, + { keywords: [], tier: "SIMPLE" }, + ]); }); it("omits adaptive fields when adaptive is disabled even if weights linger in state", () => { @@ -318,17 +315,6 @@ describe("getSemanticConfigError", () => { ).toMatch(/keyword tier rule/i); }); - it("errors when a rule has no non-empty keywords", () => { - const emptyRule = { id: "r2", keywords: ["", " "], tier: "SIMPLE" as const }; - expect( - getSemanticConfigError({ - semanticMatchingEnabled: true, - embeddingModel: "voyage-3-5", - keywordTierRules: [emptyRule], - }), - ).toMatch(/at least one keyword/i); - }); - it("returns null when enabled with both an embedding model and rules", () => { expect( getSemanticConfigError({ semanticMatchingEnabled: true, embeddingModel: "voyage-3-5", keywordTierRules: [rule] }), @@ -336,6 +322,52 @@ describe("getSemanticConfigError", () => { }); }); +describe("getKeywordTierRulesError", () => { + it("returns null when every rule carries a keyword", () => { + expect( + getKeywordTierRulesError([ + { id: "r1", keywords: ["invoice"], tier: "MEDIUM" }, + { id: "r2", keywords: ["deploy to k8s"], tier: "REASONING" }, + ]), + ).toBeNull(); + }); + + it("returns null when there are no rules at all, since the section is optional", () => { + expect(getKeywordTierRulesError([])).toBeNull(); + }); + + // The whole point of the ticket: the semantic toggle is off by default, and an unfilled row + // used to be discarded silently on an otherwise successful create. + it("rejects a row left empty while semantic matching is off", () => { + expect(getKeywordTierRulesError([{ id: "r1", keywords: [], tier: "COMPLEX" }])).toBe( + "Add at least one keyword to keyword rule(s): 1", + ); + }); + + it.each([ + ["whitespace only", [" "]], + ["blank strings, as an unfilled row between filled ones leaves behind", ["", " ", ""]], + ])("treats %s as empty rather than as a keyword", (_label, keywords) => { + expect(getKeywordTierRulesError([{ id: "r1", keywords, tier: "SIMPLE" }])).toMatch(/keyword rule\(s\): 1/); + }); + + // Row numbers have to survive rules that are fine, or the message points at the wrong input. + it("names each offending row by its position among all rules", () => { + expect( + getKeywordTierRulesError([ + { id: "r1", keywords: ["invoice"], tier: "MEDIUM" }, + { id: "r2", keywords: [], tier: "COMPLEX" }, + { id: "r3", keywords: ["billing"], tier: "SIMPLE" }, + { id: "r4", keywords: [" "], tier: "REASONING" }, + ]), + ).toBe("Add at least one keyword to keyword rule(s): 2, 4"); + }); + + it("keeps a keyword whose surrounding whitespace is the only thing trimmed", () => { + expect(getKeywordTierRulesError([{ id: "r1", keywords: [" invoice "], tier: "MEDIUM" }])).toBeNull(); + }); +}); + describe("buildComplexityRouterConfig assistant turns", () => { const llmParams: BuildComplexityRouterConfigParams = { ...baseParams, diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index cd6c697b377..dcec58479a6 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -1,5 +1,5 @@ import { KeywordTierRule } from "./KeywordTierRules"; -import { serializeKeywordTierRules } from "./complexity_router_keywords"; +import { emptyKeywordTierRuleIndexes, serializeKeywordTierRules } from "./complexity_router_keywords"; import { AdaptiveEligible, AdaptiveRouterWeights, @@ -58,6 +58,12 @@ export const getMissingTiersError = (tiers: ComplexityTiers): string | null => { return `Select a model for the following tier(s): ${missing.join(", ")}`; }; +export const getKeywordTierRulesError = (keywordTierRules: KeywordTierRule[]): string | null => { + const emptyRows = emptyKeywordTierRuleIndexes(keywordTierRules); + if (emptyRows.length === 0) return null; + return `Add at least one keyword to keyword rule(s): ${emptyRows.map((index) => index + 1).join(", ")}`; +}; + export const getSemanticConfigError = ({ semanticMatchingEnabled, embeddingModel, @@ -68,8 +74,6 @@ export const getSemanticConfigError = ({ if (!semanticMatchingEnabled) return null; if (!embeddingModel) return "Select an embedding model to use semantic keyword matching"; if (keywordTierRules.length === 0) return "Add at least one keyword tier rule to use semantic keyword matching"; - if (keywordTierRules.some((rule) => !rule.keywords.some((keyword) => keyword.trim()))) - return "Every keyword tier rule needs at least one keyword"; return null; }; @@ -94,7 +98,6 @@ export const buildComplexityRouterConfig = ({ returnRawModelName, }: BuildComplexityRouterConfigParams): ComplexityRouterConfigPayload => { const cleanedEscalationKeywords = escalationKeywords.map((keyword) => keyword.trim()).filter(Boolean); - // Trim keywords and drop empty ones; drop any rule left with no keywords. Clicking const cleanedKeywordTierRules = serializeKeywordTierRules(keywordTierRules); return { diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_keywords.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_keywords.ts index 9cfdaed4e23..6fe93cddae3 100644 --- a/ui/litellm-dashboard/src/components/add_model/complexity_router_keywords.ts +++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_keywords.ts @@ -19,13 +19,19 @@ const asKeywords = (value: unknown): string[] => : []; /** - * Drop the React-only id, trim keywords, and discard rules left empty. "Add keyword rule" - * seeds a row with no keywords, and the backend validator rejects those with a 400. + * Drop the React-only id and trim keywords, leaving one entry per rule. A rule left empty stays + * empty rather than disappearing, so getKeywordTierRulesError can name the row it came from. */ export const serializeKeywordTierRules = (rules: KeywordTierRule[]): StoredKeywordTierRule[] => - rules - .map((rule) => ({ keywords: asKeywords(rule.keywords).filter(Boolean), tier: rule.tier })) - .filter((rule) => rule.keywords.length > 0); + rules.map((rule) => ({ keywords: asKeywords(rule.keywords).filter(Boolean), tier: rule.tier })); + +/** + * Positions of the rules left without a keyword, as indexes into the caller's own array. The + * submit-time message and the inline error on the row both read this, so the row the message + * names is always the row that lights up. + */ +export const emptyKeywordTierRuleIndexes = (rules: KeywordTierRule[]): number[] => + serializeKeywordTierRules(rules).flatMap((rule, index) => (rule.keywords.length === 0 ? [index] : [])); export const hydrateKeywordTierRules = (value: unknown): KeywordTierRule[] => { if (!Array.isArray(value)) return []; diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index 971c833a0de..818dcd1f648 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -54,13 +54,16 @@ describe("buildUpdatedComplexityRouterConfig keyword matching", () => { expect(result.keyword_tier_rules).toEqual([{ keywords: ["chargeback"], tier: "COMPLEX" }]); }); - it("drops a rule left empty rather than shipping one the backend 400s on", () => { + // getKeywordTierRulesError blocks this save, so the builder never runs on a real edit. Keeping + // the rule here means that if a caller ever reaches it anyway, the stored rules are replaced by + // something the backend rejects out loud rather than by silence that reads as a clean save. + it("keeps a rule left empty rather than quietly dropping the caller's row", () => { const result = buildUpdatedComplexityRouterConfig(STORED, FORM_VALUE, undefined, { ...hydratedState, keywordTierRules: [{ id: "new-1", keywords: [" "], tier: "SIMPLE" }], }); - expect(result.keyword_tier_rules).toBeUndefined(); + expect(result.keyword_tier_rules).toEqual([{ keywords: [], tier: "SIMPLE" }]); }); it("removes the semantic trio when the toggle is turned off", () => { diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx index c0806befa52..3976e4c1381 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx @@ -118,6 +118,80 @@ describe("EditAutoRouterModal keyword matching", () => { await waitFor(() => expect(NotificationsManager.fromBackend).toHaveBeenCalled()); expect(modelPatchUpdateCall).not.toHaveBeenCalled(); }); + + // LIT-5133, edit side. Semantic matching is off here on purpose: it used to be the only thing + // that checked a rule for keywords, so with it on this save was already blocked and the test + // would pass without the fix. Off, the unfilled row was dropped and the save reported success. + it("blocks a save that adds a keyword rule and leaves it empty", async () => { + const user = userEvent.setup(); + renderWithProviders( + , + ); + + await screen.findByText(/Escalation Keywords/i); + fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + await user.click(screen.getByRole("button", { name: /add keyword rule/i })); + + // The modal renders the same controls as the create form, so it owes the same treatment: + // the row says what is missing and the save is not offered while it is. + expect(await screen.findByText("At least one keyword is required")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /save changes/i })).toBeDisabled(); + expect(modelPatchUpdateCall).not.toHaveBeenCalled(); + }); + + it("gives the save back once the added keyword rule is filled", async () => { + const user = userEvent.setup(); + renderWithProviders( + , + ); + + await screen.findByText(/Escalation Keywords/i); + fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + await user.click(screen.getByRole("button", { name: /add keyword rule/i })); + expect(screen.getByRole("button", { name: /save changes/i })).toBeDisabled(); + + await user.type( + within(screen.getByText("Keywords 2").closest("div") as HTMLElement).getByRole("combobox"), + "chargeback{enter}", + ); + + expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled(); + expect(screen.queryByText("At least one keyword is required")).not.toBeInTheDocument(); + }); }); describe("EditAutoRouterModal classifier context window", () => { diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index a70fc31d6fe..2c58cd70cb9 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -1,12 +1,12 @@ import React, { useEffect, useState } from "react"; -import { Modal, Form, Button, Select as AntdSelect } from "antd"; +import { Modal, Form, Button, Select as AntdSelect, Tooltip } from "antd"; import { Text, TextInput } from "@tremor/react"; import { modelAvailableCall, modelPatchUpdateCall } from "../networking"; import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; import RouterConfigBuilder from "../add_model/RouterConfigBuilder"; import { normalizeTierModels } from "../add_model/complexity_router_tiers"; import { isComplexityRouter } from "../add_model/auto_router_strategies"; -import { getSemanticConfigError } from "../add_model/build_complexity_router_config"; +import { getKeywordTierRulesError, getSemanticConfigError } from "../add_model/build_complexity_router_config"; import { KeywordTierRule } from "../add_model/KeywordTierRules"; import { DEFAULT_MATCH_THRESHOLD } from "../add_model/SemanticKeywordMatching"; import { hydrateKeywordTierRules, serializeKeywordTierRules } from "../add_model/complexity_router_keywords"; @@ -118,8 +118,8 @@ export const buildUpdatedComplexityRouterConfig = ( }), ...(value.return_raw_model_name && { return_raw_model_name: true }), ...(keywordMatching && { - // Mirrors buildComplexityRouterConfig: rules only when non-empty (the backend rejects - // an empty rule with a 400), escalation keywords always, semantic trio only when on. + // Mirrors buildComplexityRouterConfig: the key only when there is a rule to write, + // escalation keywords always, semantic trio only when on. ...(storedKeywordRules.length > 0 && { keyword_tier_rules: storedKeywordRules }), escalation_keywords: keywordMatching.escalationKeywords.map((k) => k.trim()).filter(Boolean), ...(keywordMatching.semanticMatchingEnabled && { @@ -145,6 +145,7 @@ const EditAutoRouterModal: React.FC = ({ const [modelInfo, setModelInfo] = useState([]); const [showCustomDefaultModel, setShowCustomDefaultModel] = useState(false); const [showCustomEmbeddingModel, setShowCustomEmbeddingModel] = useState(false); + const [showValidationErrors, setShowValidationErrors] = useState(false); const [routerConfig, setRouterConfig] = useState(null); const [customTechnicalKeywords, setCustomTechnicalKeywords] = useState([]); const [keywordTierRules, setKeywordTierRules] = useState([]); @@ -158,6 +159,15 @@ const EditAutoRouterModal: React.FC = ({ }); const isComplexityRouterModel = isComplexityRouter(modelData?.litellm_params); + // Mirrors the create form: the button says why it is unavailable and disables on the same + // answer. Tiers use this modal's own rule, which allows a partly filled router, so an edit that + // is legal today stays legal. + const submitBlockedReason = !isComplexityRouterModel + ? null + : (Object.values(complexityRouterConfig.tiers).every((models) => models.length === 0) + ? "Please select at least one model for a complexity tier" + : null) ?? getKeywordTierRulesError(keywordTierRules); + useEffect(() => { if (isVisible && modelData) { initializeForm(); @@ -295,24 +305,29 @@ const EditAutoRouterModal: React.FC = ({ if (isComplexityRouterModel) { const { tiers, classifier_type, classifier_llm_config } = complexityRouterConfig; if (Object.values(tiers).every((models) => models.length === 0)) { + setShowValidationErrors(true); NotificationsManager.fromBackend("Please select at least one model for a complexity tier"); return; } if (classifier_type === "llm" && !classifier_llm_config?.model) { + setShowValidationErrors(true); NotificationsManager.fromBackend("Please select a classifier model, or switch back to Heuristic"); return; } - // Same guard the create form applies (add_auto_router_tab.tsx). The backend rejects - // semantic_keyword_matching without an embedding model or keyword rules - // (complexity_router/config.py), so without this a save fails as a raw 400 instead of - // an inline message. + // Same guards the create form applies (add_auto_router_tab.tsx). The backend rejects a + // keyword rule with no keyword, and semantic_keyword_matching without an embedding model + // or keyword rules (complexity_router/config.py), so without these a save fails as a raw + // 400 instead of an inline message. + const keywordRulesError = getKeywordTierRulesError(keywordTierRules); + if (keywordRulesError) { + setShowValidationErrors(true); + NotificationsManager.fromBackend(keywordRulesError); + return; + } - // Same guard the create form applies (add_auto_router_tab.tsx). The backend rejects - // semantic_keyword_matching without an embedding model or keyword rules - // (complexity_router/config.py), so without this a save fails as a raw 400 instead of - // an inline message. const semanticError = getSemanticConfigError({ semanticMatchingEnabled, embeddingModel, keywordTierRules }); if (semanticError) { + setShowValidationErrors(true); NotificationsManager.fromBackend(semanticError); return; } @@ -410,9 +425,11 @@ const EditAutoRouterModal: React.FC = ({ , - , + + + , ]} width={1000} destroyOnHidden @@ -436,6 +453,7 @@ const EditAutoRouterModal: React.FC = ({ /* Complexity Router Configuration */
{ From 2d1f650e9a5a79a3e67b0e9bbe1d4f1717de3aee Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 3 Aug 2026 19:56:24 -0700 Subject: [PATCH 065/124] fix(guardrails/rubrik): attribute blocked requests to the caller that made them (#35734) The block event Rubrik receives sourced caller identity from model_call_details[metadata], where the enriched litellm metadata never lives; it sits under litellm_params. Every block therefore reported user_api_key_hash as an empty string, so a security block could not be traced to a key, user, or team. Read identity off the authenticated UserAPIKeyAuth the failure hook is already handed, via the same mapper the success path and the proxy spend logger use, so a block log and a success log describe their caller with an identical key set. --- basedpyright-code-budget.json | 4 +- litellm/integrations/rubrik.py | 68 +++++--- ruff-strict-budget.json | 2 +- .../test_litellm/integrations/test_rubrik.py | 157 +++++++++++++++--- type-discipline-budget.json | 2 +- 5 files changed, 188 insertions(+), 45 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index f6dd90077b1..90e0a283c63 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 29813 + "limit": 29811 }, "reportArgumentType": { "limit": 2645 @@ -24,7 +24,7 @@ "limit": 42 }, "reportExplicitAny": { - "limit": 9473 + "limit": 9471 }, "reportFunctionMemberAccess": { "limit": 11 diff --git a/litellm/integrations/rubrik.py b/litellm/integrations/rubrik.py index 4bcbe8bae37..9942776bc00 100644 --- a/litellm/integrations/rubrik.py +++ b/litellm/integrations/rubrik.py @@ -34,12 +34,14 @@ from litellm.types.utils import ( Function, GenericGuardrailAPIInputs, StandardLoggingPayload, + StandardLoggingUserAPIKeyMetadata, ) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import ( Logging as LiteLLMLoggingObj, ) + from litellm.proxy._types import UserAPIKeyAuth _WEBHOOK_PATH_RESPONSE_MODERATION = "/v1/after_completion/openai/v1" _WEBHOOK_PATH_PROMPT_MODERATION = "/v1/before_prompt/openai/v1" @@ -725,7 +727,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): self, request_data: dict, original_exception: Exception, - user_api_key_dict: Any, + user_api_key_dict: "UserAPIKeyAuth", traceback_str: str | None = None, ) -> None: """Log blocked requests signalled via ``ModifyResponseException`` @@ -755,20 +757,21 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): "Rubrik: block exception without stashed logging_obj. " f"litellm_call_id={request_data.get('litellm_call_id')}, " f"model={request_data.get('model')}, " - f"user_id={getattr(user_api_key_dict, 'user_id', None)}, " + f"user_id={user_api_key_dict.user_id}, " f"raising_guardrail=" f"{getattr(original_exception, 'guardrail_name', None)}" ) return call_id: str | None = None - await self._build_and_enqueue_block_event(logging_obj, original_exception, call_id) + await self._build_and_enqueue_block_event(logging_obj, original_exception, call_id, user_api_key_dict) async def _build_and_enqueue_block_event( self, logging_obj: "LiteLLMLoggingObj", exception: "ModifyResponseException", call_id: str | None, + user_api_key_dict: "UserAPIKeyAuth", ) -> None: try: call_details = logging_obj.model_call_details @@ -780,8 +783,8 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): # prevent. The flag dies with model_call_details when the request # completes; there's nothing to clean up. call_id = call_details.get("litellm_call_id") - payload = self._prepare_block_failure_payload(logging_obj, exception) - except (AttributeError, KeyError, TypeError) as e: + payload = self._prepare_block_failure_payload(logging_obj, exception, user_api_key_dict) + except (AttributeError, ImportError, KeyError, TypeError) as e: verbose_logger.error( f"Rubrik: failed to build blocked-tool payload for " f"litellm_call_id={call_id}: {e}. Event will NOT be logged.", @@ -801,17 +804,20 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): self, logging_obj: "LiteLLMLoggingObj", exception: "ModifyResponseException", + user_api_key_dict: "UserAPIKeyAuth", ) -> StandardLoggingPayload: """Build a failure-style payload using the exception text as response. Blocked-tool events are security-relevant and **bypass sampling**: every block is logged. - The deferred success-handler runs as a separately-scheduled task and - races with this hook, so ``standard_logging_object`` on - ``model_call_details`` may not yet be populated. If present we reuse - it; otherwise we fall back to a best-effort payload built from the - fields available at block time. + A non-streaming block always takes the fallback, and not because of a + race: registering a post_call guardrail sets ``_defer_async_logging``, + which parks the success handler that would have written + ``standard_logging_object``, and ``_flush_deferred_async_logging`` + returns early once an exception was raised. Streaming requests never set + that flag, so a streamed block can arrive with the object already + populated; the branch below covers it and wins over the fallback. For prompt blocks the LLM is never called, so ``standard_logging_object`` is never populated. The fallback therefore must carry enough fields to @@ -830,9 +836,11 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): ``acompletion()``, which hasn't run yet for a prompt block. - ``model_id``: not available before the LLM returns hidden_params; defaults to empty string. - - ``user_api_key_hash``: ``call_details["metadata"]["user_api_key"]`` -- - the hashed token written by ``add_user_information_to_request_data`` - before ``pre_call_hook`` fires. + - caller identity: ``_caller_metadata`` off the ``user_api_key_dict`` + the failure hook is handed. The enriched litellm metadata lives under + ``call_details["litellm_params"]["metadata"]``, never at the top + level, so the previous top-level read resolved to an empty string for + every block. - time fields: ``call_details["start_time"]`` reused for all three; end/completion times are meaningless for a prompt block. """ @@ -848,7 +856,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): f"for litellm_call_id={call_details.get('litellm_call_id')}; " "using best-effort fallback payload." ) - payload = self._build_fallback_payload(call_details) + payload = self._build_fallback_payload(call_details, user_api_key_dict) payload["response"] = exception_text @@ -863,8 +871,30 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): return payload # type: ignore[return-value] @staticmethod - def _build_fallback_payload(call_details: Mapping[str, Any]) -> dict[str, Any]: - _metadata: Mapping[str, Any] = call_details.get("metadata") or _EMPTY_MAPPING + def _caller_metadata(user_api_key_dict: "UserAPIKeyAuth") -> StandardLoggingUserAPIKeyMetadata: + """Identify the caller whose request was blocked. + + Uses the same mapper the success path and the proxy spend logger use, so + a block log and a success log agree on the caller key set. + + The import is deferred because ``litellm/integrations/`` is SDK-side + while the mapper lives under ``proxy/``: ``rubrik.py`` is imported during + guardrail discovery and must not pull proxy-only dependencies into its + import chain. It is unguarded because the only dispatcher of this hook, + ``ProxyLogging.post_call_failure_hook``, already imports fastapi at + module scope, so there is no path where this hook runs and the mapper is + missing. + """ + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + return LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict) + + @classmethod + def _build_fallback_payload( + cls, + call_details: Mapping[str, Any], + user_api_key_dict: "UserAPIKeyAuth", + ) -> dict[str, Any]: # Convert datetime to a Unix float so json.dumps can serialize it. # httpx's json= parameter uses stdlib json.dumps with no custom encoder. _raw_start = call_details.get("start_time") @@ -884,11 +914,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): "endTime": _start, "completionStartTime": _start, "messages": call_details.get("messages") or (), - "metadata": { - # "user_api_key" is the hashed token written by - # add_user_information_to_request_data before guardrails fire. - "user_api_key_hash": _metadata.get("user_api_key_hash") or _metadata.get("user_api_key") or "", - }, + "metadata": cls._caller_metadata(user_api_key_dict), "status": "failure", } diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index d3ef01940bb..003e09de0c2 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -24,7 +24,7 @@ "limit": 130 }, "ANN401": { - "limit": 1851 + "limit": 1850 }, "ASYNC230": { "limit": 14 diff --git a/tests/test_litellm/integrations/test_rubrik.py b/tests/test_litellm/integrations/test_rubrik.py index 7f589dc15bf..4a2ee487c65 100644 --- a/tests/test_litellm/integrations/test_rubrik.py +++ b/tests/test_litellm/integrations/test_rubrik.py @@ -17,6 +17,7 @@ from litellm.integrations.rubrik import ( RubrikLogger, _MalformedToolBlockingResponseError, ) +from litellm.proxy._types import UserAPIKeyAuth from tests.test_litellm.integrations.rubrik_test_helpers import ( make_inputs_with_tools, @@ -44,6 +45,18 @@ def handler(mock_env): return RubrikLogger() +@pytest.fixture +def user_api_key_dict(): + """The authenticated caller the proxy hands to async_post_call_failure_hook.""" + return UserAPIKeyAuth( + api_key="sk-block-attribution-test", + key_alias="rubrik-probe-key", + user_id="probe-user-1", + team_id="probe-team-1", + org_id="probe-org-1", + ) + + # -- Initialization ----------------------------------------------------------- @@ -1544,17 +1557,17 @@ class TestSuccessEventBlockedSkip: @pytest.mark.asyncio class TestPostCallFailureHook: - async def test_non_modify_exception_returns_immediately(self, handler): + async def test_non_modify_exception_returns_immediately(self, handler, user_api_key_dict): """Non-ModifyResponseException causes a no-op.""" await handler.async_post_call_failure_hook( request_data={"litellm_call_id": "test"}, original_exception=ValueError("unrelated error"), - user_api_key_dict=None, + user_api_key_dict=user_api_key_dict, ) assert len(handler.log_queue) == 0 async def test_modify_exception_without_stashed_logging_obj_emits_warning( - self, handler + self, handler, user_api_key_dict ): """ModifyResponseException with no _rubrik_logging_obj → warning, no enqueue.""" request_data = {"litellm_call_id": "test-123", "model": "gpt-4"} @@ -1568,12 +1581,12 @@ class TestPostCallFailureHook: await handler.async_post_call_failure_hook( request_data=request_data, original_exception=exc, - user_api_key_dict=None, + user_api_key_dict=user_api_key_dict, ) assert len(handler.log_queue) == 0 async def test_modify_exception_with_valid_logging_obj_enqueues_payload( - self, handler + self, handler, user_api_key_dict ): """ModifyResponseException + stashed logging_obj → builds and enqueues.""" logging_obj = Mock() @@ -1602,12 +1615,12 @@ class TestPostCallFailureHook: await handler.async_post_call_failure_hook( request_data=request_data, original_exception=exc, - user_api_key_dict=None, + user_api_key_dict=user_api_key_dict, ) assert len(handler.log_queue) == 1 assert "ModifyResponseException" in handler.log_queue[0]["response"] - async def test_logging_obj_popped_from_request_data(self, handler): + async def test_logging_obj_popped_from_request_data(self, handler, user_api_key_dict): """_rubrik_logging_obj must be popped from request_data so it is not forwarded downstream.""" logging_obj = Mock() @@ -1636,12 +1649,12 @@ class TestPostCallFailureHook: await handler.async_post_call_failure_hook( request_data=request_data, original_exception=exc, - user_api_key_dict=None, + user_api_key_dict=user_api_key_dict, ) assert "_rubrik_logging_obj" not in request_data async def test_build_and_enqueue_swallows_attribute_error_from_prepare_payload( - self, handler + self, handler, user_api_key_dict ): """When _prepare_block_failure_payload raises AttributeError/KeyError/TypeError, the error is logged and the event is silently dropped (lines 806-812).""" @@ -1657,10 +1670,10 @@ class TestPostCallFailureHook: ) # Must not raise - await handler._build_and_enqueue_block_event(logging_obj, exc, None) + await handler._build_and_enqueue_block_event(logging_obj, exc, None, user_api_key_dict) assert len(handler.log_queue) == 0 - async def test_build_and_enqueue_swallows_flush_exception(self, handler): + async def test_build_and_enqueue_swallows_flush_exception(self, handler, user_api_key_dict): """When _append_and_maybe_flush raises, the error is logged (lines 816-817).""" logging_obj = Mock() logging_obj.model_call_details = { @@ -1688,14 +1701,14 @@ class TestPostCallFailureHook: ) # Must not raise - await handler._build_and_enqueue_block_event(logging_obj, exc, None) + await handler._build_and_enqueue_block_event(logging_obj, exc, None, user_api_key_dict) # -- _prepare_block_failure_payload and _build_fallback_payload --------------- class TestPrepareBlockFailurePayload: - def test_uses_standard_logging_object_when_present(self, handler): + def test_uses_standard_logging_object_when_present(self, handler, user_api_key_dict): """When standard_logging_object is on model_call_details, it is used as base.""" logging_obj = Mock() logging_obj.model_call_details = { @@ -1716,12 +1729,37 @@ class TestPrepareBlockFailurePayload: guardrail_name="rubrik", ) - payload = handler._prepare_block_failure_payload(logging_obj, exc) + payload = handler._prepare_block_failure_payload(logging_obj, exc, user_api_key_dict) assert "ModifyResponseException: blocked" in payload["response"] assert payload["id"] == "call-slo" - def test_uses_fallback_when_standard_logging_object_absent(self, handler): + def test_standard_logging_object_identity_is_not_overwritten(self, handler, user_api_key_dict): + """A streamed block can arrive with the object populated; it keeps its own identity.""" + logging_obj = Mock() + logging_obj.model_call_details = { + "litellm_call_id": "call-slo-identity", + "model": "gpt-4", + "standard_logging_object": { + "id": "chatcmpl-original", + "model": "gpt-4", + "response": "original response", + "messages": [], + "metadata": {"user_api_key_hash": "hash-from-standard-logging-object"}, + }, + } + exc = ModifyResponseException( + message="blocked", + model="gpt-4", + request_data={}, + guardrail_name="rubrik", + ) + + payload = handler._prepare_block_failure_payload(logging_obj, exc, user_api_key_dict) + + assert payload["metadata"]["user_api_key_hash"] == "hash-from-standard-logging-object" + + def test_uses_fallback_when_standard_logging_object_absent(self, handler, user_api_key_dict): """When standard_logging_object is absent, _build_fallback_payload is used.""" from datetime import datetime @@ -1731,7 +1769,7 @@ class TestPrepareBlockFailurePayload: "model": "claude-3", "messages": [{"role": "user", "content": "question"}], "optional_params": {"temperature": 0.5}, - "metadata": {"user_api_key_hash": "hash-abc"}, + "metadata": {"headers": {"host": "127.0.0.1:4000", "user-agent": "curl/8.7.1"}}, "start_time": datetime(2024, 6, 1), } exc = ModifyResponseException( @@ -1741,16 +1779,15 @@ class TestPrepareBlockFailurePayload: guardrail_name="rubrik", ) - payload = handler._prepare_block_failure_payload(logging_obj, exc) + payload = handler._prepare_block_failure_payload(logging_obj, exc, user_api_key_dict) assert payload["id"] == "call-fallback" assert payload["model"] == "claude-3" assert payload["model_group"] == "claude-3" assert "ModifyResponseException: prompt blocked" in payload["response"] - assert payload["metadata"]["user_api_key_hash"] == "hash-abc" assert payload["status"] == "failure" - def test_fallback_payload_without_start_time(self, handler): + def test_fallback_payload_without_start_time(self, handler, user_api_key_dict): """_build_fallback_payload handles missing start_time gracefully.""" logging_obj = Mock() logging_obj.model_call_details = { @@ -1767,10 +1804,90 @@ class TestPrepareBlockFailurePayload: guardrail_name="rubrik", ) - payload = handler._prepare_block_failure_payload(logging_obj, exc) + payload = handler._prepare_block_failure_payload(logging_obj, exc, user_api_key_dict) assert payload["startTime"] is None +class TestBlockPayloadCallerAttribution: + """A block log must identify the caller that triggered it. + + The enriched litellm metadata lives under + ``model_call_details["litellm_params"]["metadata"]``, never at the top + level, so sourcing identity from ``call_details["metadata"]`` yielded an + empty string for every block. Identity comes from the authenticated + ``user_api_key_dict`` the failure hook is handed. + """ + + def _blocked_call_details(self): + return { + "litellm_call_id": "call-attr", + "model": "claude-3", + "messages": [{"role": "user", "content": "question"}], + "optional_params": {}, + "metadata": {"headers": {"host": "127.0.0.1:4000", "user-agent": "curl/8.7.1"}}, + } + + def test_fallback_payload_identifies_the_caller(self, handler, user_api_key_dict): + payload = handler._build_fallback_payload(self._blocked_call_details(), user_api_key_dict) + + metadata = payload["metadata"] + assert metadata["user_api_key_hash"] == user_api_key_dict.api_key + assert metadata["user_api_key_alias"] == "rubrik-probe-key" + assert metadata["user_api_key_user_id"] == "probe-user-1" + assert metadata["user_api_key_team_id"] == "probe-team-1" + assert metadata["user_api_key_org_id"] == "probe-org-1" + + def test_metadata_covers_the_full_caller_key_set(self, handler, user_api_key_dict): + """A block log and a success log agree on the caller key set.""" + from litellm.types.utils import StandardLoggingUserAPIKeyMetadata + + payload = handler._build_fallback_payload(self._blocked_call_details(), user_api_key_dict) + + expected = StandardLoggingUserAPIKeyMetadata.__required_keys__ | StandardLoggingUserAPIKeyMetadata.__optional_keys__ + assert set(payload["metadata"]) == set(expected) + + def test_virtual_key_is_logged_hashed(self, handler): + """A virtual key reaches the webhook as its hash, never as the raw token.""" + raw = "sk-block-attribution-test" + payload = handler._build_fallback_payload( + self._blocked_call_details(), UserAPIKeyAuth(api_key=raw) + ) + + assert payload["metadata"]["user_api_key_hash"] not in (raw, "") + + def test_request_header_metadata_is_not_used_as_identity(self, handler, user_api_key_dict): + """The pre-fix source is present and misleading; it must not win.""" + call_details = self._blocked_call_details() + call_details["metadata"]["user_api_key_hash"] = "stale-hash-from-request-metadata" + + payload = handler._build_fallback_payload(call_details, user_api_key_dict) + + assert payload["metadata"]["user_api_key_hash"] == user_api_key_dict.api_key + + async def test_enqueued_block_event_carries_attribution(self, handler, user_api_key_dict): + """End of the real hook chain: what actually lands on the Rubrik queue.""" + logging_obj = Mock() + logging_obj.model_call_details = self._blocked_call_details() + request_data = {"litellm_call_id": "call-attr", "_rubrik_logging_obj": logging_obj} + exc = ModifyResponseException( + message="prompt blocked", + model="claude-3", + request_data=request_data, + guardrail_name="rubrik", + ) + + await handler.async_post_call_failure_hook( + request_data=request_data, + original_exception=exc, + user_api_key_dict=user_api_key_dict, + ) + + assert len(handler.log_queue) == 1 + metadata = handler.log_queue[0]["metadata"] + assert metadata["user_api_key_hash"] == user_api_key_dict.api_key + assert metadata["user_api_key_user_id"] == "probe-user-1" + + # -- async_send_batch empty queue and flush_queue edge cases ------------------ diff --git a/type-discipline-budget.json b/type-discipline-budget.json index f071c381916..bf0b30967e7 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -3,7 +3,7 @@ "limit": 23350 }, "LIT002": { - "limit": 27256 + "limit": 27255 }, "LIT003": { "limit": 292 From 9e3a8df6c045bf00d30dcfcf92a7343d8b5d9aa9 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 3 Aug 2026 20:10:37 -0700 Subject: [PATCH 066/124] feat(spend): add net auto-router savings to the cost-optimization dashboard (#35521) * feat(spend): add net auto-router savings to the cost-optimization dashboard The dashboard credited compression and prompt caching but said nothing about the optimization that picks the model, so the driver with the largest lever on a bill was the one an operator could not see. Savings are the counterfactual: without a router a deployment runs one model, and it has to be one that can carry the hardest request, so the baseline is the priciest model in the router's hardest configured tier. A cheap tier is a choice the router made, not a ceiling it was bounded by. `auto_router_savings_baseline_model` overrides it for operators who would genuinely have run something else. Both are provider-qualified before pricing, because a bare name can resolve to a different vendor's rates or to nothing at all, and a deployment is priced by its `base_model` where it has one, which is how Azure deployments are priced everywhere else. Both arms price the request's real usage through `generic_cost_per_token` rather than re-deriving per-token arithmetic, so tiered rates, ephemeral cache-write tiers and regional uplifts stay consistent with what was actually billed. `prompt_tokens` already includes the cache buckets, so charging them again at the input rate would price the same tokens twice. Cache state is what makes this hard. The baseline serves every turn, so whether it had the prompt cached is whether the conversation was already underway. On a continuing conversation it wrote the prompt earlier and would only read it now, so this request's write is what switching cost and counts against the saving. On a first turn nothing was cached for any model, the baseline would have written the same prompt, and both arms carry the write at their own rates. Charging the write to both cases understates a first turn to a few percent of its value, and because the write premium is fixed by prompt size while the saving grows with completion length, it can render a profitable route as a loss. That shape is read off the conversation rather than remembered: a second human ask means an earlier turn was served. No cache, no session id, and no dependence on a caller sending a session header. It cannot see a switch on a turn the router did not classify, and it reads a few-shot prompt's synthetic turns as prior conversation; both err toward charging the write, which under-claims. The baseline and the shape ride on the existing `routing_decision` record, which is already carried from the router to the spend log, already classified for redaction, and already written-or-cleared per attempt. A fallback that re-enters the hook therefore cannot leave either fact behind to be attributed to a deployment that never routed, and no new metadata key crosses the trust boundary. The result is signed. Whether a switch pays off is a race between the rate gap and the cache-write cost, and a narrow gap loses; flooring at zero would hide exactly the routing behaviour an operator needs to see. The donut plots only drivers that saved, while the card and range total keep the sign. Savings accrue into a new `autorouter_savings_spend` column on the six daily rollup tables, declared `NotRequired` because rows queued by a pod on the previous release carry no such key. It is summed by the rollup merge the cross-pod Redis drain also runs, and carried through the aggregation query, the per-row accumulation and the response model, so the dashboard reads a value the API actually sends. Tests enumerate the drivers from the response model itself and assert each is summed, accumulated, carried and totalled, so one added later cannot be half-wired. * fix(spend): let the baseline pay for a continuing turn's own growth `_baseline_usage` moved every cache-creation token into the baseline's read bucket whenever the conversation was underway. That is right for a switch, where the baseline never left the model it was on and really would only read, but wrong for a turn that stayed put: the prompt grew, and the tokens written are that growth. They are new to every model, so the baseline would have paid to write them too. Forgiving it that write made the counterfactual cheaper than it was and shrank the reported saving on ordinary steady-state traffic, by about 2% per turn. The selected arm was never involved; it has always been priced on the real usage. The error sat entirely on the baseline. The condition is that the request read more than it wrote, not that it read anything. A switch onto a model already holding a small prefix of this prompt still writes most of it, and that write is the switch's own cost; keying off a nonzero read would have handed such a request the full rate gap, turning +$0.0056 into +$0.1177. Comparing the two buckets separates a warm continuation, which reads far more than it writes, from a cold arrival, which does the reverse, and it leaves the existing invariant intact: a request reading 0 and one reading 1 both still land in the same place. * fix(spend): price each arm under the key litellm billed it, and see agent turns Two ways the savings number read the wrong thing, both from identifying a model by its name when the name is not what it costs. The counterfactual was ranked and priced on the public rate for the model a deployment names. A deployment may not be charged that rate: the router registers its configured prices under the deployment's own id and deliberately keeps them off the shared model-name key so deployments sharing a backend model do not pollute each other. So a hardest-tier deployment configured above its public rate lost the ranking to a cheaper candidate, and once chosen was priced at a rate nobody pays. Which key prices a deployment is now `_select_model_name_for_cost_calc`'s decision, the resolver the real request is billed through, rather than a second rule here that would have to re-learn that per-second and tiered overrides count, that a partial override still counts, and that a deployment configured at zero is priced at zero rather than treated as unpriced. The arm being subtracted had the same fault and a sharper edge. It priced the spend log's `model`, which on Azure is the deployment name, absent from the cost map, so the whole driver silently read zero for that traffic. It no longer re-derives anything: `model_map_information.model_map_key` is what litellm actually billed the request under, recorded at request time by that same resolver with `base_model` and custom pricing already applied. Separately, the conversation-shape discriminator counted human asks, and an agent loop can run twenty turns on one of them. Its tool traffic rides `tool_result` blocks on user turns that flatten to empty text, and `tool` roles that are never read, so a long agentic conversation looked like its own first turn and was handed the arithmetic that leaves the cache write on both arms. That is the one direction this must never fail in, because it inflates. An assistant turn is the direct evidence that something answered earlier, and it is blind to how the tool plumbing is spelled on either surface. * fix(spend): give the cost-key resolver both inputs the selected arm needs The served model was resolved through one input at a time, and each choice broke the half the other fixed. `model_map_key` is the served model already resolved through `base_model`, which is the only way an Azure deployment name reaches the cost map at all; without it the selected arm priced a name absent from the map, returned nothing, and the whole driver silently read zero for that traffic. But it is built without `router_model_id`, so it never carries a deployment's own price overrides, and a custom-priced deployment was compared at its public rate while the baseline used the real override. On a deployment configured well above its public rate that inverted the answer outright: a route that lost $21.88 reported saving $0.10. `_select_model_name_for_cost_calc` takes both, so it gets both. Which key prices a deployment stays its decision rather than a rule restated here. * fix(spend): same model is only the same cost when it is the same deployment The short-circuit compared resolved model identity, so two deployments of one model collapsed to "no switch" and reported zero. They are not the same cost: a deployment can carry a negotiated rate, and routing from the dear one to the list-price one is a real saving the dashboard reported as $0.00 against a true $21.93. Both arms now carry the key litellm prices them under, so the comparison is between deployments rather than between names. * refactor(spend): price from resolved rates, not from a name we keep re-resolving Four review rounds landed on one mechanism: which identifier prices a deployment. base_model, then the deployment id, then cache-only overrides. Each round added a clause to a resolution rule that should not exist, and a wrong primitive fails once per input shape, so each shape arrived as its own finding. `Router.get_deployment_model_info` already owns this. It merges a deployment's configured prices over the built-in map, folds in `base_model` defaults for deployments whose name is not a model, and falls back to the model name when nothing is overridden. Every shape hand-rolled here (cache-only, partial, per-second, Azure) was that function re-implemented badly. `generic_cost_per_token` now accepts already-resolved rates instead of demanding a name it looks up itself, which is what forced the name-bending in the first place. Both arms resolve through the owner and pass what they got: the counterfactual by the deployment the router would have used, the served request by the deployment that served it. The invented cost-key resolver is gone, and `Baseline` carries a deployment id rather than a key we chose on litellm's behalf. Net 64 insertions against 79 deletions. * test(spend): follow _most_expensive onto the router that prices its candidates Ranking moved through `Router.get_deployment_model_info`, since what a deployment costs is the router's answer to give; these four cases were still calling the old free-function signature. * fix(spend): rank baseline candidates by what a request costs, not by two rates "Most expensive" was decided by comparing output rate then input rate. That is a property of a rate, not of a request: a deployment dearer per output token can be cheaper per cached token, so the comparison ordered cache-heavy traffic backwards and recorded the wrong counterfactual. Candidates are now costed on one reference request through the same engine the savings themselves use, which leaves cache read and write rates, tiered tables and every other billing dimension to that engine rather than to another rule restated here. The reference request is cache-heavy because auto-routed traffic is. * fix(spend): pick the baseline against the request that ran, not a stand-in for one Ranking happened in the pre-routing hook, where the request has not executed yet, so candidates were costed against a hard-coded reference workload: 20k prompt, 19k of it cached, 1k out. Which candidate is dearest depends on that mix, so a pooled hardest tier holding a deployment with non-proportional configured rates could be ranked for a request nothing like the one served. The mix is known on the spend path, so the ranking belongs there. The routing decision now carries the tier's candidates rather than a winner already chosen, and the baseline is resolved against the usage that actually happened. The reference workload is gone; nothing here assumes a traffic shape any more. The router is passed in rather than imported from `proxy_server` inside the computation, so the savings stay a pure function of their arguments and the caller owns where the router comes from. That also makes the spend path testable without a running proxy, which the previous shape was not. * refactor(spend): measure savings against one configured model, not a derived one The counterfactual was derived per request: enumerate the hardest tier's deployments, resolve each one's effective pricing, price them all, take the dearest. That machinery produced a review finding per input shape it had not anticipated, and every answer it gave was one an operator could have stated in a line of config. So they state it. `litellm_settings.autorouter_savings_baseline_model` names the model the traffic would have run on without a router, for every auto-router on the proxy, and unset means the driver is off rather than a model nobody named being guessed at. `savings_baseline.py` and its tests are deleted outright, along with the tier enumeration, the candidate list on the routing decision, and the per-deployment override that shadowed it. Cache-state handling is untouched: the baseline is still priced on this request's own read and write split, so a switch still pays for re-warming the cache and a first turn still charges the write to both arms. 45 insertions against 482 deletions. * refactor(router): compute the conversation shape once and pass it down `_classify_and_route` re-derived it from the messages the hook had already resolved, so an ordinary routed request walked the turn list twice for one boolean. The hook computes it and hands it over, which is also where the affinity-hit path already got it from. Also moves `_get_llm_router` below the imports it sat among. * fix(router): drop the dead conversation_continuing parameter off the hook It was added to `async_pre_routing_hook` by mistake and immediately overwritten by the value the hook computes, so it never did anything. It also widened a signature every pre-routing strategy shares with the protocol in `types/router.py`, leaving this one router diverged from `AutoRouter` and the interface for no reason. Also records why an unreadable request counts as continuing: no messages is no evidence a turn was served, so it pays the cache write and under-claims rather than being handed a first turn's larger saving on nothing. * fix(spend): charge a baseline its input rate for cache buckets it cannot price A model with no cache_creation_input_token_cost, which is every OpenAI, Azure and Gemini entry, resolved that rate to 0.0 and carried the whole written prompt for free, so a first turn routed onto a cheaper model reported a loss. Same hole on cache reads. Those tokens are plain input on such a model, so they move into the text bucket. * refactor(spend): build the daily upsert payloads in one shot `common_data` and `update_data` were constructed and then appended to: `request_id` conditionally for tag rows, `endpoint` unconditionally a few lines later. A dict that grows after its literal cannot be reasoned about by reading the literal, which is the whole point of building it at once. The conditional key resolves to a spreadable value before either payload, so both are single expressions and the tag branch appears once instead of twice. Not wrapped in MappingProxyType, though it was suggested: these go straight to prisma, whose query builder branches on `isinstance(value, dict)` to tell a nested node from a scalar. A mappingproxy is a Mapping but not a dict, so it falls through to the serializer and raises `TypeError: Type not serializable` inside the batch upsert, where the surrounding except would log it and leave the rollups silently unwritten. * fix(spend): keep the one-shot upsert payloads under the type-discipline budget Building both payloads as single literals traded a mutation for two dict literals, and LIT002 counts construction rather than mutation, so the change the review asked for is the one the gate charges for. The empty branch is the avoidable half: it is the same value every time, so it moves to a module constant built once instead of a literal per transaction, and it is a read-only mapping so none of the call sites that spread it can fill it in later. --- .../migration.sql | 17 + .../litellm_proxy_extras/schema.prisma | 6 + litellm/__init__.py | 1 + .../litellm_core_utils/llm_cost_calc/utils.py | 8 +- litellm/proxy/_types.py | 7 +- litellm/proxy/db/db_spend_update_writer.py | 121 +++-- .../daily_spend_update_queue.py | 4 + .../common_daily_activity.py | 9 + litellm/proxy/schema.prisma | 6 + litellm/proxy/spend_tracking/savings.py | 270 +++++++++- .../complexity_router/complexity_router.py | 49 +- .../common_daily_activity.py | 3 + litellm/types/utils.py | 2 + schema.prisma | 6 + .../test_daily_spend_update_queue.py | 60 +++ .../test_common_daily_activity.py | 63 ++- .../proxy/spend_tracking/test_savings.py | 460 +++++++++++++++++- .../router_strategy/test_complexity_router.py | 156 ++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 10 + 19 files changed, 1197 insertions(+), 61 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql new file mode 100644 index 00000000000..a7efd444fdb --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql @@ -0,0 +1,17 @@ +-- AlterTable +ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 37ea55f8c13..0d7fa8692c8 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/litellm/__init__.py b/litellm/__init__.py index a9a78846fa1..62c41c2959b 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -264,6 +264,7 @@ databricks_key: Optional[str] = None openai_like_key: Optional[str] = None azure_key: Optional[str] = None anthropic_key: Optional[str] = None +autorouter_savings_baseline_model: Optional[str] = None replicate_key: Optional[str] = None bytez_key: Optional[str] = None gdc_key: Optional[str] = None diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index face1d1b49f..2eaac7cb1ca 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -683,6 +683,7 @@ def generic_cost_per_token( custom_llm_provider: str, service_tier: str | None = None, data_residency: str | None = None, + model_info: ModelInfo | None = None, ) -> tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -700,7 +701,12 @@ def generic_cost_per_token( """ ## GET MODEL INFO - model_info = get_model_info(model=model, custom_llm_provider=custom_llm_provider) + # A caller that already resolved the deployment's effective rates passes them in + # rather than handing back a name for this to re-resolve. A name cannot express a + # per-deployment override: those are registered under the deployment id and kept off + # the shared model-name key, so resolving from the name here reads the public rate. + if model_info is None: + model_info = get_model_info(model=model, custom_llm_provider=custom_llm_provider) ## CALCULATE INPUT COST ### Cost of processing (non-cache hit + cache hit) + Cost of cache-writing (cache writing) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 268b08d0f2e..5063bc790b5 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -14,7 +14,7 @@ from pydantic import ( field_validator, model_validator, ) -from typing_extensions import Required, TypedDict +from typing_extensions import NotRequired, Required, TypedDict from litellm._uuid import uuid from litellm.constants import MCP_STDIO_ALLOWED_COMMANDS @@ -4566,6 +4566,11 @@ class BaseDailySpendTransaction(TypedDict): # cost-savings metrics (dollars, priced per request before aggregation) compression_savings_spend: float prompt_caching_savings_spend: float + # Not required: rows queued by a pod running the previous release, or replayed from + # the Redis buffer across an upgrade, carry no such key. Every reader coalesces a + # missing value to zero, so requiring it here would describe a shape the aggregation + # is explicitly tested against. + autorouter_savings_spend: NotRequired[float] # request level metrics spend: float diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 17410698aed..7c9ee96e809 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -12,7 +12,9 @@ import os import random import time import traceback +from collections.abc import Mapping from datetime import datetime, timedelta, timezone +from types import MappingProxyType from typing import ( TYPE_CHECKING, Any, @@ -68,6 +70,26 @@ else: ProxyLogging = Any +# Only tag rows carry a request_id, so the other entity types spread nothing. Built +# once here rather than as an empty literal per transaction, and read-only so it cannot +# be filled in by accident from one of the call sites that spreads it. +_NO_TAG_REQUEST_ID: Mapping[str, Any] = MappingProxyType({}) + + +def _get_llm_router(): + """The proxy's router, or None outside a running proxy. + + Injected rather than imported where it is used, so the savings computation stays + a pure function of its arguments and the caller owns where the router comes from. + """ + try: + from litellm.proxy.proxy_server import llm_router + + return llm_router + except Exception: # noqa: BLE001 # no proxy in scope; savings degrade to zero + return None + + def _extract_cache_read_tokens(usage_obj: dict) -> int: """ Anthropic: top-level cache_read_input_tokens field. @@ -1545,6 +1567,40 @@ class DBSpendUpdateWriter: # Get the table dynamically table = getattr(batcher, table_name) + # Additive metrics that older queued rows may omit; one + # enumeration feeds both the create and the increment below + optional_metrics = { + field: value + for field, value in ( + ("cache_read_input_tokens", transaction.get("cache_read_input_tokens")), + ( + "cache_creation_input_tokens", + transaction.get("cache_creation_input_tokens"), + ), + ("compression_saved_tokens", transaction.get("compression_saved_tokens")), + ( + "compression_savings_spend", + transaction.get("compression_savings_spend"), + ), + ( + "prompt_caching_savings_spend", + transaction.get("prompt_caching_savings_spend"), + ), + ("autorouter_savings_spend", transaction.get("autorouter_savings_spend")), + ) + if value is not None + } + + # Only tag rows carry a request_id. Resolved to a spreadable + # value here so both payloads are built in one shot: a dict + # appended to after construction is one nobody can reason about + # by reading its literal. + tag_request_id: Mapping[str, Any] = ( + MappingProxyType({"request_id": transaction["request_id"]}) + if entity_type == "tag" and "request_id" in transaction + else _NO_TAG_REQUEST_ID + ) + # Common data structure for both create and update common_data = { entity_id_field: entity_id, @@ -1561,34 +1617,10 @@ class DBSpendUpdateWriter: "api_requests": transaction["api_requests"], "successful_requests": transaction["successful_requests"], "failed_requests": transaction["failed_requests"], + **optional_metrics, + **tag_request_id, } - # Add cache-related fields if they exist - if "cache_read_input_tokens" in transaction: - common_data["cache_read_input_tokens"] = transaction.get( - "cache_read_input_tokens", 0 - ) - if "cache_creation_input_tokens" in transaction: - common_data["cache_creation_input_tokens"] = transaction.get( - "cache_creation_input_tokens", 0 - ) - if "compression_saved_tokens" in transaction: - common_data["compression_saved_tokens"] = transaction.get( - "compression_saved_tokens", 0 - ) - if "compression_savings_spend" in transaction: - common_data["compression_savings_spend"] = transaction.get( - "compression_savings_spend", 0 - ) - if "prompt_caching_savings_spend" in transaction: - common_data["prompt_caching_savings_spend"] = transaction.get( - "prompt_caching_savings_spend", 0 - ) - - if entity_type == "tag" and "request_id" in transaction: - common_data["request_id"] = transaction.get("request_id") - - # Create update data structure update_data = { "prompt_tokens": {"increment": transaction["prompt_tokens"]}, "completion_tokens": {"increment": transaction["completion_tokens"]}, @@ -1596,36 +1628,12 @@ class DBSpendUpdateWriter: "api_requests": {"increment": transaction["api_requests"]}, "successful_requests": {"increment": transaction["successful_requests"]}, "failed_requests": {"increment": transaction["failed_requests"]}, + **{field: {"increment": value} for field, value in optional_metrics.items()}, + # An existing row predating the endpoint column gets it filled in here + "endpoint": transaction.get("endpoint") or "", + **tag_request_id, } - # Add cache-related fields to update if they exist - if "cache_read_input_tokens" in transaction: - update_data["cache_read_input_tokens"] = { - "increment": transaction.get("cache_read_input_tokens", 0) - } - if "cache_creation_input_tokens" in transaction: - update_data["cache_creation_input_tokens"] = { - "increment": transaction.get("cache_creation_input_tokens", 0) - } - if "compression_saved_tokens" in transaction: - update_data["compression_saved_tokens"] = { - "increment": transaction.get("compression_saved_tokens", 0) - } - if "compression_savings_spend" in transaction: - update_data["compression_savings_spend"] = { - "increment": transaction.get("compression_savings_spend", 0) - } - if "prompt_caching_savings_spend" in transaction: - update_data["prompt_caching_savings_spend"] = { - "increment": transaction.get("prompt_caching_savings_spend", 0) - } - - if entity_type == "tag" and "request_id" in transaction: - update_data["request_id"] = transaction.get("request_id") - - # Add endpoint to update_data so existing rows get their endpoint field updated - update_data["endpoint"] = transaction.get("endpoint") or "" - table.upsert( where=where_clause, data={ @@ -1875,6 +1883,10 @@ class DBSpendUpdateWriter: custom_llm_provider=payload.get("custom_llm_provider", None), compression_saved_tokens=compression_saved_tokens, cache_read_input_tokens=cache_read_input_tokens, + routing_decision=_metadata.get("routing_decision"), + model_id=payload.get("model_id"), + llm_router=_get_llm_router(), + usage_object=usage_obj, ) daily_transaction = BaseDailySpendTransaction( @@ -1896,6 +1908,7 @@ class DBSpendUpdateWriter: compression_saved_tokens=compression_saved_tokens, compression_savings_spend=savings_spend.compression, prompt_caching_savings_spend=savings_spend.prompt_caching, + autorouter_savings_spend=savings_spend.autorouter, ) return daily_transaction except Exception as e: diff --git a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py index 18df25093f1..a61db128382 100644 --- a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py @@ -133,6 +133,10 @@ class DailySpendUpdateQueue(BaseUpdateQueue): payload.get("prompt_caching_savings_spend", 0) or 0 ) + daily_transaction.get("prompt_caching_savings_spend", 0) + daily_transaction["autorouter_savings_spend"] = ( + payload.get("autorouter_savings_spend", 0) or 0 + ) + daily_transaction.get("autorouter_savings_spend", 0) + else: aggregated_daily_spend_update_transactions[_key] = deepcopy(payload) return aggregated_daily_spend_update_transactions diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 0dc85f98786..132a8409e06 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -95,6 +95,9 @@ class DailySpendRecord(Protocol): @property def prompt_caching_savings_spend(self) -> float: ... + @property + def autorouter_savings_spend(self) -> float: ... + @property def api_requests(self) -> int: ... @@ -135,6 +138,7 @@ class _GroupingSetsRow(SimpleNamespace): compression_saved_tokens: int | None compression_savings_spend: float | None prompt_caching_savings_spend: float | None + autorouter_savings_spend: float | None api_requests: int | None successful_requests: int | None failed_requests: int | None @@ -158,6 +162,7 @@ def update_metrics(existing_metrics: SpendMetrics, record: DailySpendRecord) -> existing_metrics.compression_saved_tokens += record.compression_saved_tokens or 0 existing_metrics.compression_savings_spend += record.compression_savings_spend or 0 existing_metrics.prompt_caching_savings_spend += record.prompt_caching_savings_spend or 0 + existing_metrics.autorouter_savings_spend += record.autorouter_savings_spend or 0 existing_metrics.api_requests += record.api_requests or 0 existing_metrics.successful_requests += record.successful_requests or 0 existing_metrics.failed_requests += record.failed_requests or 0 @@ -590,6 +595,7 @@ def _build_aggregated_sql_query( SUM(compression_saved_tokens)::bigint AS compression_saved_tokens, SUM(compression_savings_spend)::float AS compression_savings_spend, SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend, + SUM(autorouter_savings_spend)::float AS autorouter_savings_spend, SUM(api_requests)::bigint AS api_requests, SUM(successful_requests)::bigint AS successful_requests, SUM(failed_requests)::bigint AS failed_requests @@ -732,6 +738,7 @@ def _record_to_spend_metrics(record: _GroupingSetsRow) -> SpendMetrics: compression_saved_tokens=record.compression_saved_tokens or 0, compression_savings_spend=record.compression_savings_spend or 0, prompt_caching_savings_spend=record.prompt_caching_savings_spend or 0, + autorouter_savings_spend=record.autorouter_savings_spend or 0, api_requests=record.api_requests or 0, successful_requests=record.successful_requests or 0, failed_requests=record.failed_requests or 0, @@ -986,6 +993,7 @@ async def get_daily_activity( total_compression_saved_tokens=metadata_metrics.compression_saved_tokens, total_compression_savings_spend=metadata_metrics.compression_savings_spend, total_prompt_caching_savings_spend=metadata_metrics.prompt_caching_savings_spend, + total_autorouter_savings_spend=metadata_metrics.autorouter_savings_spend, page=page, total_pages=-(-total_count // page_size), # Ceiling division has_more=(page * page_size) < total_count, @@ -1075,6 +1083,7 @@ async def get_daily_activity_aggregated( total_compression_saved_tokens=aggregated["totals"].compression_saved_tokens, total_compression_savings_spend=aggregated["totals"].compression_savings_spend, total_prompt_caching_savings_spend=aggregated["totals"].prompt_caching_savings_spend, + total_autorouter_savings_spend=aggregated["totals"].autorouter_savings_spend, page=1, total_pages=1, has_more=False, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 37ea55f8c13..0d7fa8692c8 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index ad9d02052bb..1a1c813d784 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -8,15 +8,22 @@ are known) and summed into the daily tables; tokens cannot be priced after they have been aggregated across models. """ -from typing import NamedTuple +from collections.abc import Mapping +from typing import TYPE_CHECKING, NamedTuple import litellm from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token + +if TYPE_CHECKING: + from litellm.router import Router +from litellm.types.utils import ModelInfo, PromptTokensDetailsWrapper, Usage class SavingsSpend(NamedTuple): compression: float prompt_caching: float + autorouter: float = 0.0 def _input_and_cache_read_cost(model: str | None, custom_llm_provider: str | None) -> tuple[float, float]: @@ -44,11 +51,243 @@ def _input_and_cache_read_cost(model: str | None, custom_llm_provider: str | Non return input_cost, float(cache_read_cost) +class _ModelIdentity(NamedTuple): + model: str + provider: str + + +def _resolve_model(model: str | None, custom_llm_provider: str | None) -> _ModelIdentity | None: + """Canonical ``(model, provider)``, or ``None`` when the model cannot be resolved. + + The two sides of the comparison arrive spelled differently: the spend log records a + normalized model name alongside its provider, while the baseline arrives as the + operator wrote it in config, with the provider prefixed, implied, or absent. Raw + string equality therefore reads `anthropic/claude-opus-5` as a switch away from + `claude-opus-5`, and pricing a bare name with no provider can resolve it to a + different vendor's rates than the deployment it names. + """ + if not model: + return None + try: + resolved_model, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + except Exception as e: # noqa: BLE001 # get_llm_provider raises for unroutable names; degrade to zero savings + verbose_proxy_logger.debug( + "savings: cannot resolve provider for model=%s custom_llm_provider=%s (%s)", model, custom_llm_provider, e + ) + return None + return _ModelIdentity(model=resolved_model, provider=provider) + + +def _effective_model_info(router: "Router | None", deployment_id: str | None, model: str) -> ModelInfo | None: + """What a deployment is actually charged, or ``None`` to price by name. + + `Router.get_deployment_model_info` owns this: it merges a deployment's configured + prices over the built-in map, folds in `base_model` defaults for deployments whose + name is not a model, and falls back to the model name when nothing is overridden. + Resolving a name here instead reads the public rate, which a deployment with a + negotiated price does not pay, and an Azure deployment name prices to nothing at all. + """ + if router is None or deployment_id is None: + return None + try: + return router.get_deployment_model_info(deployment_id, model) + except Exception as e: # noqa: BLE001 # a dashboard metric must not fail the spend write + verbose_proxy_logger.debug("savings: no deployment pricing for %s (%s)", model, e) + return None + + +def _model_info(model: _ModelIdentity) -> ModelInfo | None: + """The public rates for ``model``, or ``None`` when it has none.""" + try: + return litellm.get_model_info(model=model.model, custom_llm_provider=model.provider) + except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models + verbose_proxy_logger.debug("savings: no pricing for provider=%s model=%s (%s)", model.provider, model.model, e) + return None + + +def _cost_of_usage(model: _ModelIdentity, usage: Usage, model_info: ModelInfo | None = None) -> float | None: + """What ``usage`` costs on ``model``, or ``None`` when the model has no pricing.""" + try: + prompt_cost, completion_cost = generic_cost_per_token( + model=model.model, usage=usage, custom_llm_provider=model.provider, model_info=model_info + ) + except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings + verbose_proxy_logger.debug( + "savings: cannot price usage for provider=%s model=%s (%s)", model.provider, model.model, e + ) + return None + return prompt_cost + completion_cost + + +def _cache_token_split(usage: Usage) -> tuple[int, int]: + """``(cache_read_tokens, cache_creation_tokens)`` for a request.""" + details = usage.prompt_tokens_details + if details is None: + return 0, 0 + read = getattr(details, "cached_tokens", 0) or 0 + created = (getattr(details, "cache_creation_tokens", 0) or 0) or (getattr(details, "cache_write_tokens", 0) or 0) + return int(read), int(created) + + +_CACHE_SPLIT_FIELDS = frozenset( + ("cached_tokens", "cache_creation_tokens", "cache_write_tokens", "cache_creation_token_details", "text_tokens") +) + + +def _baseline_cache_rate_keys(baseline_info: ModelInfo | None) -> tuple[bool, bool]: + """Whether the baseline model has a ``(cache read, cache write)`` rate of its own. + + A missing rate is not a free bucket. `_get_token_base_cost` resolves an absent + `cache_read_input_token_cost` or `cache_creation_input_token_cost` to 0.0, so a + baseline whose provider prices caching implicitly, which is every OpenAI, Azure and + Gemini entry for cache writes, would carry the whole prompt for nothing and turn a + profitable route into a reported loss. Such a model pays its plain input rate for + those tokens, so the buckets it cannot price become ordinary input below. + """ + if baseline_info is None: + return True, True + return bool(baseline_info.get("cache_read_input_token_cost")), bool( + baseline_info.get("cache_creation_input_token_cost") + ) + + +def _baseline_usage(usage: Usage, conversation_continuing: bool, baseline_info: ModelInfo | None = None) -> Usage: + """The same request as a single-model baseline would have met it. + + The baseline is one model serving every turn, so whether it had this prompt cached + is simply whether the conversation was already underway. On a continuing + conversation it wrote the prompt on an earlier turn and would only read it now, so + the cache tokens move into the read bucket and whatever this request paid to write + counts against the saving; that write is what switching models costs. + + On a conversation's first turn nothing was cached anywhere, for any model. The + baseline would have written the same prompt, so the cache buckets stay where they are + and both arms carry the write at their own rates, unless the baseline has no rate for + a bucket, in which case those tokens are its plain input. Charging the write to this case + too, which is all a single rollup row can support, understates a first turn to a + few percent of its value and can render a profitable route as a loss. + + A continuing turn that mostly read from cache is the third case: the selected model + was already warm, so it is the one that has been serving this conversation and the + baseline's cache holds exactly what its does. The tokens written are the turn's own + growth, new to every model, and the baseline would have paid to write them too. + Moving them would forgive the baseline a write it really owes and shrink the + reported saving. "Mostly read" rather than "read anything" on purpose: a switch onto + a model holding a small prefix of this prompt still writes most of it, and must keep + counting that write against the saving. + + Only the cache buckets move. Every other field the request was priced on travels + through untouched, audio and image and video counts among them, because the baseline + is this same request served by a model that happened to be warm; naming the fields to + keep instead would price the baseline on a request that never ran, and would go stale + the next time a priced field is added. + """ + cache_read, cache_creation = _cache_token_split(usage) + details = usage.prompt_tokens_details + if details is None or (cache_read <= 0 and cache_creation <= 0): + return usage + + # The tokens this request paid to write move into the cached count and the creation + # charge is dropped: on one model that cache was already warm, so the baseline would + # have read them rather than paying to create them. The 5m/1h breakdown goes with + # them; left behind it re-charges the write. + warm = conversation_continuing and cache_creation > 0 and cache_read <= cache_creation + reads = cache_read + cache_creation if warm else cache_read + writes = 0 if warm else cache_creation + + prices_reads, prices_writes = _baseline_cache_rate_keys(baseline_info) + reads = reads if prices_reads else 0 + writes = writes if prices_writes else 0 + if (reads, writes) == (cache_read, cache_creation): + return usage + + other_modalities = sum( + (getattr(details, field, 0) or 0) for field in ("audio_tokens", "image_tokens", "video_tokens") + ) + return Usage( + prompt_tokens=usage.prompt_tokens, + completion_tokens=usage.completion_tokens, + total_tokens=usage.total_tokens, + completion_tokens_details=usage.completion_tokens_details, + prompt_tokens_details=PromptTokensDetailsWrapper( + **details.model_dump(exclude=_CACHE_SPLIT_FIELDS), + cached_tokens=reads, + cache_creation_tokens=writes, + cache_write_tokens=writes, + cache_creation_token_details=details.cache_creation_token_details if writes else None, + # Whatever no longer sits in a cache bucket is plain input on the baseline. + text_tokens=max(usage.prompt_tokens - reads - writes - other_modalities, 0), + ), + ) + + +def compute_autorouter_savings( + baseline_model: str | None, + selected_model: str | None, + selected_provider: str | None, + usage: Usage, + conversation_continuing: bool = True, + selected_info: ModelInfo | None = None, +) -> float: + """Net dollars the router saved, or cost, by serving this request on ``selected_model``. + + Signed on purpose. Switching models leaves the new one with a cold cache, so the + request pays a cache-creation charge that staying on one model would not have + incurred; when that charge outweighs the cheaper rates, routing lost money and the + dashboard has to be able to say so. Zero when both sides resolve to the same + deployment, or when either cannot be resolved or priced. + + ``conversation_continuing`` says whether the baseline would already have had this + prompt cached. It defaults to True because that is the conservative reading: a + request whose shape the router could not determine is charged the write and + under-claims rather than inflating a savings figure. + """ + # No provider argument for the baseline on purpose: it arrives from the routing + # metadata as a single self-describing string, already qualified by the auto-router, + # so there is no second field that could disagree with it. + baseline = _resolve_model(baseline_model, None) + selected = _resolve_model(selected_model, selected_provider) + if baseline is None or selected is None: + return 0.0 + # Same model is only the same cost when it is also the same deployment. Two + # deployments of one model can carry different negotiated rates, and routing from + # the dear one to the cheap one is a real saving that short-circuiting on the model + # name alone reports as zero. + if baseline == selected: + return 0.0 + baseline_info = _model_info(baseline) + baseline_cost = _cost_of_usage( + baseline, _baseline_usage(usage, conversation_continuing, baseline_info), baseline_info + ) + selected_cost = _cost_of_usage(selected, usage, selected_info) + if baseline_cost is None or selected_cost is None: + return 0.0 + return baseline_cost - selected_cost + + +def _usage_from_spend_log(usage_object: Mapping[str, object] | None) -> Usage | None: + """Rebuild the request's ``Usage`` from the copy the spend log recorded.""" + if not usage_object: + return None + try: + return Usage(**usage_object) + except Exception as e: # noqa: BLE001 # a malformed usage_object must not fail the daily spend write + # Warning, not debug: this silently zeroes the auto-router driver for every + # affected row, and a shape change in Usage would otherwise show up only as a + # dashboard that quietly reads $0.00. + verbose_proxy_logger.warning("savings: unusable usage_object, auto-router savings will read zero (%s)", e) + return None + + def compute_savings_spend( model: str | None, custom_llm_provider: str | None, compression_saved_tokens: int, cache_read_input_tokens: int, + routing_decision: Mapping[str, object] | None = None, + usage_object: Mapping[str, object] | None = None, + model_id: str | None = None, + llm_router: "Router | None" = None, ) -> SavingsSpend: """ Dollar savings for one request, split by optimization driver. @@ -56,8 +295,35 @@ def compute_savings_spend( Compression savings price the tokens compression removed at the model's input rate. Prompt-caching savings price the cache-read tokens at the difference between the input rate and the discounted cache-read rate. + Auto-router savings compare the served ``model`` against the counterfactual + baseline the router recorded on its ``routing_decision``, and are zero unless the + two differ. That record also says whether the conversation was already underway, + which is what tells a mid-conversation switch from a first turn. """ input_cost, cache_read_cost = _input_and_cache_read_cost(model, custom_llm_provider) compression = max(compression_saved_tokens, 0) * input_cost prompt_caching = max(cache_read_input_tokens, 0) * max(input_cost - cache_read_cost, 0.0) - return SavingsSpend(compression=compression, prompt_caching=prompt_caching) + + usage = _usage_from_spend_log(usage_object) + if usage is None or not model: + return SavingsSpend(compression=compression, prompt_caching=prompt_caching) + + # The counterfactual is one model an operator would have run instead of the router, + # configured once for the proxy rather than derived per request. Unset means the + # driver is off; a routing decision is what says this request was auto-routed at all. + decision = routing_decision if isinstance(routing_decision, Mapping) else {} + autorouter = ( + compute_autorouter_savings( + baseline_model=litellm.autorouter_savings_baseline_model, + selected_model=model, + selected_provider=custom_llm_provider, + usage=usage, + # Absent means the router never recorded a shape, which is the conservative + # reading: charge the cache write rather than claim a first turn's saving. + conversation_continuing=decision.get("conversation_continuing") is not False, + selected_info=_effective_model_info(llm_router, model_id, model or ""), + ) + if decision + else 0.0 + ) + return SavingsSpend(compression=compression, prompt_caching=prompt_caching, autorouter=autorouter) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 6f2bf61834b..ca2e7fcc038 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -224,6 +224,40 @@ def _iter_human_asks_newest_first(messages: Sequence[Mapping[str, object]]) -> I ) +def _conversation_is_continuing(messages: Sequence[Mapping[str, object]] | None) -> bool: + """Whether this request continues a conversation that was already underway. + + The counterfactual the savings driver prices against is one model serving every + turn, so whether that model had this prompt cached is just whether an earlier turn + exists. An assistant turn in the history is the direct evidence of one: something + answered before, so a single-model deployment wrote the prompt then and would only + read it now, and the write this request paid is what switching models cost. A + conversation's first turn has no assistant turn, nothing was cached for any model, + and the baseline would have paid the same write. + + Assistant turns rather than human asks, because an agent loop can run twenty turns + on one human ask: its tool traffic rides `tool_result` blocks on user turns that + flatten to empty text, and on `tool` roles, so counting asks reads a long + conversation as its own first turn and hands it the untouched-write arithmetic. That + is the one direction this must never fail in, since it inflates. + + Reading the conversation rather than remembering it keeps this free of a cache, a + session id and their failure modes, and it works for callers that send no session + header at all. A few-shot prompt's synthetic assistant turns read as prior + conversation, which charges the write and under-claims; that is the safe side. + + So is an unreadable request. No messages says nothing about whether a turn was + served, and a surface that carries its turns somewhere this cannot see, or a + genuinely single-turn call arriving with none, is treated as continuing: it pays the + cache write and under-claims rather than being handed a first turn's larger saving + on no evidence. That direction is deliberate in both cases and is the only one that + cannot inflate. + """ + if not messages: + return True + return any(message.get("role") == "assistant" for message in messages) + + def _newest_turn_ask(messages: Sequence[Mapping[str, object]]) -> str | None: """The human ask on the newest user turn, or None when that turn carries only plumbing. @@ -647,6 +681,7 @@ class ComplexityRouter(CustomLogger): escalation_keyword: str | None = None, escalated: bool = False, classifier_model: str | None = None, + conversation_continuing: bool = True, ) -> StandardLoggingRoutingDecision: """Assemble the per-request provenance record for this router's decision. @@ -660,6 +695,7 @@ class ComplexityRouter(CustomLogger): router_type="complexity", routed_model=routed_model, cause=cause, + conversation_continuing=conversation_continuing, ) if tier is not None: decision["tier"] = tier.value @@ -1392,6 +1428,8 @@ class ComplexityRouter(CustomLogger): if isinstance(metadata, dict): metadata[RETURN_RAW_MODEL_NAME_METADATA_KEY] = True + conversation_continuing = _conversation_is_continuing(self._resolve_messages(messages, request_kwargs)) + use_session_affinity = self.config.session_affinity and not self.config.plugins session_id = self._get_session_id_from_request_kwargs(request_kwargs) if use_session_affinity else None cache_key = self._get_session_affinity_cache_key(session_id, request_kwargs) if session_id is not None else None @@ -1438,6 +1476,7 @@ class ComplexityRouter(CustomLogger): cause=cause, escalation_keyword=pin_escalation_keyword, escalated=escalated, + conversation_continuing=conversation_continuing, ), ) @@ -1447,6 +1486,7 @@ class ComplexityRouter(CustomLogger): messages=messages, input=input, specific_deployment=specific_deployment, + conversation_continuing=conversation_continuing, ) if cache_key is not None and response is not None: await self.litellm_router_instance.cache.async_set_cache( @@ -1463,6 +1503,7 @@ class ComplexityRouter(CustomLogger): messages: list[dict[str, Any]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, + conversation_continuing: bool = True, ) -> PreRoutingHookResponse | None: """ Classifies the request by complexity and returns the appropriate model. @@ -1509,7 +1550,11 @@ class ComplexityRouter(CustomLogger): return PreRoutingHookResponse( model=routed_model, messages=messages if has_original_messages else None, - routing_decision=self._build_routing_decision(routed_model=routed_model, cause="default_fallback"), + routing_decision=self._build_routing_decision( + routed_model=routed_model, + cause="default_fallback", + conversation_continuing=conversation_continuing, + ), ) newest_ask = _newest_turn_ask(resolved_messages) @@ -1532,6 +1577,7 @@ class ComplexityRouter(CustomLogger): messages=messages if has_original_messages else None, routing_decision=self._build_routing_decision( routed_model=routed_model, + conversation_continuing=conversation_continuing, cause=keyword_cause, tier=routed_tier, matched_keyword=override.matched_keyword, @@ -1579,6 +1625,7 @@ class ComplexityRouter(CustomLogger): messages=messages if has_original_messages else None, routing_decision=self._build_routing_decision( routed_model=routed_model, + conversation_continuing=conversation_continuing, cause=outcome.cause, tier=tier, score=score, diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 00f67d0f39c..21b7ffca3f2 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -25,6 +25,7 @@ class SpendMetrics(BaseModel): compression_saved_tokens: int = Field(default=0) compression_savings_spend: float = Field(default=0.0) prompt_caching_savings_spend: float = Field(default=0.0) + autorouter_savings_spend: float = Field(default=0.0) total_tokens: int = Field(default=0) successful_requests: int = Field(default=0) failed_requests: int = Field(default=0) @@ -85,6 +86,7 @@ class DailySpendMetadata(BaseModel): total_compression_saved_tokens: int = Field(default=0) total_compression_savings_spend: float = Field(default=0.0) total_prompt_caching_savings_spend: float = Field(default=0.0) + total_autorouter_savings_spend: float = Field(default=0.0) page: int = Field(default=1) total_pages: int = Field(default=1) has_more: bool = Field(default=False) @@ -111,6 +113,7 @@ class LiteLLM_DailyUserSpend(BaseModel): compression_saved_tokens: int = 0 compression_savings_spend: float = 0.0 prompt_caching_savings_spend: float = 0.0 + autorouter_savings_spend: float = 0.0 spend: float = 0.0 api_requests: int = 0 successful_requests: int = 0 diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 3539ac0f27a..21d69d9925a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2734,6 +2734,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): classifier_model: str escalated: bool tier_boundaries: StandardLoggingRoutingDecisionTierBoundaries + conversation_continuing: bool # Fields whose values quote the caller's prompt. Dropped when an operator turns message @@ -2753,6 +2754,7 @@ DERIVED_ROUTING_DECISION_FIELDS: FrozenSet[str] = frozenset( "classifier_model", "escalated", "tier_boundaries", + "conversation_continuing", } ) diff --git a/schema.prisma b/schema.prisma index 37ea55f8c13..0d7fa8692c8 100644 --- a/schema.prisma +++ b/schema.prisma @@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py index c7e7ef5d469..a55d4f0dcfd 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py @@ -16,6 +16,9 @@ from litellm.proxy._types import ( Litellm_EntityType, SpendUpdateQueueItem, ) +from typing import get_args + +from litellm.proxy._types import BaseDailySpendTransaction from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import ( DailySpendUpdateQueue, ) @@ -209,6 +212,7 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key(): "compression_saved_tokens": 0, "compression_savings_spend": 0, "prompt_caching_savings_spend": 0, + "autorouter_savings_spend": 0, } updates = [{test_key: test_transaction1}, {test_key: test_transaction2}] @@ -259,6 +263,7 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions( "compression_saved_tokens": 0, "compression_savings_spend": 0, "prompt_caching_savings_spend": 0, + "autorouter_savings_spend": 0, } # Add updates to queue @@ -527,3 +532,58 @@ async def test_compression_saved_tokens_aggregation(daily_spend_update_queue): assert agg["cache_creation_input_tokens"] == 7 assert agg["compression_savings_spend"] == pytest.approx(0.0076) assert agg["prompt_caching_savings_spend"] == pytest.approx(0.0108) + + +@pytest.mark.asyncio +async def test_every_optional_daily_metric_aggregates(daily_spend_update_queue): + """Every additive metric must survive the merge, not just the ones wired by hand. + + Two requests landing on one rollup key before a flush is the common case under + load, and this same merge runs again on every cross-pod Redis drain. A metric + persisted by the database write but skipped here is silently dropped on both + paths, so the driver reads as zero on the dashboard however much it saved. + """ + test_key = "user1_2023-01-01_key123_claude-haiku-4-5_anthropic" + def _numeric(annotation): + # additive metrics may be declared NotRequired[float] for rows queued by a pod + # running the previous release, so unwrap before matching + args = get_args(annotation) + return (args[0] if args else annotation) in (int, float) + + numeric_fields = [ + name for name, annotation in BaseDailySpendTransaction.__annotations__.items() if _numeric(annotation) + ] + assert "autorouter_savings_spend" in numeric_fields + increments = {field: index + 1 for index, field in enumerate(numeric_fields)} + + await daily_spend_update_queue.add_update({test_key: dict(increments)}) + await daily_spend_update_queue.add_update({test_key: dict(increments)}) + await daily_spend_update_queue.aggregate_queue_updates() + updates = await daily_spend_update_queue.flush_all_updates_from_in_memory_queue() + + agg = updates[0][test_key] + for field, value in increments.items(): + assert agg[field] == pytest.approx(value * 2), f"{field} did not accumulate" + + +@pytest.mark.asyncio +async def test_optional_metric_missing_from_an_older_payload_still_aggregates( + daily_spend_update_queue, +): + """A queued row written before a metric existed must not zero it out.""" + test_key = "user1_2023-01-01_key123_claude-haiku-4-5_anthropic" + base = { + "spend": 1.0, + "prompt_tokens": 10, + "completion_tokens": 5, + "api_requests": 1, + "successful_requests": 1, + "failed_requests": 0, + } + + await daily_spend_update_queue.add_update({test_key: dict(base)}) + await daily_spend_update_queue.add_update({test_key: {**base, "autorouter_savings_spend": 0.25}}) + await daily_spend_update_queue.aggregate_queue_updates() + updates = await daily_spend_update_queue.flush_all_updates_from_in_memory_queue() + + assert updates[0][test_key]["autorouter_savings_spend"] == pytest.approx(0.25) diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index c2a0d34a915..f2749be5d6e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -19,7 +19,10 @@ from litellm.proxy.management_endpoints.common_daily_activity import ( get_daily_activity_aggregated, update_metrics, ) -from litellm.types.proxy.management_endpoints.common_daily_activity import SpendMetrics +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + DailySpendMetadata, + SpendMetrics, +) @pytest.mark.asyncio @@ -153,6 +156,7 @@ async def test_get_daily_activity_aggregated_with_endpoint_breakdown(): "compression_saved_tokens": 0, "compression_savings_spend": 0.0, "prompt_caching_savings_spend": 0.0, + "autorouter_savings_spend": 0.0, "failed_requests": 0, } mock_rows = [ @@ -498,6 +502,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_1.compression_saved_tokens = 0 mock_record_1.compression_savings_spend = 0.0 mock_record_1.prompt_caching_savings_spend = 0.0 + mock_record_1.autorouter_savings_spend = 0.0 mock_record_1.api_requests = 10 mock_record_1.successful_requests = 9 mock_record_1.failed_requests = 1 @@ -520,6 +525,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_2.compression_saved_tokens = 0 mock_record_2.compression_savings_spend = 0.0 mock_record_2.prompt_caching_savings_spend = 0.0 + mock_record_2.autorouter_savings_spend = 0.0 mock_record_2.api_requests = 5 mock_record_2.successful_requests = 5 mock_record_2.failed_requests = 0 @@ -582,6 +588,7 @@ async def test_aggregated_activity_preserves_metadata_for_deleted_keys(): "compression_saved_tokens": 0, "compression_savings_spend": 0.0, "prompt_caching_savings_spend": 0.0, + "autorouter_savings_spend": 0.0, "failed_requests": 0, } mock_rows = [ @@ -669,6 +676,7 @@ def _daily_user_spend_record(*, user_id, api_key, spend, model="gpt-4", model_gr compression_saved_tokens=0, compression_savings_spend=0.0, prompt_caching_savings_spend=0.0, + autorouter_savings_spend=0.0, api_requests=1, successful_requests=1, failed_requests=0, @@ -973,6 +981,7 @@ async def test_get_daily_activity_aggregated_empty_result_set(): "compression_saved_tokens": None, "compression_savings_spend": None, "prompt_caching_savings_spend": None, + "autorouter_savings_spend": None, "api_requests": None, "successful_requests": None, "failed_requests": None, @@ -1016,6 +1025,7 @@ def _no_spend_record(): compression_saved_tokens=None, compression_savings_spend=None, prompt_caching_savings_spend=None, + autorouter_savings_spend=None, api_requests=None, successful_requests=None, failed_requests=None, @@ -1050,3 +1060,54 @@ def test_update_metrics_handles_none_values(): assert metrics.cache_read_input_tokens == 0 assert metrics.cache_creation_input_tokens == 0 assert metrics.compression_saved_tokens == 0 + + +class TestEverySavingsDriverSurvivesTheReadPath: + """A savings driver is only real if it survives the whole read path. + + The write path can price a driver correctly and persist it to all six rollup + tables, and the dashboard can still render a permanent $0.00 because the + aggregation query never summed the column or the response model never + declared it. That failure is silent: the card renders, the number is just + always zero, which is indistinguishable from having saved nothing. These + tests enumerate the drivers from the response model itself, so a driver added + later cannot be half-wired. + """ + + def _drivers(self) -> list[str]: + drivers = [field for field in SpendMetrics.model_fields if field.endswith("_savings_spend")] + assert drivers, "expected the dashboard response to expose at least one savings driver" + return drivers + + def test_every_driver_is_summed_by_the_rollup_query(self): + sql, _ = _build_aggregated_sql_query( + table_name="litellm_dailyuserspend", + entity_id_field="user_id", + entity_id="user-1", + start_date="2026-07-01", + end_date="2026-07-31", + model=None, + api_key=None, + timezone_offset_minutes=None, + ) + for driver in self._drivers(): + assert f"SUM({driver})" in sql, f"{driver} is never summed, so it reads as zero" + + def test_every_driver_is_accumulated_across_rows(self): + for driver in self._drivers(): + record = _no_spend_record() + setattr(record, driver, 1.25) + metrics = update_metrics(SpendMetrics(), record) + assert getattr(metrics, driver) == pytest.approx(1.25), f"{driver} is dropped when accumulating rows" + + def test_every_driver_is_carried_by_a_single_row_conversion(self): + for driver in self._drivers(): + record = _no_spend_record() + setattr(record, driver, 2.5) + assert getattr(_record_to_spend_metrics(record), driver) == pytest.approx(2.5) + + def test_every_driver_has_a_range_total(self): + for driver in self._drivers(): + assert f"total_{driver}" in DailySpendMetadata.model_fields, ( + f"total_{driver} is missing, so the range summary omits the driver" + ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 704cf7a63dd..6d0195b632b 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -6,7 +6,14 @@ sys.path.insert(0, os.path.abspath("../../../..")) import pytest import litellm -from litellm.proxy.spend_tracking.savings import compute_savings_spend +from litellm.router import Router +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.proxy.spend_tracking.savings import ( + _baseline_usage, + compute_autorouter_savings, + compute_savings_spend, +) +from litellm.types.utils import Usage def _anthropic_costs(model: str) -> tuple[float, float]: @@ -16,6 +23,39 @@ def _anthropic_costs(model: str) -> tuple[float, float]: return input_cost, cache_read_cost +def _cached_usage_object() -> dict: + """A cache-heavy Anthropic request, shaped as the spend log records it. + + `prompt_tokens` is the inclusive total: 3 uncached text tokens plus 500 read + from cache plus 12304 written to cache. + """ + return { + "prompt_tokens": 12807, + "completion_tokens": 500, + "total_tokens": 13307, + "prompt_tokens_details": {"cached_tokens": 500, "cache_creation_tokens": 12304, "text_tokens": 3}, + "cache_creation_input_tokens": 12304, + "cache_read_input_tokens": 500, + } + + +def _cost_on(model: str, usage_object: dict) -> float: + prompt_cost, completion_cost = generic_cost_per_token( + model=model, usage=Usage(**usage_object), custom_llm_provider="anthropic" + ) + return prompt_cost + completion_cost + + +def _flat_rates(model: str) -> tuple[float, float, float]: + info = litellm.get_model_info(model=model, custom_llm_provider="anthropic") + input_cost = info["input_cost_per_token"] or 0.0 + return ( + input_cost, + info["output_cost_per_token"] or 0.0, + info.get("cache_creation_input_token_cost") or input_cost, + ) + + def test_compression_savings_priced_at_input_rate(): input_cost, _ = _anthropic_costs("claude-sonnet-5") result = compute_savings_spend( @@ -76,3 +116,421 @@ def test_negative_token_counts_clamp_to_zero(): ) assert result.compression == 0.0 assert result.prompt_caching == 0.0 + + +def _usage(fresh: int, cached: int, written: int, out: int) -> Usage: + """Usage as the spend log records it; `prompt_tokens` is the inclusive total.""" + return Usage( + prompt_tokens=fresh + cached + written, + completion_tokens=out, + total_tokens=fresh + cached + written + out, + prompt_tokens_details={"cached_tokens": cached, "cache_creation_tokens": written, "text_tokens": fresh}, + cache_read_input_tokens=cached, + cache_creation_input_tokens=written, + ) + + +def _savings(baseline: str, selected: str, usage: Usage, continuing: bool = True) -> float: + """Savings for a request, defaulting to a conversation already underway. + + `continuing=True` is the mid-conversation case, where the baseline had the prompt + cached and this request's write is what the switch cost. `continuing=False` is a + conversation's first turn, where nothing was cached for any model. + """ + return compute_autorouter_savings( + baseline_model=baseline, + selected_model=selected, + selected_provider="anthropic", + usage=usage, + conversation_continuing=continuing, + ) + + +def test_switching_models_mid_conversation_charges_the_cold_cache_write(): + """Staying on one model writes the cache once and reads it thereafter. Switching + leaves the new model cold, so it pays to write the whole prompt again; when that + charge outweighs the cheaper rates the route lost money and must report a loss. + + Pricing the baseline as if it too re-wrote the cache credits a charge it never + paid, which is how a losing switch used to read as the largest saving on the page. + """ + usage = _usage(fresh=3, cached=500, written=12304, out=500) + result = _savings("claude-sonnet-5", "claude-haiku-4-5", usage) + + sonnet = litellm.get_model_info("claude-sonnet-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + warm_baseline = ( + 3 * sonnet["input_cost_per_token"] + + 12804 * sonnet["cache_read_input_token_cost"] + + 500 * sonnet["output_cost_per_token"] + ) + actually_paid = ( + 3 * haiku["input_cost_per_token"] + + 500 * haiku["cache_read_input_token_cost"] + + 12304 * haiku["cache_creation_input_token_cost"] + + 500 * haiku["output_cost_per_token"] + ) + assert result == pytest.approx(warm_baseline - actually_paid) + assert result < 0, "a cache-thrashing switch must report a loss, not a saving" + + phantom = 12304 * sonnet["cache_creation_input_token_cost"] + assert result != pytest.approx(warm_baseline + phantom - actually_paid) + + +def test_a_cold_switch_never_beats_turning_caching_off(): + """Switching to a cold model makes it write the whole prompt again. That write is a + real cost of switching, so the same traffic must look worse than if caching were off + entirely. + + The baseline is priced as a warm cache even though this request read nothing: a + switch reads nothing precisely because the new model's cache is empty, and staying + on one model would have had the prompt cached already. Gating the warm baseline on + a read charged the baseline a write it would never repeat, which made a cold switch + report a larger saving than no caching at all. + """ + cold_switch = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(0, 0, 20_000, 1_000)) + caching_off = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(20_000, 0, 0, 1_000)) + + assert cold_switch < caching_off + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + warm_baseline = 20_000 * opus["cache_read_input_token_cost"] + 1_000 * opus["output_cost_per_token"] + actually_paid = 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] + assert cold_switch == pytest.approx(warm_baseline - actually_paid) + + +def test_moving_one_token_between_cache_buckets_does_not_move_the_answer(): + """A continuing conversation writes a few new tokens and reads the rest. Treating the + presence of a write as the signal for a switch made that ordinary increment flip the + result, so a request reading 19,999 and writing 1 landed somewhere entirely different + from one reading 20,000 and writing none. + """ + reads_nothing = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(0, 0, 20_000, 1_000)) + reads_one = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(0, 1, 19_999, 1_000)) + assert reads_one == pytest.approx(reads_nothing, abs=1e-4) + + +def test_multimodal_prompts_are_priced_on_the_baseline_too(): + """The baseline is this same request met by a warm cache, so every field it was + priced on has to survive. Rebuilding the details from the cache buckets alone + dropped the image and audio counts, which priced the baseline as a text-only + request that never ran and shrank the reported saving on multimodal traffic. + """ + details = {"cached_tokens": 0, "cache_creation_tokens": 16_000, "text_tokens": 0, "image_tokens": 4_000} + with_images = Usage( + prompt_tokens=20_000, + completion_tokens=1_000, + total_tokens=21_000, + prompt_tokens_details=details, + ) + baseline = _baseline_usage(with_images, conversation_continuing=True) + + assert baseline.prompt_tokens_details.image_tokens == 4_000, "image tokens must survive into the baseline" + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + priced, _ = generic_cost_per_token(model="claude-opus-5", usage=baseline, custom_llm_provider="anthropic") + text_only = 20_000 * opus["cache_read_input_token_cost"] + assert priced > text_only, "dropping the image tokens undercharges the baseline and hides the saving" + + +def test_the_baseline_is_never_charged_a_cache_write(): + """Carrying the details through must not carry the 5m/1h creation breakdown with + them. `generic_cost_per_token` charges a creation cost whenever that breakdown is + present, even against a zeroed creation count, which would put the phantom write + back on the baseline for every long-cache request. + """ + long_cache = Usage( + prompt_tokens=20_000, + completion_tokens=1_000, + total_tokens=21_000, + prompt_tokens_details={ + "cached_tokens": 0, + "cache_creation_tokens": 20_000, + "text_tokens": 0, + "cache_creation_token_details": {"ephemeral_1h_input_tokens": 20_000}, + }, + ) + baseline = _baseline_usage(long_cache, conversation_continuing=True) + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + priced, _ = generic_cost_per_token(model="claude-opus-5", usage=baseline, custom_llm_provider="anthropic") + assert priced == pytest.approx(20_000 * opus["cache_read_input_token_cost"]), ( + "the baseline reads a warm cache; it never pays to create one" + ) + + +def test_uncached_request_is_the_plain_rate_difference(): + usage = _usage(fresh=2000, cached=0, written=0, out=500) + sonnet = litellm.get_model_info("claude-sonnet-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + assert _savings("claude-sonnet-5", "claude-haiku-4-5", usage) == pytest.approx( + 2000 * (sonnet["input_cost_per_token"] - haiku["input_cost_per_token"]) + + 500 * (sonnet["output_cost_per_token"] - haiku["output_cost_per_token"]) + ) + + +def test_escalation_reports_its_real_cost(): + """Routing up to a pricier model is a real cost; hiding it behind a zero floor + would let the dashboard only ever move in one direction.""" + usage = _usage(fresh=2000, cached=0, written=0, out=500) + assert _savings("claude-haiku-4-5", "claude-sonnet-5", usage) < 0 + + +def test_autorouter_savings_zero_when_model_unchanged(): + assert _savings("claude-opus-5", "claude-opus-5", _usage(3, 500, 12304, 500)) == 0.0 + + +def test_autorouter_savings_unknown_baseline_fails_open_to_zero(): + assert _savings("totally-made-up-model-xyz", "claude-haiku-4-5", _usage(3, 500, 12304, 500)) == 0.0 + + +def test_autorouter_savings_zero_without_baseline(): + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + cache_read_input_tokens=0, + routing_decision=None, + usage_object=_cached_usage_object(), + ) + assert result.autorouter == 0.0 + + +def test_compute_savings_spend_carries_a_losing_switch_through(monkeypatch): + """The signed value must survive into SavingsSpend; clamping it here would put the + dashboard back to only ever showing gains.""" + monkeypatch.setattr(litellm, "autorouter_savings_baseline_model", "claude-sonnet-5") + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + cache_read_input_tokens=0, + routing_decision={"conversation_continuing": True}, + usage_object=_cached_usage_object(), + ) + assert result.autorouter < 0 + + +def test_the_driver_is_off_until_a_baseline_is_configured(): + """No configured counterfactual means there is nothing to measure against, so the + driver reports zero rather than inventing a model the operator never named.""" + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=1000, + cache_read_input_tokens=0, + routing_decision={"conversation_continuing": True}, + usage_object=_cached_usage_object(), + ) + assert result.autorouter == 0.0 + assert result.compression > 0, "the other drivers keep working" + + +def test_malformed_usage_object_does_not_fail_the_spend_write(): + """The daily spend write must survive an unusable usage_object; losing one row's + savings is recoverable, losing the row is not.""" + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=1000, + cache_read_input_tokens=0, + routing_decision={"conversation_continuing": True}, + usage_object={"prompt_tokens": ["not", "a", "number"]}, + ) + assert result.autorouter == 0.0 + assert result.compression > 0 + + +def test_model_without_cache_read_pricing_yields_no_caching_savings(): + """A model with no discounted cache-read rate cannot have saved anything by + reading from cache, so the driver must report zero rather than the full input rate.""" + model = "azure/gpt-3.5-turbo" + assert litellm.get_model_info(model=model).get("cache_read_input_token_cost") is None + result = compute_savings_spend( + model=model, + custom_llm_provider="azure", + compression_saved_tokens=0, + cache_read_input_tokens=5000, + ) + assert result.prompt_caching == 0.0 + + +def test_the_same_deployment_spelled_two_ways_is_not_a_switch(): + """The spend log records a normalized model name while the baseline arrives as the + operator wrote it in config. Comparing the raw strings makes a request that never + changed model look like a switch, and prices one deployment against itself.""" + # Must be a cached request: the baseline arm is priced against a warm cache and the + # selected arm against what was actually paid, so treating one deployment as two + # charges it a cold-cache write it never took, inventing a loss on a request that + # never changed model. An uncached request prices identically either way and would + # make this assertion vacuous. + usage = _usage(fresh=3, cached=500, written=12304, out=500) + assert _savings("anthropic/claude-opus-5", "claude-opus-5", usage) == 0.0 + assert _savings("claude-opus-5", "anthropic/claude-opus-5", usage) == 0.0 + + +def test_baseline_is_priced_under_its_own_provider(): + """Two providers can serve the same bare model name at different rates, so dropping + the provider prices the baseline against a vendor the operator never named. Here it + decides whether routing reads as a saving or a loss.""" + usage = Usage(prompt_tokens=100_000, completion_tokens=10_000, total_tokens=110_000) + azure = compute_autorouter_savings( + baseline_model="azure_ai/deepseek-r1", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=usage, + ) + deepseek = compute_autorouter_savings( + baseline_model="deepseek/deepseek-r1", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=usage, + ) + assert azure != pytest.approx(deepseek) + assert azure > 0 > deepseek + + +def test_unresolvable_baseline_fails_open_to_zero(): + usage = _usage(fresh=2000, cached=0, written=0, out=500) + assert _savings("no-such-provider-xyz/no-such-model", "claude-haiku-4-5", usage) == 0.0 + + +def test_a_first_turn_is_the_rate_difference_not_a_switch_penalty(): + """Nothing was cached anywhere on a conversation's first turn, so the baseline would + have paid the same cache write. Charging it to the selected arm alone reported a + fraction of the real saving; on this shape roughly 4% of it. + """ + usage = _usage(fresh=0, cached=0, written=20_000, out=1_000) + first_turn = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, continuing=False) + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + both_write = (20_000 * opus["cache_creation_input_token_cost"] + 1_000 * opus["output_cost_per_token"]) - ( + 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] + ) + assert first_turn == pytest.approx(both_write) + + mid_conversation = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage) + assert first_turn > mid_conversation * 10, "a first turn must not be priced as a switch" + + +def test_a_first_turn_that_saves_money_never_reports_a_loss(): + """The write premium is fixed by prompt size while the saving grows with completion + length, so charging the write to a first turn made short answers over a large cached + prompt read as losses on requests that genuinely saved. That is the shape most likely + to be on the dashboard, and the sign has to be right. + """ + short_answer = _usage(fresh=0, cached=0, written=20_000, out=200) + assert _savings("anthropic/claude-opus-5", "claude-haiku-4-5", short_answer, continuing=False) > 0 + assert _savings("anthropic/claude-opus-5", "claude-haiku-4-5", short_answer) < 0 + + +def test_an_undetermined_conversation_shape_stays_conservative(): + """The default must charge the write. A caller that cannot be read, or a surface the + router never classified, has said nothing about whether the baseline was warm, and a + savings figure must not inflate on a guess. + """ + usage = _usage(fresh=0, cached=0, written=20_000, out=1_000) + defaulted = compute_autorouter_savings( + baseline_model="anthropic/claude-opus-5", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=usage, + ) + assert defaulted == pytest.approx(_savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage)) + assert defaulted < _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, continuing=False) + + +def test_a_continuing_turn_on_the_same_model_writes_its_growth_on_both_arms(): + """A conversation that grew by a few tokens writes those on whatever model serves + it, and they are new to every model, so the baseline would have written them too. + Moving them into the baseline's read bucket forgives it a write it really owes and + shrinks the reported saving on ordinary steady-state traffic. + """ + usage = _usage(fresh=0, cached=19_900, written=100, out=1_000) + opus = litellm.get_model_info("claude-opus-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + + def cost(info: dict) -> float: + return ( + 19_900 * info["cache_read_input_token_cost"] + + 100 * info["cache_creation_input_token_cost"] + + 1_000 * info["output_cost_per_token"] + ) + + both_write_the_growth = cost(opus) - cost(haiku) + assert _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage) == pytest.approx(both_write_the_growth) + + +def test_a_switch_onto_a_partly_cached_model_still_pays_for_the_write(): + """A model holding a small prefix of this prompt still has to write the rest, and + that write is the switch's cost. Keying the same-model case off reading *anything* + rather than reading *most of it* would hand this request the full rate gap and + inflate the saving by an order of magnitude. + """ + mostly_written = _usage(fresh=0, cached=500, written=19_500, out=1_000) + reported = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", mostly_written) + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + if_treated_as_same_model = ( + 500 * opus["cache_read_input_token_cost"] + + 19_500 * opus["cache_creation_input_token_cost"] + + 1_000 * opus["output_cost_per_token"] + ) - ( + 500 * haiku["cache_read_input_token_cost"] + + 19_500 * haiku["cache_creation_input_token_cost"] + + 1_000 * haiku["output_cost_per_token"] + ) + assert reported < if_treated_as_same_model / 10, "a mostly-cold switch must not be priced as a continuation" + +def test_a_baseline_that_prices_caching_implicitly_still_pays_for_its_prompt(): + """OpenAI, Azure and Gemini entries carry no `cache_creation_input_token_cost`, + because those providers cache implicitly and charge nothing to write. Leaving this + request's written tokens in the creation bucket priced them at the 0.0 the cost + resolver falls back to, so the baseline carried a 20k prompt for free and a first + turn that saved money reported a loss. Those tokens are plain input on such a model. + """ + first_turn = _usage(fresh=0, cached=0, written=20_000, out=1_000) + reported = compute_autorouter_savings( + baseline_model="gpt-5", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=first_turn, + conversation_continuing=False, + ) + + gpt5 = litellm.get_model_info("gpt-5", "openai") + assert gpt5.get("cache_creation_input_token_cost") is None, "pick a baseline with no cache-write rate" + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + baseline_pays_input = 20_000 * gpt5["input_cost_per_token"] + 1_000 * gpt5["output_cost_per_token"] + actually_paid = ( + 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] + ) + assert reported == pytest.approx(baseline_pays_input - actually_paid) + assert reported > 0, "routing a cold first turn onto a cheaper model is a saving, not a loss" + + +def test_a_baseline_with_no_cache_read_rate_is_charged_its_input_rate(): + """The same hole on the other bucket. A baseline whose entry has no + `cache_read_input_token_cost` reads for 0.0, so a continuing turn priced the whole + prompt at nothing and every switch away from it reported a loss. + """ + continuing = _usage(fresh=0, cached=0, written=20_000, out=1_000) + reported = compute_autorouter_savings( + baseline_model="xai/grok-4", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=continuing, + conversation_continuing=True, + ) + + grok = litellm.get_model_info("grok-4", "xai") + assert grok.get("cache_read_input_token_cost") is None, "pick a baseline with no cache-read rate" + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + baseline_pays_input = 20_000 * grok["input_cost_per_token"] + 1_000 * grok["output_cost_per_token"] + actually_paid = ( + 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] + ) + assert reported == pytest.approx(baseline_pays_input - actually_paid) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 3a94b1e0f85..2f249241f21 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -4094,6 +4094,23 @@ class TestRecordRoutingDecision: assert request_kwargs == {} + def test_clearing_the_decision_takes_the_savings_facts_with_it(self): + """A fallback to a plain model group re-enters the hook with the same + `request_kwargs`. The baseline and the conversation shape ride inside the + decision rather than beside it, so one clear cannot leave either behind and + attribute an auto-router saving to a deployment that never routed.""" + decision = { + "router_model_name": "smart-router", + "router_type": "complexity", + "routed_model": "gpt-4o-mini", + "savings_baseline_model": "anthropic/claude-opus-5", + "conversation_continuing": False, + } + request_kwargs: Dict = {"litellm_metadata": {"routing_decision": decision}} + Router._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None) + assert request_kwargs["litellm_metadata"] == {} + + class TestEscalationIsRecordedConsistently: """An escalation keyword records two separate facts on every path: that the caller asked, and whether the tier actually moved. Dropping the ask when there is nowhere @@ -5039,3 +5056,142 @@ class TestClassifierTrustBoundary: assert "Classify only the current message" not in system_prompt assert "using the earlier turns quoted above it as context" in system_prompt assert "rate the work it approves rather than the reply itself" in system_prompt + + +class TestConversationShapeDiscriminator: + """Whether the counterfactual single model would already have had the prompt cached.""" + + @staticmethod + def _router(mock_router_instance, basic_config) -> ComplexityRouter: + return ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, "session_affinity": False}, + ) + + @pytest.mark.asyncio + async def test_a_single_ask_is_a_first_turn(self, mock_router_instance, basic_config): + """Nothing is cached for any model yet, so the baseline would have paid the same + cache write and the saving is the plain rate difference.""" + mock_router_instance.cache = DualCache() + result = await self._router(mock_router_instance, basic_config).async_pre_routing_hook( + model="test-model", + request_kwargs={"metadata": {}}, + messages=[{"role": "user", "content": "Hello!"}], + ) + assert result.routing_decision["conversation_continuing"] is False + + @pytest.mark.asyncio + async def test_a_second_ask_means_the_baseline_was_already_warm(self, mock_router_instance, basic_config): + """An earlier turn was served, so a single-model deployment wrote the prompt then + and would only read it now; this request's write is what switching cost.""" + mock_router_instance.cache = DualCache() + result = await self._router(mock_router_instance, basic_config).async_pre_routing_hook( + model="test-model", + request_kwargs={"metadata": {}}, + messages=[ + {"role": "user", "content": "First question about the codebase"}, + {"role": "assistant", "content": "Here is the answer"}, + {"role": "user", "content": "Hello!"}, + ], + ) + assert result.routing_decision["conversation_continuing"] is True + + @pytest.mark.asyncio + async def test_it_needs_no_session_id(self, mock_router_instance, basic_config): + """The whole point of reading the conversation rather than remembering it: a + caller that sends no session header is still classified correctly.""" + mock_router_instance.cache = DualCache() + router = self._router(mock_router_instance, basic_config) + first = await router.async_pre_routing_hook( + model="test-model", request_kwargs={}, messages=[{"role": "user", "content": "Hello!"}] + ) + later = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[ + {"role": "user", "content": "First question"}, + {"role": "assistant", "content": "Answer"}, + {"role": "user", "content": "Hello!"}, + ], + ) + assert first.routing_decision["conversation_continuing"] is False + assert later.routing_decision["conversation_continuing"] is True + + @pytest.mark.asyncio + async def test_it_touches_no_cache(self, mock_router_instance, basic_config): + """Reading the request instead of remembering it is what removes the routing-path + round-trip, and with it a cache failure that would read as a first turn.""" + cache = AsyncMock() + cache.async_get_cache = AsyncMock(return_value=None) + mock_router_instance.cache = cache + result = await self._router(mock_router_instance, basic_config).async_pre_routing_hook( + model="test-model", + request_kwargs={"metadata": {}}, + messages=[{"role": "user", "content": "Hello!"}], + ) + assert result.routing_decision["conversation_continuing"] is False + assert cache.async_get_cache.await_count == 0 + assert cache.async_set_cache.await_count == 0 + + @pytest.mark.parametrize( + "history", + [ + pytest.param( + [ + {"role": "user", "content": "do X"}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "1", "name": "t", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "1", "content": "r"}]}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "2", "name": "t", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "2", "content": "r"}]}, + ], + id="messages-api-tool-result-blocks", + ), + pytest.param( + [ + {"role": "user", "content": "do X"}, + {"role": "assistant", "tool_calls": [{"id": "1"}]}, + {"role": "tool", "tool_call_id": "1", "content": "r"}, + ], + id="chat-completions-tool-role", + ), + ], + ) + def test_an_agent_loop_on_one_human_ask_is_not_a_first_turn(self, history): + """An agent can run twenty turns on a single human ask: its tool traffic rides + `tool_result` blocks that flatten to empty text and `tool` roles. Counting human + asks read that as a first turn and handed it the untouched-write arithmetic, + which is the one direction this must never fail in, because it inflates.""" + from litellm.router_strategy.complexity_router.complexity_router import _conversation_is_continuing + + assert _conversation_is_continuing(history) is True + + def test_a_system_prompt_does_not_make_a_first_turn_look_continued(self): + from litellm.router_strategy.complexity_router.complexity_router import _conversation_is_continuing + + assert _conversation_is_continuing([{"role": "system", "content": "s"}, {"role": "user", "content": "hi"}]) is False + + def test_unreadable_messages_stay_conservative(self): + """No messages says nothing about the baseline's cache, so it keeps charging the + write and under-claims rather than inflating.""" + from litellm.router_strategy.complexity_router.complexity_router import _conversation_is_continuing + + assert _conversation_is_continuing(None) is True + assert _conversation_is_continuing([]) is True + assert _conversation_is_continuing([{"role": "user", "content": ""}]) is False + + @pytest.mark.asyncio + async def test_the_shape_travels_on_every_pre_routing_response(self): + """A response without it defaults to charging the write, silently undoing the fix + for whichever routing path forgot it.""" + import inspect + + from litellm.router_strategy.complexity_router import complexity_router as module + + source = inspect.getsource(module.ComplexityRouter.async_pre_routing_hook) + inspect.getsource( + module.ComplexityRouter._classify_and_route + ) + builds = source.split("self._build_routing_decision(")[1:] + assert builds + missing = [i for i, block in enumerate(builds) if "conversation_continuing=conversation_continuing" not in block.split("),")[0]] + assert not missing, f"routing decisions {missing} do not carry the conversation shape" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 0f4eacc820c..1ea4113b821 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -23521,6 +23521,11 @@ export interface components { * @default 0 */ total_api_requests: number; + /** + * Total Autorouter Savings Spend + * @default 0 + */ + total_autorouter_savings_spend: number; /** * Total Cache Creation Input Tokens * @default 0 @@ -31427,6 +31432,11 @@ export interface components { * @default 0 */ api_requests: number; + /** + * Autorouter Savings Spend + * @default 0 + */ + autorouter_savings_spend: number; /** * Cache Creation Input Tokens * @default 0 From 22f68c0c6ba334369ab619994b5adb39c7d9f4c6 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 3 Aug 2026 20:46:09 -0700 Subject: [PATCH 067/124] fix(spend): read what a request cost from the record instead of pricing it again (#35736) The auto-router savings driver recomputes what the served request cost, but that request is not a counterfactual: it ran, and the cost calculator already billed it and wrote the number down. Recomputing means restating every pricing dimension the biller applied, and the two this missed were enough to halve it. A request billed at a priority tier is recomputed at standard rates, and a regional host's uplift is dropped entirely, so the driver writes a savings figure into the same rollup row as the `spend` it disagrees with. On `gpt-5.4-mini` at priority the row is billed 0.024 and the driver prices the same usage at 0.012. Neither omission cancels between the two arms, because both are per-model. The uplift is a multiplier read off each model's own entry, so 1.1*A - 1.1*B is 1.1*(A-B) and a model without one does not move at all. Tier coverage is sparser and asymmetric: `gpt-5.6` has priority rates and `gpt-5.4-nano` has none. `cost_breakdown` already carries the answer and already reaches the call site. The cost calculator records it, it rides the standard logging payload into the spend log's metadata, and OTEL, the log drawer and the response headers all read it rather than re-deriving; this driver was the only downstream consumer in the tree still pricing a completed request from its tokens. `input_cost` and `output_cost` sum to exactly what the pricer returns, so the served arm reads them. Tool spend, discount and margin stay out, since the counterfactual cannot be priced with them and charging them to one arm alone would read as the router losing money on every tool call. The baseline never ran, so it is still priced through the cost engine, now on the basis the biller used. `CostBreakdown` carries that basis because it cannot be recovered afterwards: the tier the biller used comes from `optional_params`, which no log record keeps, and the served tier that does survive on the usage object is a different fact with the opposite precedence. Rows written before this shipped carry no basis and price at standard rates, exactly as they do today; there is no backfill. Two smaller things in the same path. The router is passed as a provider rather than a router, so a spend write that was never auto-routed no longer fetches and discards one, and the complexity router resolves its messages once per hook instead of once per consumer. --- litellm/cost_calculator.py | 11 ++ litellm/litellm_core_utils/litellm_logging.py | 6 + litellm/proxy/db/db_spend_update_writer.py | 3 +- litellm/proxy/spend_tracking/savings.py | 124 ++++++++++++++++-- .../complexity_router/complexity_router.py | 15 ++- litellm/types/utils.py | 11 +- .../proxy/spend_tracking/test_savings.py | 83 ++++++++++++ 7 files changed, 239 insertions(+), 14 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index f04a9d61d4a..25d448e48e9 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1051,6 +1051,8 @@ def _store_cost_breakdown_in_logging_obj( cache_read_cost: float | None = None, cache_creation_cost: float | None = None, reasoning_cost: float | None = None, + service_tier: str | None = None, + data_residency: str | None = None, ) -> None: """ Helper function to store cost breakdown in the logging object. @@ -1068,6 +1070,8 @@ def _store_cost_breakdown_in_logging_obj( margin_percent: Margin percentage applied (0.10 = 10%) margin_fixed_amount: Fixed margin amount in USD margin_total_amount: Total margin added in USD + service_tier: Tier the costs above were priced on, already resolved + data_residency: Region uplift the costs above were priced on, already resolved """ if litellm_logging_obj is None: return @@ -1089,6 +1093,8 @@ def _store_cost_breakdown_in_logging_obj( cache_read_cost=cache_read_cost, cache_creation_cost=cache_creation_cost, reasoning_cost=reasoning_cost, + service_tier=service_tier, + data_residency=data_residency, ) except Exception as breakdown_error: @@ -1469,6 +1475,8 @@ def completion_cost( margin_percent=margin_percent, margin_fixed_amount=margin_fixed_amount, margin_total_amount=margin_total_amount, + service_tier=service_tier, + data_residency=data_residency, ) return _final_cost @@ -1657,6 +1665,8 @@ def completion_cost( cache_read_cost=_cache_read_cost, cache_creation_cost=_cache_creation_cost, reasoning_cost=_reasoning_cost, + service_tier=service_tier, + data_residency=data_residency, ) return _final_cost @@ -2351,6 +2361,7 @@ def handle_realtime_stream_cost_calculation( cost_for_built_in_tools_cost_usd_dollar=0.0, total_cost_usd_dollar=total_cost, additional_costs={"transcription_cost": transcription_cost} if transcription_cost > 0 else None, + data_residency=data_residency, ) return total_cost diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 66d82bd18f1..f38b2859259 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1286,6 +1286,8 @@ class Logging(LiteLLMLoggingBaseClass): cache_read_cost: float | None = None, cache_creation_cost: float | None = None, reasoning_cost: float | None = None, + service_tier: str | None = None, + data_residency: str | None = None, ) -> None: """ Helper method to store cost breakdown in the logging object. @@ -1302,6 +1304,8 @@ class Logging(LiteLLMLoggingBaseClass): margin_percent: Margin percentage applied (0.10 = 10%) margin_fixed_amount: Fixed margin amount in USD margin_total_amount: Total margin added in USD + service_tier: Tier the costs above were priced on, already resolved + data_residency: Region uplift the costs above were priced on, already resolved """ self.cost_breakdown = CostBreakdown( @@ -1309,6 +1313,8 @@ class Logging(LiteLLMLoggingBaseClass): output_cost=output_cost, total_cost=total_cost, tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar, + service_tier=service_tier, + data_residency=data_residency, ) if cache_read_cost is not None and cache_read_cost > 0: self.cost_breakdown["cache_read_cost"] = cache_read_cost diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 7c9ee96e809..b24ed4b8282 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -1885,8 +1885,9 @@ class DBSpendUpdateWriter: cache_read_input_tokens=cache_read_input_tokens, routing_decision=_metadata.get("routing_decision"), model_id=payload.get("model_id"), - llm_router=_get_llm_router(), + llm_router=_get_llm_router, usage_object=usage_obj, + cost_breakdown=_metadata.get("cost_breakdown"), ) daily_transaction = BaseDailySpendTransaction( diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 1a1c813d784..a318b11e4c0 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -8,7 +8,7 @@ are known) and summed into the daily tables; tokens cannot be priced after they have been aggregated across models. """ -from collections.abc import Mapping +from collections.abc import Callable, Mapping from typing import TYPE_CHECKING, NamedTuple import litellm @@ -105,11 +105,81 @@ def _model_info(model: _ModelIdentity) -> ModelInfo | None: return None -def _cost_of_usage(model: _ModelIdentity, usage: Usage, model_info: ModelInfo | None = None) -> float | None: +class PricingBasis(NamedTuple): + """The tier and region a request was priced on, as the cost calculator resolved them. + + Read back off the request's recorded ``cost_breakdown`` rather than re-derived. The + tier the biller used comes from ``optional_params``, which no log record carries, and + the served tier that does survive on the usage object is a different fact with the + opposite precedence, so a spend-time re-derivation would disagree with the invoice on + exactly the requests where the tier changed the price. + """ + + service_tier: str | None = None + data_residency: str | None = None + + +_STANDARD_RATES = PricingBasis() + + +def _pricing_basis(cost_breakdown: Mapping[str, object] | None) -> PricingBasis: + """The basis recorded on a request, defaulting to standard rates when absent. + + Rows written before this field shipped carry neither key, and there is no backfill: + they price at standard rates, which is what they already did. + + Both values survive a JSON round trip on the way here, so neither is guaranteed to be + a string. `generic_cost_per_token` calls `.lower()` on both without a type check, and + the resulting `AttributeError` would be swallowed into a silent zero by the caller's + `except`, so anything that is not a string is dropped here instead. + """ + if not cost_breakdown: + return _STANDARD_RATES + service_tier = cost_breakdown.get("service_tier") + data_residency = cost_breakdown.get("data_residency") + return PricingBasis( + service_tier=service_tier if isinstance(service_tier, str) else None, + data_residency=data_residency if isinstance(data_residency, str) else None, + ) + + +def _recorded_token_cost(cost_breakdown: Mapping[str, object] | None) -> float | None: + """What the biller charged for this request's tokens, or ``None`` when unrecorded. + + ``input_cost`` already carries the cache buckets, so it and ``output_cost`` sum to + exactly what `generic_cost_per_token` returns for the same request; the separate + ``cache_read_cost`` and ``cache_creation_cost`` entries decompose that sum rather than + adding to it, and including them would charge those tokens twice. + + Built-in tool cost, discount and margin are deliberately left out. They are properties + of the request and the operator's contract rather than of the model the router picked, + so they land on both sides of the comparison or neither, and only the total the + counterfactual can also be priced on belongs here. + """ + if not cost_breakdown: + return None + input_cost = cost_breakdown.get("input_cost") + output_cost = cost_breakdown.get("output_cost") + if not isinstance(input_cost, (int, float)) or not isinstance(output_cost, (int, float)): + return None + return float(input_cost) + float(output_cost) + + +def _cost_of_usage( + model: _ModelIdentity, + usage: Usage, + model_info: ModelInfo | None = None, + basis: PricingBasis = _STANDARD_RATES, +) -> float | None: """What ``usage`` costs on ``model``, or ``None`` when the model has no pricing.""" try: prompt_cost, completion_cost = generic_cost_per_token( - model=model.model, usage=usage, custom_llm_provider=model.provider, model_info=model_info + model=model.model, + usage=usage, + custom_llm_provider=model.provider, + service_tier=basis.service_tier, + data_residency=basis.data_residency, + model_info=model_info, ) except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings verbose_proxy_logger.debug( @@ -228,6 +298,7 @@ def compute_autorouter_savings( usage: Usage, conversation_continuing: bool = True, selected_info: ModelInfo | None = None, + cost_breakdown: Mapping[str, object] | None = None, ) -> float: """Net dollars the router saved, or cost, by serving this request on ``selected_model``. @@ -237,6 +308,20 @@ def compute_autorouter_savings( dashboard has to be able to say so. Zero when both sides resolve to the same deployment, or when either cannot be resolved or priced. + Only one side of this subtraction is a counterfactual. What the request cost on the + model that served it is a number the operator was actually billed, and the cost + calculator already wrote it down, so ``cost_breakdown`` is read rather than + re-derived. Recomputing it means restating every pricing dimension the biller + applied, and each one omitted is a silent disagreement with the ``spend`` column + beside it; a request billed at a priority tier recomputed at standard rates reads as + half its real cost. + + The baseline has no such record, since it never ran, so it is priced through the same + cost engine on the basis the biller used for this request. An operator running that + one model instead of the router would have sent this request to the same tier and the + same region, because both are properties of the request and the deployment's + contract, not of which model the router happened to pick. + ``conversation_continuing`` says whether the baseline would already have had this prompt cached. It defaults to True because that is the conservative reading: a request whose shape the router could not determine is charged the write and @@ -255,11 +340,16 @@ def compute_autorouter_savings( # name alone reports as zero. if baseline == selected: return 0.0 + basis = _pricing_basis(cost_breakdown) baseline_info = _model_info(baseline) baseline_cost = _cost_of_usage( - baseline, _baseline_usage(usage, conversation_continuing, baseline_info), baseline_info + baseline, _baseline_usage(usage, conversation_continuing, baseline_info), baseline_info, basis ) - selected_cost = _cost_of_usage(selected, usage, selected_info) + # Falls back to pricing the request only when the biller recorded nothing, which is + # every row written before the breakdown carried its basis. + selected_cost = _recorded_token_cost(cost_breakdown) + if selected_cost is None: + selected_cost = _cost_of_usage(selected, usage, selected_info, basis) if baseline_cost is None or selected_cost is None: return 0.0 return baseline_cost - selected_cost @@ -287,7 +377,8 @@ def compute_savings_spend( routing_decision: Mapping[str, object] | None = None, usage_object: Mapping[str, object] | None = None, model_id: str | None = None, - llm_router: "Router | None" = None, + llm_router: "Callable[[], Router | None] | None" = None, + cost_breakdown: Mapping[str, object] | None = None, ) -> SavingsSpend: """ Dollar savings for one request, split by optimization driver. @@ -299,6 +390,17 @@ def compute_savings_spend( baseline the router recorded on its ``routing_decision``, and are zero unless the two differ. That record also says whether the conversation was already underway, which is what tells a mid-conversation switch from a first turn. + + ``llm_router`` is passed as a provider rather than a router because every spend write + calls this and only auto-routed ones need one, so looking it up eagerly at the call + site would fetch and discard it on the rest. + + ``cost_breakdown`` is what the cost calculator recorded for this request, and it + carries both what the request really cost and the tier and region it was priced on. + Only the auto-router driver reads it. Compression and prompt caching price a + hypothetical token delta off flat rate keys, so they are blind to tiered pricing in + the same way; that is pre-existing behaviour on two shipped drivers rather than + something introduced here, and moving those numbers is its own change. """ input_cost, cache_read_cost = _input_and_cache_read_cost(model, custom_llm_provider) compression = max(compression_saved_tokens, 0) * input_cost @@ -311,19 +413,23 @@ def compute_savings_spend( # The counterfactual is one model an operator would have run instead of the router, # configured once for the proxy rather than derived per request. Unset means the # driver is off; a routing decision is what says this request was auto-routed at all. + # Both are checked before anything is resolved, because every spend write reaches + # here and only auto-routed ones can produce a number. + baseline_model = litellm.autorouter_savings_baseline_model decision = routing_decision if isinstance(routing_decision, Mapping) else {} autorouter = ( compute_autorouter_savings( - baseline_model=litellm.autorouter_savings_baseline_model, + baseline_model=baseline_model, selected_model=model, selected_provider=custom_llm_provider, usage=usage, # Absent means the router never recorded a shape, which is the conservative # reading: charge the cache write rather than claim a first turn's saving. conversation_continuing=decision.get("conversation_continuing") is not False, - selected_info=_effective_model_info(llm_router, model_id, model or ""), + selected_info=_effective_model_info(llm_router() if llm_router else None, model_id, model or ""), + cost_breakdown=cost_breakdown, ) - if decision + if decision and baseline_model else 0.0 ) return SavingsSpend(compression=compression, prompt_caching=prompt_caching, autorouter=autorouter) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index ca2e7fcc038..11a1b686c2e 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1428,7 +1428,11 @@ class ComplexityRouter(CustomLogger): if isinstance(metadata, dict): metadata[RETURN_RAW_MODEL_NAME_METADATA_KEY] = True - conversation_continuing = _conversation_is_continuing(self._resolve_messages(messages, request_kwargs)) + # Resolved once for the whole hook. Resolution converts Responses API input into + # chat-completions messages, so it is real work on every non-chat surface, and + # both the conversation shape and the classifier read the same list. + resolved_messages = self._resolve_messages(messages, request_kwargs) + conversation_continuing = _conversation_is_continuing(resolved_messages) use_session_affinity = self.config.session_affinity and not self.config.plugins session_id = self._get_session_id_from_request_kwargs(request_kwargs) if use_session_affinity else None @@ -1440,7 +1444,6 @@ class ComplexityRouter(CustomLogger): routed_model: str | None = pinned_model pin_escalation_keyword: str | None = None if self.escalation_keywords: - resolved_messages = self._resolve_messages(messages, request_kwargs) user_message = _newest_turn_ask(resolved_messages) if resolved_messages else None if user_message is not None: pin_escalation_keyword = self._matched_escalation_keyword(user_message) @@ -1487,6 +1490,7 @@ class ComplexityRouter(CustomLogger): input=input, specific_deployment=specific_deployment, conversation_continuing=conversation_continuing, + resolved_messages=resolved_messages, ) if cache_key is not None and response is not None: await self.litellm_router_instance.cache.async_set_cache( @@ -1504,6 +1508,7 @@ class ComplexityRouter(CustomLogger): input: str | list | None = None, specific_deployment: bool | None = False, conversation_continuing: bool = True, + resolved_messages: Sequence[Mapping[str, object]] | None = None, ) -> PreRoutingHookResponse | None: """ Classifies the request by complexity and returns the appropriate model. @@ -1516,13 +1521,17 @@ class ComplexityRouter(CustomLogger): messages: The messages in the request. input: Optional input for Responses API or embeddings. specific_deployment: Whether a specific deployment was requested. + resolved_messages: Messages the caller already resolved, to avoid converting + the request format a second time. Resolved here when absent, so a direct + caller does not have to. Returns: PreRoutingHookResponse with the routed model, or None if no routing needed. """ from litellm.types.router import PreRoutingHookResponse - resolved_messages = self._resolve_messages(messages, request_kwargs) + if resolved_messages is None: + resolved_messages = self._resolve_messages(messages, request_kwargs) if not resolved_messages: verbose_router_logger.debug("ComplexityRouter: No messages could be resolved, skipping routing") diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 21d69d9925a..c63f4d53572 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2991,9 +2991,18 @@ class CachingDetails(TypedDict): class CostBreakdown(TypedDict, total=False): """ - Detailed cost breakdown for a request + Detailed cost breakdown for a request. + + ``service_tier`` and ``data_residency`` record the pricing basis the cost was + computed on, not what the caller asked for. A consumer that has to price a + counterfactual against this request (what another model would have charged for + it) needs the same basis to compare like with like, and re-deriving it from the + request is not possible after the fact: the tier the biller used comes from + ``optional_params``, which no log record carries. """ + service_tier: Optional[str] + data_residency: Optional[str] input_cost: float # Cost of raw (non-cached) input tokens only cache_read_cost: float # Cost of cache-read tokens (discounted rate) cache_creation_cost: float # Cost of cache-write tokens (premium rate) diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 6d0195b632b..1d6b0da561d 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -534,3 +534,86 @@ def test_a_baseline_with_no_cache_read_rate_is_charged_its_input_rate(): 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] ) assert reported == pytest.approx(baseline_pays_input - actually_paid) + + +def _breakdown(input_cost: float, output_cost: float = 0.0, **extra: object) -> dict: + """A `cost_breakdown` as the cost calculator records it on the spend log.""" + return {"input_cost": input_cost, "output_cost": output_cost, **extra} + + +def test_the_served_arm_is_read_from_the_record_not_repriced(): + """What the request cost on the model that served it is not a counterfactual; the + cost calculator already billed it and wrote the number down. Recomputing it restates + every pricing dimension the biller applied and drops the ones it forgets, so the + driver disagrees with the `spend` column beside it. + + Pinned with a negotiated rate no public map lookup can produce, so re-pricing from + the model name cannot land on this number. Tool spend and margin are recorded too and + must stay out: the baseline cannot be priced with them, so charging them to the + served arm alone would read as the router losing money on every tool call. + """ + usage = _usage(fresh=20_000, cached=0, written=0, out=1_000) + negotiated_input, negotiated_output = 0.0123, 0.0456 + + reported = compute_autorouter_savings( + baseline_model="anthropic/claude-opus-5", + selected_model="gpt-5.5", + selected_provider="openai", + usage=usage, + conversation_continuing=False, + cost_breakdown=_breakdown( + negotiated_input, + negotiated_output, + tool_usage_cost=5.0, + margin_total_amount=2.0, + total_cost=negotiated_input + negotiated_output + 7.0, + ), + ) + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + public = 20_000 * opus["input_cost_per_token"] + 1_000 * opus["output_cost_per_token"] + assert reported == pytest.approx(public - (negotiated_input + negotiated_output)) + + +@pytest.mark.parametrize( + "basis, expected_multiplier", + [ + pytest.param({"service_tier": "priority"}, 2.0, id="priority tier doubles the baseline"), + pytest.param({"data_residency": "eu"}, 1.1, id="eu residency uplifts the baseline"), + pytest.param({}, 1.0, id="no basis recorded prices at standard"), + pytest.param(None, 1.0, id="row predating the field prices at standard"), + pytest.param({"service_tier": True, "data_residency": 17}, 1.0, id="a non-string basis is dropped"), + ], +) +def test_the_baseline_is_priced_on_the_basis_the_request_was_billed_at(basis, expected_multiplier): + """A request billed at a priority tier, or through a regional host, would have been + billed the same way on the single model an operator ran instead of the router, so the + counterfactual carries that basis too. Dropping it prices the two arms from different + books; neither multiplier cancels out of the difference, because both are per-model. + + The served model has no tiered rates and no uplift of its own, so only the baseline + can move: a fix that forwards the basis to the served arm alone leaves these numbers + unchanged. The non-string case guards the JSON round trip, where `.lower()` inside + the pricer would raise and be swallowed into a silent $0.00 for the whole row. + """ + gpt = litellm.get_model_info("gpt-5.5", "openai") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + assert gpt.get("input_cost_per_token_priority") == 2 * gpt["input_cost_per_token"] + assert gpt.get("regional_processing_uplift_multiplier_eu") == 1.1 + assert haiku.get("input_cost_per_token_priority") is None, "served model must not move with the basis" + assert haiku.get("regional_processing_uplift_multiplier_eu") is None + + usage = _usage(fresh=20_000, cached=0, written=0, out=1_000) + served = 20_000 * haiku["input_cost_per_token"] + 1_000 * haiku["output_cost_per_token"] + + reported = compute_autorouter_savings( + baseline_model="openai/gpt-5.5", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=usage, + conversation_continuing=False, + cost_breakdown=None if basis is None else _breakdown(served, **basis), + ) + + baseline = 20_000 * gpt["input_cost_per_token"] + 1_000 * gpt["output_cost_per_token"] + assert reported == pytest.approx(expected_multiplier * baseline - served) From 042ef4cc487974dacc3dab8486663107b6dd4763 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Tue, 4 Aug 2026 05:47:43 +0200 Subject: [PATCH 068/124] perf: install hiredis so redis-py parses replies with its C parser (#35709) --- pyproject.toml | 3 + tests/test_litellm/test_redis.py | 19 +++++++ uv.lock | 94 ++++++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index addf56a3588..9b9fa00f5ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,9 @@ proxy = [ "pyyaml>=6.0.3,<7.0", "rq>=2.7.0,<3.0", "orjson>=3.11.6,<4.0", + # redis-py's C response parser. It arrives with redis (via rq) either way; naming + # it here is what makes redis-py select _HiredisParser instead of the Python one. + "hiredis>=3.0.0,<4.0", "apscheduler>=3.11.2,<4.0", "fastapi-sso>=0.19.0,<1.0", "PyJWT>=2.13.0,<3.0", diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index e0fa800723d..14c242f1096 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -845,3 +845,22 @@ def test_url_config_drops_kwargs_the_connection_cannot_accept(client_only_kwarg, assert pool is not None pool.make_connection() assert pool.connection_kwargs.get("socket_timeout") == 5.0 + + +def test_redis_uses_the_hiredis_response_parser(): + """The C parser must be the one redis-py actually picks. + + hiredis is declared in the `proxy` extra purely for speed; nothing imports it, so + dropping it from pyproject.toml would silently fall back to the pure-Python parser + with no other symptom. redis-py selects it at import time, so asserting on the + selection is what catches that. + """ + from redis._parsers import _HiredisParser + from redis.connection import HIREDIS_AVAILABLE, DefaultParser + + assert HIREDIS_AVAILABLE, "hiredis is not installed; redis-py fell back to the pure-Python parser" + assert DefaultParser is _HiredisParser, f"redis-py selected {DefaultParser.__name__}, expected _HiredisParser" + + client = get_redis_client(host="redis-host", port=6379) + connection = client.connection_pool.make_connection() + assert isinstance(connection._parser, _HiredisParser) diff --git a/uv.lock b/uv.lock index 15e65c9dffd..70048926945 100644 --- a/uv.lock +++ b/uv.lock @@ -3159,6 +3159,98 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/94/3b66b148778ee100dcfd69c2ca22b57b41b44d3063ceec934f209e9184ce/hf_xet-1.5.0-cp37-abi3-win_arm64.whl", hash = "sha256:b6c9df403040248c76d808d3e047d64db2d923bae593eb244c41e425cf6cd7be", size = 3806916, upload-time = "2026-05-06T06:18:21.7Z" }, ] +[[package]] +name = "hiredis" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1f/e2/1654d65851f39fd94e91a77a5655d09d4b64901fdc594020d8348db697b2/hiredis-3.4.0.tar.gz", hash = "sha256:da19331354433af6a2c54c21f2d70ba084933c0d7d2c43578ec5c5b446674ad5", size = 137169, upload-time = "2026-06-03T16:23:46.226Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/48/b0c0e2826eab7543c08980dfab871b3e7c83c47d7496134b04a94df55a1a/hiredis-3.4.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:69d0326f20354ce278cbb86f5ae47cb390e22bb94a66877031038af907c42fa5", size = 138470, upload-time = "2026-06-03T16:21:49.96Z" }, + { url = "https://files.pythonhosted.org/packages/6f/75/2a08a6062228720747570a07ab42e6f5826725f09c9a95d75b7b5b938022/hiredis-3.4.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:4863b99b1bf739eaa60961798efc709f657864fbf5a142cb9b99d3e36a37208e", size = 74496, upload-time = "2026-06-03T16:21:51.14Z" }, + { url = "https://files.pythonhosted.org/packages/e0/3d/28c61f9c628dfaf1f96bb8b8592cb006eaad3747248c95f9ae7f694abb47/hiredis-3.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:98e28c10e43d076f50ce9fa9f4017303d5796c3058b1b651f507c2a7d6ef402c", size = 70083, upload-time = "2026-06-03T16:21:52.125Z" }, + { url = "https://files.pythonhosted.org/packages/31/06/21e254be776b6ecd38e7c955c2fe205f2828c091621fbe400406bd2e382e/hiredis-3.4.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6774f1fe2723001ca0cd42bf5d8b1235301226273915c581c5c1260d4d114c43", size = 304408, upload-time = "2026-06-03T16:21:53.148Z" }, + { url = "https://files.pythonhosted.org/packages/fc/c8/46d48dc674d0aacc66cdc8c400261fad3c08b352dd823e6ffa0a0536259d/hiredis-3.4.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:12eca9aea1450d1a85dc15574a985c227e52abbc2b6466f48ad2aa3b82124701", size = 336932, upload-time = "2026-06-03T16:21:54.391Z" }, + { url = "https://files.pythonhosted.org/packages/50/29/05a3cf8f605f6cdeec2c6f54d022fa51242e3cf77fe4940e89fe1446b068/hiredis-3.4.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:12ea5facb5b08fa23e4c101ec2151f3a3de8ecec412fec58dbde0a6eebca02c7", size = 347528, upload-time = "2026-06-03T16:21:55.541Z" }, + { url = "https://files.pythonhosted.org/packages/26/c9/4e9cd249afc101ac283943295fe3359bdd711a0bb8c667752eb0da80609d/hiredis-3.4.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4de6869be2b33490569dae0712366bb794b7f5e7a8b674de3e092b3e95712d6d", size = 310142, upload-time = "2026-06-03T16:21:56.534Z" }, + { url = "https://files.pythonhosted.org/packages/4a/71/d069db71ba4a5f40bb1390eebaf00e4d161c5c1f48e623880ca22a946618/hiredis-3.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4190bd07dd7879a8a7ddbb2a4f74d402721f3898276e35beb98851b85b5f539c", size = 298868, upload-time = "2026-06-03T16:21:57.559Z" }, + { url = "https://files.pythonhosted.org/packages/0f/cd/83793cc2fb161ddd5d394adc7122ec023b0e1d9289a294d0f80214d910bf/hiredis-3.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e29267ecdd08758926f1a9221af2671d90f475480c40aff409921b1f362f1bd5", size = 328564, upload-time = "2026-06-03T16:21:58.57Z" }, + { url = "https://files.pythonhosted.org/packages/3d/fa/66fed95ab85d85a4dc87acb8df69e22ff943a3bf7a26e791d5a1ff173577/hiredis-3.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:45c6c296056641b5df37cedafe7d1553f33bc247e2f81603a4d038b39261879b", size = 329725, upload-time = "2026-06-03T16:21:59.519Z" }, + { url = "https://files.pythonhosted.org/packages/a9/0d/ac2d1f1c30eb6d7ab5f099da17f76125f1bb0f9274623178508a6c736acf/hiredis-3.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f7c7596fbb2b5202e943180353958e89014e763c7f25877a92f70bbde6cd7f19", size = 309144, upload-time = "2026-06-03T16:22:00.691Z" }, + { url = "https://files.pythonhosted.org/packages/d6/41/ec1dc1c27e8fcbe2dc635d49cc751848972600a7d277569fb9ba77ee501c/hiredis-3.4.0-cp310-cp310-win32.whl", hash = "sha256:1bfb9ccfb13be63883e5f2e5ff7f6fc87bf256f8243af594257dfbed9dbc3cf0", size = 38820, upload-time = "2026-06-03T16:22:01.657Z" }, + { url = "https://files.pythonhosted.org/packages/24/9d/38b85c7fd3ec49c8b1b089288307f8f17e138439be7b079fab2221e113a8/hiredis-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:c2245c46b4ced5f689469e6dcdfc8a0895bf873840a6600f5ea759cdf1b26a8b", size = 40048, upload-time = "2026-06-03T16:22:02.457Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ce/bdc7602ddc7b60ada44be4c4246c1b4d54a0b444a2b5f17ec936c0ce0faf/hiredis-3.4.0-cp310-cp310-win_arm64.whl", hash = "sha256:0bcb630add6bc9ea136fce691ddff0c46aa91cb860df4ca789fe44127eb7e90d", size = 36849, upload-time = "2026-06-03T16:22:03.3Z" }, + { url = "https://files.pythonhosted.org/packages/92/d1/09d7323c76d097ff3f6530228d2422c19817b6052716f9a652ecd6e2f68e/hiredis-3.4.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:7f7fc1535f6e1a190089eae46dee25f0c6b72bb221d377be07092803b8208733", size = 138467, upload-time = "2026-06-03T16:22:04.09Z" }, + { url = "https://files.pythonhosted.org/packages/ed/27/c4ebeb0f7ecc8a23d4356efd3ef2b6243ed74d24584d86ff8065fa14a350/hiredis-3.4.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:ed1dba2695f6de009c67d63b39ff978cb43b8a79362f697acedffb7743e50d21", size = 74504, upload-time = "2026-06-03T16:22:04.998Z" }, + { url = "https://files.pythonhosted.org/packages/ae/d7/4f456f36f5c5224bc11a2fad964116a3cc37259d09dd840628aea5fdbf28/hiredis-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a3796094f616f72976ff51e4dc1a016e753c0f9af5393b2df96920b6bae1e19b", size = 70080, upload-time = "2026-06-03T16:22:05.76Z" }, + { url = "https://files.pythonhosted.org/packages/04/ba/a16d44b2bd71e72a10673faa94d07cc4e9de90240b65ce2511af0cce065b/hiredis-3.4.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ccc5c660e31d788ca534a20f2ccb7a80b946b960e18ed4e1db950fcac122b405", size = 304968, upload-time = "2026-06-03T16:22:06.614Z" }, + { url = "https://files.pythonhosted.org/packages/4a/3a/78ca23fe899f8da7ee2caf9c502ac1a63da15d521f33a3fc617a7adbf2e0/hiredis-3.4.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f3c67f39b112dc35f68d5b59ee111db6121f037d1a60cf3840ecffbb2ec5686b", size = 337465, upload-time = "2026-06-03T16:22:07.622Z" }, + { url = "https://files.pythonhosted.org/packages/ba/11/2df9a12f170e9d61739e7df5f06712141414b2dce2cf385fc1fb6f31a46b/hiredis-3.4.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bca175f02a2b0150ffe7f5dc8bf49c798f34d2c7024d17ace0ec97a7583560e3", size = 348293, upload-time = "2026-06-03T16:22:08.677Z" }, + { url = "https://files.pythonhosted.org/packages/88/07/716ffeb049377d92da6261c5563e554b82336ce3eafb11eb4510c5558be7/hiredis-3.4.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43004b0b48abc628dda1ac3ac4871e1326c126f8cd9f11164d61934d827d7a3b", size = 310697, upload-time = "2026-06-03T16:22:09.661Z" }, + { url = "https://files.pythonhosted.org/packages/5d/03/ef3697bdee359b4521101bdc16e8e4965a5ebd8634b605fc7cf9c01b6b82/hiredis-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8aaaab18314fd25453b5cf59c8cdca4110e419455bcb4c0737d19d4151513e75", size = 299377, upload-time = "2026-06-03T16:22:10.777Z" }, + { url = "https://files.pythonhosted.org/packages/bc/a7/2a12a2f828c2d611b74dcf2229998c4d2570fe6ed6b4903d6a4c3add84af/hiredis-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:5359caad5b57da0bce11d2880f22617ba3710f0866121a924745447848448034", size = 329008, upload-time = "2026-06-03T16:22:11.82Z" }, + { url = "https://files.pythonhosted.org/packages/66/a9/cdfda214af93eeb9f93a83a099d06f26ae5569f188209ddc8a7c977ed446/hiredis-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:44660a91e0fbc803c29b337c1a9194c8d7b4cd3a3868d28f747cbec2df165483", size = 330103, upload-time = "2026-06-03T16:22:12.935Z" }, + { url = "https://files.pythonhosted.org/packages/87/05/cdc7e2e07b56c716426db4644b917b260a4f6fdc8d16cc3bbac4b27d0a17/hiredis-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a315009b441a0105a373a9a780ebb1c6f7d9ead88ac6ea5f2a15791353c6f590", size = 309582, upload-time = "2026-06-03T16:22:14.157Z" }, + { url = "https://files.pythonhosted.org/packages/f2/36/304a0e029cb6e44add3b0d664315de25c483f6e8f8e1d413c68de969a3d0/hiredis-3.4.0-cp311-cp311-win32.whl", hash = "sha256:282c4310af72afbe18b07d416459f4febeaeb805a067a7df790136e0e550fcb2", size = 38823, upload-time = "2026-06-03T16:22:15.14Z" }, + { url = "https://files.pythonhosted.org/packages/f0/19/7ea1fdbee1c42cbac140005e66e60a1198548eea04456e17dab5c285e31b/hiredis-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:bb44efa4fa3e3ed7779ad0ade3c08ed5d75ca7a6336893e9a4f2722093b4168a", size = 40040, upload-time = "2026-06-03T16:22:15.886Z" }, + { url = "https://files.pythonhosted.org/packages/20/e4/2122980b75a3fa8980540e2265028c757564ecc4d813b40298d29dd876ea/hiredis-3.4.0-cp311-cp311-win_arm64.whl", hash = "sha256:4404c557fd49bcfe24dff41f1209e4221c76d1607df2fb2dfd39474b5b086dcb", size = 36851, upload-time = "2026-06-03T16:22:16.644Z" }, + { url = "https://files.pythonhosted.org/packages/d2/84/f74deb132d238a0d5a3eb1618bf7558c65230b279421f909a9753231c516/hiredis-3.4.0-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:9e88048a66dfffec7a3f578f2a2a0fd907c75b5bd85b3c9184f76f0149ea399f", size = 138679, upload-time = "2026-06-03T16:22:17.598Z" }, + { url = "https://files.pythonhosted.org/packages/a2/13/399fe51d399b8d4f5717aa68cb1dafcb8c244b19b1b9b0afaaa526c1be94/hiredis-3.4.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:8b3f1d03046765c0a83558bf1756811101e3947649c7ca22a71d9dc3c92929d1", size = 74657, upload-time = "2026-06-03T16:22:18.819Z" }, + { url = "https://files.pythonhosted.org/packages/a4/cf/6a0bcf454b1642997c4dd007bd89beada43f38b22781afdf475060e427ac/hiredis-3.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:24751054bb11353016d242d09a4a902ecf8f25e3b56fe396cccb6f056fdda016", size = 70115, upload-time = "2026-06-03T16:22:19.649Z" }, + { url = "https://files.pythonhosted.org/packages/98/99/62340215f80e59680c79ae5080c5422311da105870c57bbefc5d87487025/hiredis-3.4.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:258f820cdd6ee6be39ae6a8ea94a76b8856d34113de6604f63bc81327ef06240", size = 306481, upload-time = "2026-06-03T16:22:20.608Z" }, + { url = "https://files.pythonhosted.org/packages/f1/be/97f349e5bb0dcab0ef28b15523443d9bbe81f8ccbd3dadff56594dfa82fe/hiredis-3.4.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3774461209688790734b5db8934400a4456493fc1a172fb5298cc5d72201aceb", size = 339560, upload-time = "2026-06-03T16:22:21.861Z" }, + { url = "https://files.pythonhosted.org/packages/1e/3f/eb6a9632bcc13a3fbefce5de90090052fb1ae1cd3d57faf687f20149d592/hiredis-3.4.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ccdb63363c82ea9cea2d48126bc8e9241437b8b3b36413e967647a17add59643", size = 351549, upload-time = "2026-06-03T16:22:22.969Z" }, + { url = "https://files.pythonhosted.org/packages/1e/8c/440369f727dcb856f3eeda238d6e67781b180feaa831bd28997d8af10c3b/hiredis-3.4.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:452cff764acb30c106d1e33f1bdf03fa9d4a9b0a9c995d722d4d39c998b40582", size = 313066, upload-time = "2026-06-03T16:22:23.987Z" }, + { url = "https://files.pythonhosted.org/packages/ec/d1/3d76c4d5c46cd2e7b38641f7c8b325e0cab7d49d565ea573256eb3837d0c/hiredis-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1fb0a139cd52535f3e5a532816b5c36b3aea95817410fbf28ca4a676026347a5", size = 300827, upload-time = "2026-06-03T16:22:25.287Z" }, + { url = "https://files.pythonhosted.org/packages/c5/bc/d112dd9704ae47243a515fb021ec4d0b5a1b8d83a7a3eff3284c0248412d/hiredis-3.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:163d8c43e2706d23490532ea0de8736fc1493cfa52f0ee65f85b0f074f2fe017", size = 331284, upload-time = "2026-06-03T16:22:26.385Z" }, + { url = "https://files.pythonhosted.org/packages/e9/7b/8a4dc0a15e4658c81a9e79b2c167fbfbf750e0c1c7ef13e00e69d4273ced/hiredis-3.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4b8f52844cd260d7805eca55c834e3e06b4c0d5b53a4178143b92242c2517c0d", size = 332962, upload-time = "2026-06-03T16:22:27.392Z" }, + { url = "https://files.pythonhosted.org/packages/1d/52/d3d0bb234de8deb4cbd432cdc63d001a6cad1f9c05fe07d2fa652f8cf412/hiredis-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03374d663b0e025e4039757ef5fad02e3ff714f7a01e5b34c88de2a9c91359dc", size = 311698, upload-time = "2026-06-03T16:22:28.442Z" }, + { url = "https://files.pythonhosted.org/packages/04/5b/54a052eccaf901703b57d7c28509e74341fa0da08d770f485345397ea1e5/hiredis-3.4.0-cp312-cp312-win32.whl", hash = "sha256:696e0a2118e1df5ccacf8ecf8abe528cf0c4f1f1d867f64c34579bef77778cdb", size = 38921, upload-time = "2026-06-03T16:22:29.39Z" }, + { url = "https://files.pythonhosted.org/packages/a7/64/6508236eda66765fbe873d1d0a0722e38059302e96dc9915b162ff17b35a/hiredis-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:ee6b4beb79a71df67af15a8451366babc2687fcac674d5c6eacec4197e4ce8c1", size = 40090, upload-time = "2026-06-03T16:22:30.204Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1c/7333aba1b4b7cef2591b244140aec0f1aad903397bbaa31c1858722b2fe4/hiredis-3.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:14524fdc751e3960d78d848872576b5442b40baae3cac14fbab1ba7ac523891f", size = 36875, upload-time = "2026-06-03T16:22:31.087Z" }, + { url = "https://files.pythonhosted.org/packages/7b/e5/9e47dda8f1d55e77293c6cdf4169182b7f2f55b56913d1fb16a0ddf63a3d/hiredis-3.4.0-cp313-cp313-macosx_10_15_universal2.whl", hash = "sha256:4f0e3536eea76c03435d411099d165850bc3c9d873efe62843b995027135a763", size = 138688, upload-time = "2026-06-03T16:22:31.825Z" }, + { url = "https://files.pythonhosted.org/packages/1e/07/039bcf7ce8262ed66db736349c121486874826248ccd70c98c2f830ec9da/hiredis-3.4.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:82860f050aabd08c046f304eb57c105bb3d5a7370f79a4a0b74d2b771767cc13", size = 74666, upload-time = "2026-06-03T16:22:32.758Z" }, + { url = "https://files.pythonhosted.org/packages/29/6d/692c50d846a0a36578e9ef0c62c6193ce01a48f353f6961de9de88a30b37/hiredis-3.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:74bcfb26189939daba2a0eb4bad05a6a30773bb2461f3d9967b8ced224bd0de9", size = 70119, upload-time = "2026-06-03T16:22:33.692Z" }, + { url = "https://files.pythonhosted.org/packages/28/5d/c8b9ca711b4d6b7637eae744d6b45ea47f6bded61bac0232bb42ed8c583e/hiredis-3.4.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d95b602ab022f3505288ce51feaa48c072a62e57da55d6a7a38ecb8c5ad67d81", size = 306364, upload-time = "2026-06-03T16:22:34.62Z" }, + { url = "https://files.pythonhosted.org/packages/c4/7e/e940eea3c2ee1aa5947f2e6224f03a1dfd38a5813307259a25f580411820/hiredis-3.4.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:de3e2297a182253dfa4400883a9a4fb46d44946aed3157ea2da873b93e2525c4", size = 339454, upload-time = "2026-06-03T16:22:35.87Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ea/b8147da5c270a2a5b85090c97d0ff7e2fae6e7c5f7749f8c3c2decadd3ac/hiredis-3.4.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:454236d2a5bd917daf38914ce363e71aeef41240e6800f4799e04ee82689bfd2", size = 351457, upload-time = "2026-06-03T16:22:36.95Z" }, + { url = "https://files.pythonhosted.org/packages/33/b5/ff8fe4f812348f09d2943b109cb64c5301af4f601e1cf026518e93a72fff/hiredis-3.4.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:35ab3653569b9867b8d8a3b4c0684a20dc769fe45d4666bedfe9a3391a61b30b", size = 312970, upload-time = "2026-06-03T16:22:38.004Z" }, + { url = "https://files.pythonhosted.org/packages/b3/2a/c90dff527cb2521ee1687e9e30bdf1156f2f4acfd47833b44dc52fec3ec6/hiredis-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:afff0876dafad6d3bb446c907da2836954876243f6bb9d5e44915d175e424aa4", size = 300850, upload-time = "2026-06-03T16:22:39.146Z" }, + { url = "https://files.pythonhosted.org/packages/90/0b/c48e93a1e524198b10ccc26d770368547c0c29d126a992fd4b4aa533f1ac/hiredis-3.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d5c33eb2da5c9ccd281c396e1c618cfe6a91eb841e957f17d2fa520383b3111d", size = 331430, upload-time = "2026-06-03T16:22:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/95/12/ed5bdc482d5c98930ffa264dd707dfb04b83118b2f7f760760c5dfbe6782/hiredis-3.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:04e54fc3bcecf8c7cb2846947b84baf7ce1507caba641bd23590c52fefade865", size = 333021, upload-time = "2026-06-03T16:22:41.363Z" }, + { url = "https://files.pythonhosted.org/packages/e6/42/d4a2e7be82f2b2db7b67ec622806ba099d8fe09d218568f71197922cbe79/hiredis-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5f1ddfe6429f9adc0a8d705afbcd40530fddeafa919873ffbb11f59eda44dbb9", size = 311747, upload-time = "2026-06-03T16:22:42.374Z" }, + { url = "https://files.pythonhosted.org/packages/d6/33/b5ac3420bd803ca9affd68a4a2a6111812bd26bfb9d6b41a721e009d79d9/hiredis-3.4.0-cp313-cp313-win32.whl", hash = "sha256:165e6405b48f9bd66ddb4ad52ce28b0c0041a0308654d7a0cb4357a1939134dc", size = 38921, upload-time = "2026-06-03T16:22:43.513Z" }, + { url = "https://files.pythonhosted.org/packages/03/e7/76e68122b1cf680b93b951a82953fff5b5883dc08ec93f63677eb3653591/hiredis-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:306aae11a52e495aaf0a14e3efcd7b51029e632c74b847bc03159e1e1f6db591", size = 40095, upload-time = "2026-06-03T16:22:44.296Z" }, + { url = "https://files.pythonhosted.org/packages/20/05/9313dc27ed159512dc22b4ecf8a62a84d0aa5fbd500ffdad955b361cb2a8/hiredis-3.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:975a8e75a10425442037dd9c7abbaae31941c34328d9f01b1ca42d9db44ac31d", size = 36884, upload-time = "2026-06-03T16:22:45.134Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ea/cbc922aeaa5af11f1c1235d8b2b04ff8cdf6e3e95c785a500521f32d8d70/hiredis-3.4.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:d3a12ae5685e9621a988af07b5af0ad685c7d19d6a7246ac852e35060178cff4", size = 138762, upload-time = "2026-06-03T16:22:45.927Z" }, + { url = "https://files.pythonhosted.org/packages/d4/e9/e004067ffad9f707174cde04d117c985d5f22dd4d9409f0983892738cb44/hiredis-3.4.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0a70df45cf167b5af99b9fe3e2044716919e30580a869dfa766f2a6467c0c320", size = 74696, upload-time = "2026-06-03T16:22:46.924Z" }, + { url = "https://files.pythonhosted.org/packages/5a/d1/5fe5b6d05e59116d78f9d228d9cc0022efbb84d234333c5fbe6a0c6e13fe/hiredis-3.4.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0a68b0e48509e6e66f4c212e53d98f29178addf83b0701a71bf0fce792954419", size = 70163, upload-time = "2026-06-03T16:22:47.798Z" }, + { url = "https://files.pythonhosted.org/packages/db/93/c86f0a7ae2cd10b72e30476f87aafd1af22992e080feb4b5d2ec1cbdf4e4/hiredis-3.4.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a45822bc8487da8151fe67c788de74b834582b1d510c67b888fcda64bf6ba4bb", size = 306631, upload-time = "2026-06-03T16:22:48.671Z" }, + { url = "https://files.pythonhosted.org/packages/e8/10/3746b028d9c43fab1fa4126fe69c6967df89ab9819140092930322b0550c/hiredis-3.4.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0b82cab9ad7a1574ab273a78942f780c1b1496101eb342b630c46c3e918ca21b", size = 339758, upload-time = "2026-06-03T16:22:49.662Z" }, + { url = "https://files.pythonhosted.org/packages/59/f3/c6fb383854237891039a4d94d3e66dc5eec8a2993fed6020c983d63c5393/hiredis-3.4.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:db13f8039ad8229f77f0e242be14e53bd67e8f3aadeb16f3af30944287cca092", size = 351360, upload-time = "2026-06-03T16:22:50.779Z" }, + { url = "https://files.pythonhosted.org/packages/70/b7/32110aa458690722a1069c7349b8ebe374a6ba0bdf9ef8925a9f37a74978/hiredis-3.4.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54b6267918c66d8ba4a3cf519db1235a4bd56d2a0969ca5b2ae3c6b6b7d9ed79", size = 313070, upload-time = "2026-06-03T16:22:51.966Z" }, + { url = "https://files.pythonhosted.org/packages/bb/23/bccfa0fb7b1b529cff35c8725cfd99a2d18fa4123f52f52bf03e84210855/hiredis-3.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:88396e6a24b80c86f4dc180964d9cc467ba3aa3c886af6532fe077c5a5dc0c3c", size = 300927, upload-time = "2026-06-03T16:22:53.085Z" }, + { url = "https://files.pythonhosted.org/packages/3e/0f/e1e2295ee863efc7ce8c88ec10bcc4b1504352373998cb493f10e900dbe5/hiredis-3.4.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:73dd607b47863633d8070f1eb3bab1b3b097ee747783fe69c0dd0f93ec673d8b", size = 331764, upload-time = "2026-06-03T16:22:54.194Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/11b1de2ac85dfd7a8713d72a6ed7ac0f1a6e28d906bd362e0df3a27f5c86/hiredis-3.4.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e6e8d5fa63ec2a0738d188488e828818cbe4cb4d37c0c706836cf3888d82c53d", size = 333144, upload-time = "2026-06-03T16:22:55.277Z" }, + { url = "https://files.pythonhosted.org/packages/6f/10/4b104565c936d51b4b02597352ec068937c9d6a73a3c4c9609c08ae3923e/hiredis-3.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d77901d058923a09ed25063ea6fb2842c153bbe75060a46e3949e73ad12ce352", size = 311593, upload-time = "2026-06-03T16:22:56.573Z" }, + { url = "https://files.pythonhosted.org/packages/70/ae/c9eda3c116bef50fcf0dc7e44379e3577f3627caca4ffd7af04675b02d98/hiredis-3.4.0-cp314-cp314-win32.whl", hash = "sha256:05384fcfe5851b5af868bf24265c14ab86f38562679f9c6f712895b67a98163c", size = 39662, upload-time = "2026-06-03T16:22:57.683Z" }, + { url = "https://files.pythonhosted.org/packages/f2/c8/cedb336a0386a97271761ace460a362cb2433c6cdf1d1ba760ad99225734/hiredis-3.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:53233656e4fecf9f8ec654f1f4c5d445bf1c2957d7f63ffdedbba2682c9d1584", size = 40682, upload-time = "2026-06-03T16:22:58.526Z" }, + { url = "https://files.pythonhosted.org/packages/4c/ea/3a05247ce4e2afe56f59d24b73ba38e37f2b324dba8290beba56fbd9fd1f/hiredis-3.4.0-cp314-cp314-win_arm64.whl", hash = "sha256:3348ba4e101f3a96c927447ff2edcb3e0026dc6df375ba117485a43edcbb6980", size = 37541, upload-time = "2026-06-03T16:22:59.307Z" }, + { url = "https://files.pythonhosted.org/packages/35/14/caeaa1be1205ebdc1cf6760c5f6882afbdb3b82a6bdf0559d01205b1c857/hiredis-3.4.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:3159c54fe560aa30bf1ab76e65c4c23dc45ad79d7cf4aecc25ec9942f5ea4cea", size = 139787, upload-time = "2026-06-03T16:23:00.139Z" }, + { url = "https://files.pythonhosted.org/packages/49/85/8f52b485b9d835e0f8da063a635290d916a6f5ab60c18db5411ecea344d1/hiredis-3.4.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:be4a41496a0a48c3abf57ef1bbeb11980060ce9c7a1dd8b92caa028a813a9c59", size = 75136, upload-time = "2026-06-03T16:23:01.705Z" }, + { url = "https://files.pythonhosted.org/packages/9f/09/ee568562f36f481395d5cea3ab75fd9350cd77d98d55ee5f9b395f3fc358/hiredis-3.4.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a2f9a9a591b3eaade523f3e778dfcd8684965ee6e954ae25cd2fd6d8c75e881d", size = 70772, upload-time = "2026-06-03T16:23:02.765Z" }, + { url = "https://files.pythonhosted.org/packages/7f/0d/3cb03fbbe72f86541f42ee49dba95ff428c87908815152970fbf24bdcf4c/hiredis-3.4.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c2852eaa26c0a73be4a30118cd5ad6a77c095d224ccb5ac38e40cb865747d22", size = 315571, upload-time = "2026-06-03T16:23:03.826Z" }, + { url = "https://files.pythonhosted.org/packages/52/fc/c8667282e41153bc20930aeba8ba0dff989cbaa9eb7594f8bcac02558dea/hiredis-3.4.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:18ff3d9b23ebe6c8248c3debca2402ad209d60c48495e7ed76407c2fe54cb9b4", size = 348131, upload-time = "2026-06-03T16:23:05.077Z" }, + { url = "https://files.pythonhosted.org/packages/99/13/5431ace8330904b2b9d9ce5425c13b7a8fa2b443ff272a92f248c07e6400/hiredis-3.4.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:94f83352295bf3d332678689ecd4ce190a4d233a20ad2f432724efd3ce03e49a", size = 359915, upload-time = "2026-06-03T16:23:06.293Z" }, + { url = "https://files.pythonhosted.org/packages/be/57/30dab05cf2a70905e5d2807edd4afa30a4747599070faf80f18e61375e11/hiredis-3.4.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:393d5e7c8c67cdddf7109a8e925d885e788f3f43e5b1043f84390df40c59944b", size = 321426, upload-time = "2026-06-03T16:23:07.447Z" }, + { url = "https://files.pythonhosted.org/packages/33/6f/0a6e030d96d927000735b39aa8b8fef03b43fafdf4a79c80755be351a0f5/hiredis-3.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7e7ab4c1c8c4d365b02d9e82cdf25b01a065edf2ededd7b5acb043201ff80203", size = 309862, upload-time = "2026-06-03T16:23:08.672Z" }, + { url = "https://files.pythonhosted.org/packages/11/48/26b2771d2b2403124c1f97c2a6d45df0ba3fa59f0c2d4d244e90543722fb/hiredis-3.4.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:cfe23f8dcf2c0f4e03d107ff68a9ee9707f9d76abeddbe59633e5de1564a650c", size = 339568, upload-time = "2026-06-03T16:23:09.755Z" }, + { url = "https://files.pythonhosted.org/packages/07/b1/01c18f676d5dea65e894c01ffae8da2f15df1fceed1c69b16877ba57be60/hiredis-3.4.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a7e76904148c229549db7240a4f9963deb8bb328c0c0844fc9f2320aca05b530", size = 341424, upload-time = "2026-06-03T16:23:10.964Z" }, + { url = "https://files.pythonhosted.org/packages/fb/58/ab3a5672e506f282e1dd6dfb1c0c3f7e17f02398280c2a2994f8d7b478ba/hiredis-3.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:92b570225f6097430615a82543c3eb7974ca354738a6cef38053138f7d983151", size = 320386, upload-time = "2026-06-03T16:23:12.174Z" }, + { url = "https://files.pythonhosted.org/packages/15/af/3f26324cca720f56ace408883c1c7311ce71b571e82e6434515f7ba4eb59/hiredis-3.4.0-cp314-cp314t-win32.whl", hash = "sha256:decc176d86127c620b5d280b3fe5f97a788be58ca945971f3852c3bf54f4d5ad", size = 40516, upload-time = "2026-06-03T16:23:13.179Z" }, + { url = "https://files.pythonhosted.org/packages/8b/18/e011a424a9608ff152ebeb7bbae2be3163e5716e92cf75baddcb5a8fc312/hiredis-3.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:05c852c58fec65d4c9fb861372dd7391d8b2ce96c960ba8714145f8cd85cd0ec", size = 41453, upload-time = "2026-06-03T16:23:14.091Z" }, + { url = "https://files.pythonhosted.org/packages/43/5f/829287555ce7286be8d6c87c69f93aa1f38fe67c46740806416142231cf3/hiredis-3.4.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7ff29c9f5d3c91fda948c2fde58f457b3244550781d3bc0891b1b9d93c10f47f", size = 37968, upload-time = "2026-06-03T16:23:14.948Z" }, +] + [[package]] name = "hpack" version = "4.1.0" @@ -4178,6 +4270,7 @@ proxy = [ { name = "fastapi-sso" }, { name = "granian" }, { name = "gunicorn" }, + { name = "hiredis" }, { name = "inquirerpy" }, { name = "litellm-enterprise" }, { name = "litellm-proxy-extras" }, @@ -4359,6 +4452,7 @@ requires-dist = [ { name = "grpcio", marker = "extra == 'grpc'", specifier = "==1.78.0" }, { name = "grpcio", marker = "extra == 'proxy-runtime'", specifier = "==1.78.0" }, { name = "gunicorn", marker = "extra == 'proxy'", specifier = ">=23.0.0,<24.0" }, + { name = "hiredis", marker = "extra == 'proxy'", specifier = ">=3.0.0,<4.0" }, { name = "httpx", specifier = ">=0.28.0,<1.0" }, { name = "importlib-metadata", specifier = ">=8.0.0,<9.0" }, { name = "inquirerpy", marker = "extra == 'cli'", specifier = ">=0.3.4,<1.0" }, From 20399812103a622482f6a149daf82ee3f3f04d5c Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 3 Aug 2026 20:50:11 -0700 Subject: [PATCH 069/124] feat(ui): show auto-router savings on the cost-optimization dashboard (#35522) Adds the auto-router as a third optimization driver beside compression and prompt caching: a summary card, a donut segment, and a series in the savings graph across both the cumulative and per-day views. The number is signed, because a switch that thrashes the prompt cache can cost more than the cheaper rates save and an operator needs to see that. The donut plots only drivers that saved, since a negative slice has no meaning, while the card and the range total keep the sign. `usd()` sizes and signs off the magnitude so a small loss renders as -$0.01 rather than "$-0.00". The card's popover states the counterfactual and its two consequences: that a switch pays to re-warm the cache, and that a first turn the router could not identify is charged that write and therefore under-reported. --- .../_components/UsageTab.test.tsx | 112 +++++++++++++++++- .../_components/UsageTab.tsx | 74 +++++++----- .../_components/costOptimizationUtils.test.ts | 59 ++++++++- .../_components/costOptimizationUtils.ts | 30 ++++- .../src/components/UsagePage/types.ts | 1 + 5 files changed, 238 insertions(+), 38 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx index 5c26ac30477..fe3e792eeee 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx @@ -29,12 +29,14 @@ vi.mock("@/components/shared/charts", () => ({ colors, showLegend, maxBarSize, + stack, }: { data: unknown; categories: string[]; colors?: readonly string[]; showLegend?: boolean; maxBarSize?: number; + stack?: boolean; }) => (
({ data-colors={(colors ?? []).join(",")} data-show-legend={String(showLegend ?? true)} data-max-bar-size={maxBarSize === undefined ? "" : String(maxBarSize)} + data-stack={String(stack ?? false)} data-series={JSON.stringify(data)} /> ), @@ -216,8 +219,8 @@ describe("UsageTab", () => { const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); expect(slices).toEqual([ - { driver: "Compression", usd: expect.closeTo(0.14, 5) }, - { driver: "Prompt caching", usd: expect.closeTo(0.016, 5) }, + { driver: "Compression", color: "emerald", usd: expect.closeTo(0.14, 5) }, + { driver: "Prompt caching", color: "blue", usd: expect.closeTo(0.016, 5) }, ]); }); @@ -225,7 +228,110 @@ describe("UsageTab", () => { const { getByTestId } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]); const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); - expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]); + expect(slices).toEqual([{ driver: "Compression", color: "emerald", usd: expect.closeTo(0.04, 5) }]); + }); + + it("does not stack the per-day drivers, because one of them can be negative", async () => { + // Stacking sums the series into one bar. Auto-router savings go negative when a + // model switch pays for a cold cache, and that segment would be drawn below the + // axis while the rest of the bar still read as the day's total. + const { getByRole, getByTestId } = renderWith([ + day("2026-07-12", { + compression_savings_spend: 0.1, + prompt_caching_savings_spend: 0.02, + autorouter_savings_spend: -0.05, + }), + ]); + + await userEvent.click(getByRole("tab", { name: "Per day" })); + const bars = getByTestId("bar-chart"); + expect(bars.getAttribute("data-stack")).toBe("false"); + expect(readSeries(bars)[0]).toMatchObject({ "Auto-router": -0.05 }); + }); + + it("lays the savings header out with the card's own slots so nothing shifts between tabs", async () => { + // The subtitle differs in length between the tabs ("Running total saved" vs "Saved + // per day"). Hand-rolled rows made it compete with the legend and the toggle for + // width, so the header grew a line on one tab and the chart moved with it. CardHeader + // sizes the action column to its content and gives the rest to the title column. + const { getByRole, getByTestId, container } = renderWith(twoDays()); + + const header = () => { + const legend = getByTestId("chart-legend"); + const action = legend.closest('[data-slot="card-action"]') as HTMLElement; + const cardHeader = action.parentElement as HTMLElement; + const description = cardHeader.querySelector('[data-slot="card-description"]') as HTMLElement; + return { action, cardHeader, description }; + }; + + const before = header(); + expect(before.action).toBeTruthy(); + expect(before.description).toBeTruthy(); + // the toggle rides in the same action slot as the legend, so neither moves alone + expect(before.action.contains(getByRole("tablist"))).toBe(true); + // the subtitle lives outside that slot, so its length cannot reposition the controls + expect(before.action.contains(before.description)).toBe(false); + expect(before.description.textContent).toContain("Running total saved"); + + await userEvent.click(getByRole("tab", { name: "Per day" })); + + const after = header(); + expect(after.action).toBe(before.action); + expect(after.cardHeader).toBe(before.cardHeader); + expect(after.action.contains(after.description)).toBe(false); + expect(after.description.textContent).toContain("Saved per day"); + expect(container.textContent).toContain("Savings"); + }); + + it("subtracts a losing auto-router route from the total and keeps it out of the donut", () => { + // Switching models leaves the new one with a cold cache, so a route can cost more + // than the baseline would have. A negative slice is meaningless in a donut, but the + // total has to keep the loss or the page can only ever report good news. + const { getByText, getByTestId } = renderWith([ + day("2026-07-12", { + compression_savings_spend: 0.1, + prompt_caching_savings_spend: 0.02, + autorouter_savings_spend: -0.05, + }), + ]); + + expect(getByText("$0.0700")).toBeInTheDocument(); + expect(getByText("-$0.0500")).toBeInTheDocument(); + + const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); + expect(slices.map((d: { driver: string }) => d.driver)).toEqual(["Compression", "Prompt caching"]); + expect(getByTestId("donut-chart").getAttribute("data-label")).toBe("$0.1200"); + }); + + it("carries auto-router savings into the summary card, donut slice, and cumulative series", () => { + const { getByText, getByTestId } = renderWith([ + day("2026-07-12", { + compression_savings_spend: 0.04, + prompt_caching_savings_spend: 0.006, + autorouter_savings_spend: 0.02, + }), + day("2026-07-13", { + compression_savings_spend: 0.1, + prompt_caching_savings_spend: 0.01, + autorouter_savings_spend: 0.05, + }), + ]); + + // Total saved now sums three drivers, and the auto-router card carries its own total. + expect(getByText("$0.2260")).toBeInTheDocument(); + expect(getByText("$0.0700")).toBeInTheDocument(); + + // The driver donut gains a third slice priced from the range totals. + const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); + expect(slices).toEqual([ + { driver: "Compression", color: "emerald", usd: expect.closeTo(0.14, 5) }, + { driver: "Prompt caching", color: "blue", usd: expect.closeTo(0.016, 5) }, + { driver: "Auto-router", color: "amber", usd: expect.closeTo(0.07, 5) }, + ]); + + // And the cumulative line accumulates the auto-router series alongside the others. + const series = readSeries(getByTestId("area-chart")); + expect(series[2]["Auto-router"]).toBeCloseTo(0.07, 5); }); it("renders spend-by-tool bars from the tool spend endpoint", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx index ec37418e0b5..b6287602210 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx @@ -5,7 +5,7 @@ import { Info } from "lucide-react"; import { AreaChart, BarChart, CustomLegend, DonutChart, SEQUENTIAL_COLOR_RAMP } from "@/components/shared/charts"; import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Card, CardAction, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { getToolSpend, ToolSpendResponse } from "@/components/networking"; @@ -16,6 +16,8 @@ import { formatRangeLabel, localIsoDay, MAX_POINTS_WITH_DOTS, + SAVINGS_COLORS, + SAVINGS_DRIVERS, SAVINGS_SERIES, SavingsAccumulation, SavingsPoint, @@ -38,8 +40,6 @@ const EMPTY_TOOL_SPEND: ToolSpendResponse = { end_date: null, }; -const SAVINGS_COLORS = ["emerald", "blue"] as const; - const shortDate = (iso: string): string => new Date(`${iso}T00:00:00`).toLocaleDateString("en-US", { month: "short", day: "numeric" }); @@ -47,6 +47,7 @@ const isoDay = (d: Date): string => d.toISOString().slice(0, 10); const compressionOf = (m: SpendMetrics): number => m.compression_savings_spend ?? 0; const cachingOf = (m: SpendMetrics): number => m.prompt_caching_savings_spend ?? 0; +const autorouterOf = (m: SpendMetrics): number => m.autorouter_savings_spend ?? 0; const savedTokensOf = (m: SpendMetrics): number => m.compression_saved_tokens ?? 0; const SummaryCard = ({ label, value, hint, info }: { label: string; value: string; hint?: string; info?: string }) => ( @@ -105,8 +106,9 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { const compressionTotal = useMemo(() => results.reduce((sum, d) => sum + compressionOf(d.metrics), 0), [results]); const cachingTotal = useMemo(() => results.reduce((sum, d) => sum + cachingOf(d.metrics), 0), [results]); + const autorouterTotal = useMemo(() => results.reduce((sum, d) => sum + autorouterOf(d.metrics), 0), [results]); const savedTokensTotal = useMemo(() => results.reduce((sum, d) => sum + savedTokensOf(d.metrics), 0), [results]); - const totalSaved = compressionTotal + cachingTotal; + const totalSaved = compressionTotal + cachingTotal + autorouterTotal; const [accumulation, setAccumulation] = useState("cumulative"); @@ -122,6 +124,7 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { date: shortDate(d.date), Compression: compressionOf(d.metrics), "Prompt caching": cachingOf(d.metrics), + "Auto-router": autorouterOf(d.metrics), })), [results], ); @@ -143,14 +146,19 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { .filter(Boolean) .join(" \u00b7 "); + // A driver can come out negative (auto-router pays a cold-cache write on every + // model switch), and a negative slice has no meaning in a donut, so only drivers + // that actually saved are plotted; the range total keeps the signed truth. const byDriver = useMemo( () => - [ - { driver: "Compression", usd: compressionTotal }, - { driver: "Prompt caching", usd: cachingTotal }, - ].filter((d) => d.usd > 0), - [compressionTotal, cachingTotal], + SAVINGS_DRIVERS.map(({ name, color }) => ({ + driver: name, + color, + usd: { Compression: compressionTotal, "Prompt caching": cachingTotal, "Auto-router": autorouterTotal }[name], + })).filter((d) => d.usd > 0), + [compressionTotal, cachingTotal, autorouterTotal], ); + const plottedDriverTotal = useMemo(() => byDriver.reduce((sum, d) => sum + d.usd, 0), [byDriver]); const topTools = useMemo(() => topToolsBySpend(toolSpend?.by_tool ?? []), [toolSpend]); const topToolNames = useMemo(() => topTools.map((t) => t.tool_name), [topTools]); @@ -174,11 +182,11 @@ const UsageTab: React.FC = ({ accessToken, activity }) => {
-
+
= ({ accessToken, activity }) => { hint="Cache read discount" info="Tokens the provider served from cache, priced at the discount between the input and cache-read rates." /> +
+ {/* CardHeader's own slots rather than hand-rolled rows: the action column is + sized to its content and the title column takes the rest, so the subtitle + never competes with the controls for width and neither moves when it grows. + The controls wrap within their column instead of pushing past the card */} -
-
- Savings -

{savingsSubtitle}

-
-
- - setAccumulation(value as SavingsAccumulation)}> - - Cumulative - {intervalLabel} - - -
-
+ Savings + {savingsSubtitle} + + + setAccumulation(value as SavingsAccumulation)}> + + Cumulative + {intervalLabel} + + + {accumulation === "cumulative" ? ( @@ -225,12 +239,14 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { showDots={overTime.length <= MAX_POINTS_WITH_DOTS} /> ) : ( + // Not stacked: a driver can be negative once a model switch is charged + // for its cold cache, and stacking would draw that segment below the axis + // while the remaining bar still read as the day's total @@ -247,10 +263,10 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { data={byDriver} index="driver" category="usd" - colors={["emerald", "blue"]} + colors={byDriver.map((d) => d.color)} valueFormatter={usd} showLabel - label={usd(totalSaved)} + label={usd(plottedDriverTotal)} /> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts index dc08799a3c4..14fb26c53ef 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts @@ -3,6 +3,9 @@ import { describe, expect, it } from "vitest"; import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; import type { ToolSpendDailyEntry, ToolSpendEntry } from "@/components/networking"; import { + SAVINGS_COLORS, + SAVINGS_DRIVERS, + SAVINGS_SERIES, buildDailyToolSeries, computeCacheLeakage, formatRangeLabel, @@ -10,6 +13,7 @@ import { localIsoDay, toCumulative, topToolsBySpend, + usd, withStartAnchor, } from "./costOptimizationUtils"; @@ -239,22 +243,25 @@ describe("localIsoDay", () => { }); describe("toCumulative", () => { - const point = (date: string, compression: number, caching: number) => ({ + const point = (date: string, compression: number, caching: number, autorouter: number = 0) => ({ date, Compression: compression, "Prompt caching": caching, + "Auto-router": autorouter, }); it("turns each reading into everything saved up to that point", () => { const running = toCumulative([point("Jul 1", 1, 10), point("Jul 2", 2, 20), point("Jul 3", 3, 30)]); expect(running.map((p) => p.Compression)).toEqual([1, 3, 6]); expect(running.map((p) => p["Prompt caching"])).toEqual([10, 30, 60]); + expect(running.map((p) => p["Auto-router"])).toEqual([0, 0, 0]); }); it("accumulates each driver on its own, so one flat series cannot lift the other", () => { const running = toCumulative([point("Jul 1", 0, 5), point("Jul 2", 0, 5)]); expect(running.map((p) => p.Compression)).toEqual([0, 0]); expect(running.map((p) => p["Prompt caching"])).toEqual([5, 10]); + expect(running.map((p) => p["Auto-router"])).toEqual([0, 0]); }); it("never falls, even across a quiet interval", () => { @@ -267,13 +274,19 @@ describe("toCumulative", () => { expect(running.map((p) => p.date)).toEqual(["9am", "10am"]); expect(toCumulative([])).toEqual([]); }); + + it("accumulates auto-router savings like other drivers", () => { + const running = toCumulative([point("Jul 1", 1, 1, 5), point("Jul 2", 1, 1, 10)]); + expect(running.map((p) => p["Auto-router"])).toEqual([5, 15]); + }); }); describe("withStartAnchor", () => { - const point = (date: string, compression: number, caching: number) => ({ + const point = (date: string, compression: number, caching: number, autorouter: number = 0) => ({ date, Compression: compression, "Prompt caching": caching, + "Auto-router": autorouter, }); it("lifts a single-day cumulative off a floating dot by prepending a $0 origin", () => { @@ -285,6 +298,7 @@ describe("withStartAnchor", () => { const anchored = withStartAnchor([point("Jul 16", 5, 1), point("Jul 17", 9, 4)], "Jul 16"); expect(anchored.map((p) => p.Compression)).toEqual([0, 5, 9]); expect(anchored.map((p) => p["Prompt caching"])).toEqual([0, 1, 4]); + expect(anchored.map((p) => p["Auto-router"])).toEqual([0, 0, 0]); }); it("leaves an empty series alone so the chart's own no-data state can show", () => { @@ -306,3 +320,44 @@ describe("formatRangeLabel", () => { expect(formatRangeLabel(new Date(2026, 6, 23), undefined)).toBe(""); }); }); + +describe("usd", () => { + it("keeps four decimals for sub-dollar amounts so small savings stay visible", () => { + expect(usd(0.05)).toBe("$0.0500"); + expect(usd(1.5)).toBe("$1.50"); + expect(usd(0)).toBe("$0.00"); + }); + + it("signs a loss ahead of the symbol and keeps its precision", () => { + // A driver can be negative once a model switch is charged for its cold cache. + // Sizing decimals off the raw value would render this as "$-0.00". + expect(usd(-0.05)).toBe("-$0.0500"); + expect(usd(-0.0004)).toBe("-$0.0004"); + expect(usd(-12.4)).toBe("-$12.40"); + }); +}); + +describe("savings driver colours", () => { + it("keeps a driver's colour when a driver above it is filtered out", () => { + // Charts colour by position in the data they are given, and the donut is given + // only drivers that saved something. Compression is zero on any deployment not + // running the compression guardrail, so the survivors must not slide onto the + // colours of the drivers dropped above them. + const totals = { Compression: 0, "Prompt caching": 4, "Auto-router": 7 } as const; + const plotted = SAVINGS_DRIVERS.map(({ name, color }) => ({ name, color, usd: totals[name] })).filter( + (d) => d.usd > 0, + ); + + expect(plotted.map((d) => [d.name, d.color])).toEqual([ + ["Prompt caching", "blue"], + ["Auto-router", "amber"], + ]); + }); + + it("agrees with the legend, which is built from the unfiltered list", () => { + const legend = new Map(SAVINGS_SERIES.map((name, i) => [SAVINGS_COLORS[i], name])); + for (const { name, color } of SAVINGS_DRIVERS) { + expect(legend.get(color)).toBe(name); + } + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts index 32eb6ae198d..d63266c5ee7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts @@ -3,8 +3,11 @@ import { ToolSpendDailyEntry, ToolSpendEntry } from "@/components/networking"; import { formatNumberWithCommas } from "@/utils/dataUtils"; export const usd = (value: number): string => { - const decimals = value > 0 && value < 1 ? 4 : 2; - return `$${formatNumberWithCommas(value, decimals)}`; + // Sized and signed off the magnitude: a driver can come out negative, and a small + // loss rendered at two decimals would read as "$-0.00" + const magnitude = Math.abs(value); + const decimals = magnitude > 0 && magnitude < 1 ? 4 : 2; + return `${value < 0 ? "-" : ""}$${formatNumberWithCommas(magnitude, decimals)}`; }; export const pct = (ratio: number): string => `${formatNumberWithCommas(ratio * 100, 1)}%`; @@ -161,9 +164,27 @@ export type SavingsPoint = { date: string; Compression: number; "Prompt caching": number; + "Auto-router": number; }; -export const SAVINGS_SERIES = ["Compression", "Prompt caching"] as const; +/** + * The savings drivers, each owning its own colour. + * + * One list rather than a names list beside a colours list, because the donut is + * given only the drivers that saved anything and charts assign colours by position + * in the data they receive. Two lists that line up by index therefore stop lining + * up the moment a driver is filtered out: the survivors slide down and inherit the + * colours of the drivers above them, while the legend still reports the original + * mapping. Colour travels with the driver so filtering cannot separate them. + */ +export const SAVINGS_DRIVERS = [ + { name: "Compression", color: "emerald" }, + { name: "Prompt caching", color: "blue" }, + { name: "Auto-router", color: "amber" }, +] as const; + +export const SAVINGS_SERIES = SAVINGS_DRIVERS.map((d) => d.name); +export const SAVINGS_COLORS = SAVINGS_DRIVERS.map((d) => d.color); /** * Running total of each series across the selected window. The total restarts @@ -179,6 +200,7 @@ export const toCumulative = (points: readonly SavingsPoint[]): SavingsPoint[] => date: point.date, Compression: (previous?.Compression ?? 0) + point.Compression, "Prompt caching": (previous?.["Prompt caching"] ?? 0) + point["Prompt caching"], + "Auto-router": (previous?.["Auto-router"] ?? 0) + point["Auto-router"], }, ]; }, []); @@ -193,7 +215,7 @@ export const toCumulative = (points: readonly SavingsPoint[]): SavingsPoint[] => export const withStartAnchor = (cumulative: readonly SavingsPoint[], startLabel: string): SavingsPoint[] => cumulative.length === 0 ? [...cumulative] - : [{ date: startLabel, Compression: 0, "Prompt caching": 0 }, ...cumulative]; + : [{ date: startLabel, Compression: 0, "Prompt caching": 0, "Auto-router": 0 }, ...cumulative]; /** "Jul 16 – Jul 23", collapsing to a single date when the range is one day. */ export const formatRangeLabel = (from: Date | undefined, to: Date | undefined): string => { diff --git a/ui/litellm-dashboard/src/components/UsagePage/types.ts b/ui/litellm-dashboard/src/components/UsagePage/types.ts index bf33fa37111..b10fc79be15 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/types.ts +++ b/ui/litellm-dashboard/src/components/UsagePage/types.ts @@ -11,6 +11,7 @@ export interface SpendMetrics { compression_saved_tokens?: number; compression_savings_spend?: number; prompt_caching_savings_spend?: number; + autorouter_savings_spend?: number; } export type DailyData = { From d39c5577438f117a6265eb46516a79a0be0ea11f Mon Sep 17 00:00:00 2001 From: tin Date: Tue, 4 Aug 2026 03:50:33 +0000 Subject: [PATCH 070/124] fix(bedrock): drop conflicting tool_choice.type when toolConfig.toolChoice is set Converse rejects a request that carries both toolConfig.toolChoice and an additionalModelRequestFields.tool_choice.type, so any request that pairs parallel_tool_calls with an explicit tool_choice 400s with "The additional field tool_choice/type conflicts with the existing field toolConfig.toolChoice.auto". That pairing is what agentic clients send by default; Codex CLI sends tool_choice "auto" and parallel_tool_calls false on every turn, so tool calling was broken outright on Bedrock models that advertise supports_parallel_tool_use_config. Drop the type from the Anthropic passthrough once toolChoice carries it, and keep disable_parallel_tool_use, which has no toolConfig equivalent and is accepted alongside toolChoice. Measured against Bedrock directly: toolChoice plus {disable_parallel_tool_use} succeeds for auto, any and tool, while an empty tool_choice with no toolChoice is rejected for a missing type, so the type still has to be emitted when the caller sends no tool_choice. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../bedrock/chat/converse_transformation.py | 15 ++++ .../chat/test_converse_transformation.py | 72 +++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 2b34c9f2654..4d5e6fdfe5f 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1213,6 +1213,20 @@ class AmazonConverseConfig(BaseConfig): } return {**additional_request_params, **merged_entries} + @staticmethod + def _drop_tool_choice_type_conflicting_with_tool_config(additional_request_params: dict) -> None: + """Drop ``tool_choice.type`` from the Anthropic passthrough fields. + + Converse rejects a request carrying both ``toolConfig.toolChoice`` and an + ``additionalModelRequestFields.tool_choice.type``, so once the caller asked for a + tool choice the type has to come from ``toolChoice`` alone. Sibling keys such as + ``disable_parallel_tool_use`` have no ``toolConfig`` equivalent and are accepted + alongside ``toolChoice``, so they stay. + """ + tool_choice = additional_request_params.get("tool_choice") + if isinstance(tool_choice, dict): + tool_choice.pop("type", None) + def _prepare_request_params( self, optional_params: dict, model: str, drop_params: bool = False ) -> tuple[dict, dict, dict, OutputConfigBlock | None]: @@ -1569,6 +1583,7 @@ class AmazonConverseConfig(BaseConfig): ) if tool_choice_values is not None: bedrock_tool_config["toolChoice"] = tool_choice_values + self._drop_tool_choice_type_conflicting_with_tool_config(additional_request_params) data: CommonRequestObject = { "inferenceConfig": self._transform_inference_params(inference_params=inference_params), diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index cca4f4232f2..6d318bb8729 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -4263,6 +4263,78 @@ def test_parallel_tool_calls_emits_typed_auto_tool_choice(parallel_tool_calls, e } +@pytest.mark.parametrize( + "tool_choice, expected_tool_config_choice", + [ + ("auto", {"auto": {}}), + ("required", {"any": {}}), + ({"type": "function", "function": {"name": "get_current_weather"}}, {"tool": {"name": "get_current_weather"}}), + ], +) +def test_parallel_tool_calls_with_explicit_tool_choice_omits_conflicting_type(tool_choice, expected_tool_config_choice): + config = AmazonConverseConfig() + model = "us.anthropic.claude-opus-4-8" + messages = [{"role": "user", "content": "What's the weather in SF and NYC?"}] + + optional_params = config.map_openai_params( + non_default_params={"parallel_tool_calls": False, "tool_choice": tool_choice, "tools": _TOOL_PARAM}, + optional_params={}, + model=model, + drop_params=False, + ) + + request_data = config.transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert request_data["toolConfig"]["toolChoice"] == expected_tool_config_choice + assert request_data["additionalModelRequestFields"]["tool_choice"] == {"disable_parallel_tool_use": True} + + +def test_tool_choice_type_kept_when_no_tool_config_choice_conflicts(): + config = AmazonConverseConfig() + model = "us.anthropic.claude-opus-4-8" + + optional_params = config.map_openai_params( + non_default_params={"parallel_tool_calls": False, "tools": _TOOL_PARAM}, + optional_params={}, + model=model, + drop_params=False, + ) + + request_data = config.transform_request( + model=model, + messages=[{"role": "user", "content": "What's the weather in SF and NYC?"}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert "toolChoice" not in request_data["toolConfig"] + assert request_data["additionalModelRequestFields"]["tool_choice"] == { + "type": "auto", + "disable_parallel_tool_use": True, + } + + +def test_drop_tool_choice_type_leaves_other_passthrough_fields_untouched(): + additional_request_params = { + "tool_choice": {"type": "tool", "name": "get_weather", "disable_parallel_tool_use": True}, + "anthropic_beta": ["some-beta"], + } + + AmazonConverseConfig._drop_tool_choice_type_conflicting_with_tool_config(additional_request_params) + + assert additional_request_params == { + "tool_choice": {"name": "get_weather", "disable_parallel_tool_use": True}, + "anthropic_beta": ["some-beta"], + } + + def test_parallel_tool_use_merge_preserves_user_tool_choice_type(): merged = AmazonConverseConfig._merge_parallel_tool_use_config( {"tool_choice": {"type": "tool", "name": "get_weather", "disable_parallel_tool_use": False}}, From c9887a1f94bc1e7e4bdfe64d640f0509a0bc19dd Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Tue, 4 Aug 2026 06:34:52 +0200 Subject: [PATCH 071/124] perf: build log messages lazily so filtered-out log records cost nothing (#35703) --- litellm/_redis.py | 8 +- litellm/a2a_protocol/card_resolver.py | 4 +- .../a2a_protocol/exception_mapping_utils.py | 8 +- .../litellm_completion_bridge/handler.py | 14 +- .../transformation.py | 8 +- litellm/a2a_protocol/main.py | 20 +- .../providers/bedrock_agentcore/handler.py | 6 +- .../bedrock_agentcore/transformation.py | 2 +- .../providers/pydantic_ai_agents/handler.py | 4 +- .../pydantic_ai_agents/transformation.py | 10 +- .../providers/watsonx_orchestrate/handler.py | 6 +- .../watsonx_orchestrate/transformation.py | 2 +- litellm/a2a_protocol/streaming_iterator.py | 10 +- litellm/anthropic_beta_headers_manager.py | 8 +- litellm/batches/batch_utils.py | 4 +- litellm/batches/main.py | 4 +- litellm/caching/azure_blob_cache.py | 10 +- litellm/caching/caching.py | 6 +- litellm/caching/caching_handler.py | 2 +- litellm/caching/dual_cache.py | 6 +- litellm/caching/gcs_cache.py | 9 +- litellm/caching/redis_cache.py | 21 +- litellm/caching/redis_cluster_cache.py | 2 +- litellm/caching/redis_semantic_cache.py | 2 +- litellm/caching/s3_cache.py | 17 +- .../transformation.py | 42 +- litellm/cost_calculator.py | 33 +- litellm/experimental_mcp_client/client.py | 105 ++--- .../google_genai/adapters/transformation.py | 9 +- .../SlackAlerting/batching_handler.py | 4 +- .../SlackAlerting/slack_alerting.py | 10 +- .../anthropic_cache_control_hook.py | 10 +- litellm/integrations/argilla.py | 14 +- litellm/integrations/arize/_utils.py | 2 +- litellm/integrations/arize/arize_phoenix.py | 2 +- .../arize/arize_phoenix_prompt_manager.py | 2 +- .../azure_sentinel/azure_sentinel.py | 8 +- .../azure_storage/azure_storage.py | 34 +- .../bitbucket/bitbucket_prompt_manager.py | 2 +- .../integrations/braintrust_mock_client.py | 2 +- litellm/integrations/cloudzero/cloudzero.py | 18 +- litellm/integrations/custom_batch_logger.py | 2 +- litellm/integrations/custom_guardrail.py | 4 +- litellm/integrations/custom_logger.py | 20 +- litellm/integrations/custom_secret_manager.py | 2 +- litellm/integrations/datadog/datadog.py | 24 +- .../datadog/datadog_cost_management.py | 8 +- .../integrations/datadog/datadog_llm_obs.py | 38 +- .../integrations/datadog/datadog_metrics.py | 8 +- litellm/integrations/deepeval/api.py | 4 +- litellm/integrations/gcs_bucket/gcs_bucket.py | 12 +- .../gcs_bucket/gcs_bucket_mock_client.py | 4 +- litellm/integrations/gcs_pubsub/pub_sub.py | 6 +- .../generic_api/generic_api_callback.py | 37 +- .../gitlab/gitlab_prompt_manager.py | 2 +- litellm/integrations/lago.py | 8 +- litellm/integrations/langfuse/langfuse.py | 29 +- .../integrations/langfuse/langfuse_otel.py | 4 +- .../langfuse/langfuse_prompt_management.py | 4 +- litellm/integrations/langsmith.py | 20 +- litellm/integrations/literal_ai.py | 10 +- litellm/integrations/logfire_logger.py | 4 +- litellm/integrations/mlflow.py | 2 +- litellm/integrations/mock_client_factory.py | 18 +- litellm/integrations/newrelic/newrelic.py | 41 +- litellm/integrations/opentelemetry.py | 3 +- litellm/integrations/opik/opik.py | 20 +- .../opik/opik_payload_builder/extractors.py | 6 +- .../opik_payload_builder/payload_builders.py | 2 +- litellm/integrations/posthog.py | 30 +- litellm/integrations/prometheus.py | 69 ++-- litellm/integrations/prometheus_services.py | 2 +- litellm/integrations/rubrik.py | 60 +-- litellm/integrations/s3.py | 14 +- litellm/integrations/s3_v2.py | 59 +-- litellm/integrations/sqs.py | 12 +- litellm/integrations/traceloop.py | 4 +- .../vector_store_pre_call_hook.py | 20 +- litellm/integrations/weave/weave_otel.py | 4 +- .../websearch_interception/handler.py | 131 +++--- .../websearch_interception/transformation.py | 10 +- litellm/integrations/weights_biases.py | 4 +- litellm/interactions/streaming_iterator.py | 2 +- .../exception_mapping_utils.py | 12 +- litellm/litellm_core_utils/fallback_utils.py | 2 +- litellm/litellm_core_utils/litellm_logging.py | 83 ++-- .../litellm_core_utils/llm_cost_calc/utils.py | 12 +- .../llm_response_utils/get_api_base.py | 2 +- .../logging_callback_manager.py | 13 +- litellm/litellm_core_utils/logging_utils.py | 8 +- litellm/litellm_core_utils/logging_worker.py | 6 +- .../prompt_templates/factory.py | 15 +- .../litellm_core_utils/realtime_streaming.py | 12 +- .../litellm_core_utils/streaming_handler.py | 10 +- litellm/litellm_core_utils/token_counter.py | 21 +- litellm/llms/__init__.py | 14 +- .../chat/guardrail_translation/handler.py | 8 +- .../llms/anthropic/count_tokens/handler.py | 16 +- .../anthropic/count_tokens/token_counter.py | 4 +- .../adapters/streaming_iterator.py | 2 +- .../editors/clear_tool_uses.py | 4 +- .../messages/mcp_handler.py | 5 +- .../responses_adapters/streaming_iterator.py | 2 +- litellm/llms/anthropic/files/handler.py | 4 +- .../azure/chat/o_series_transformation.py | 2 +- litellm/llms/azure/common_utils.py | 13 +- litellm/llms/azure/cost_calculation.py | 5 +- .../llms/azure/image_generation/__init__.py | 2 +- .../responses/o_series_transformation.py | 2 +- .../llms/azure/responses/transformation.py | 10 +- litellm/llms/azure_ai/agents/handler.py | 24 +- .../llms/azure_ai/agents/transformation.py | 2 +- .../anthropic/count_tokens/handler.py | 16 +- .../anthropic/count_tokens/token_counter.py | 4 +- litellm/llms/azure_ai/chat/transformation.py | 2 +- litellm/llms/azure_ai/cost_calculator.py | 2 +- .../azure_ai/image_generation/__init__.py | 2 +- litellm/llms/azure_ai/ocr/common_utils.py | 4 +- .../document_intelligence/transformation.py | 12 +- litellm/llms/azure_ai/ocr/transformation.py | 12 +- .../files/azure_blob_storage_backend.py | 10 +- .../base_llm/files/storage_backend_factory.py | 2 +- .../base_managed_resource.py | 6 +- litellm/llms/bedrock/base_aws_llm.py | 32 +- .../bedrock/chat/agentcore/transformation.py | 53 +-- .../bedrock/chat/converse_transformation.py | 8 +- .../chat/invoke_agent/transformation.py | 24 +- litellm/llms/bedrock/chat/invoke_handler.py | 2 +- .../count_tokens/bedrock_token_counter.py | 4 +- litellm/llms/bedrock/count_tokens/handler.py | 20 +- litellm/llms/bedrock/files/transformation.py | 3 +- litellm/llms/bedrock/realtime/handler.py | 16 +- .../llms/bedrock/realtime/transformation.py | 10 +- .../bedrock_mantle/chat/transformation.py | 2 +- .../black_forest_labs/image_edit/handler.py | 8 +- .../image_generation/handler.py | 8 +- .../llms/custom_httpx/aiohttp_transport.py | 10 +- litellm/llms/custom_httpx/http_handler.py | 17 +- litellm/llms/custom_httpx/llm_http_handler.py | 46 ++- litellm/llms/databricks/common_utils.py | 2 +- litellm/llms/databricks/streaming_utils.py | 8 +- litellm/llms/gemini/files/transformation.py | 10 +- .../llms/gemini/realtime/transformation.py | 10 +- litellm/llms/gigachat/authenticator.py | 4 +- litellm/llms/gigachat/chat/transformation.py | 2 +- litellm/llms/gigachat/file_handler.py | 16 +- litellm/llms/github_copilot/authenticator.py | 34 +- .../embedding/transformation.py | 2 +- .../responses/transformation.py | 9 +- litellm/llms/groq/chat/transformation.py | 2 +- .../llms/huggingface/chat/transformation.py | 2 +- litellm/llms/langflow/chat/transformation.py | 6 +- litellm/llms/langgraph/chat/sse_iterator.py | 6 +- litellm/llms/langgraph/chat/transformation.py | 12 +- .../litellm_proxy/skills/code_execution.py | 12 +- litellm/llms/litellm_proxy/skills/handler.py | 10 +- .../litellm_proxy/skills/prompt_injection.py | 8 +- .../litellm_proxy/skills/sandbox_executor.py | 18 +- litellm/llms/manus/files/transformation.py | 6 +- .../llms/manus/responses/transformation.py | 10 +- litellm/llms/mistral/ocr/transformation.py | 6 +- litellm/llms/ollama/common_utils.py | 4 +- .../llms/ollama/completion/transformation.py | 2 +- .../openai/chat/o_series_transformation.py | 2 +- litellm/llms/openai/cost_calculation.py | 20 +- .../image_generation/cost_calculator.py | 2 +- litellm/llms/openai/openai.py | 2 +- .../openai/responses/count_tokens/handler.py | 14 +- .../responses/count_tokens/token_counter.py | 4 +- .../llms/openai/responses/transformation.py | 14 +- litellm/llms/openai_like/dynamic_config.py | 5 +- litellm/llms/openai_like/json_loader.py | 2 +- .../llms/perplexity/chat/transformation.py | 6 +- .../image_generation/transformation.py | 6 +- .../runwayml/text_to_speech/transformation.py | 6 +- litellm/llms/sagemaker/common_utils.py | 10 +- litellm/llms/sap/credentials.py | 8 +- litellm/llms/together_ai/chat.py | 2 +- .../vertex_ai/agent_engine/transformation.py | 14 +- litellm/llms/vertex_ai/common_utils.py | 3 +- litellm/llms/vertex_ai/cost_calculator.py | 6 +- .../vertex_and_google_ai_studio_gemini.py | 35 +- .../vertex_ai/ocr/deepseek_transformation.py | 4 +- litellm/llms/vertex_ai/ocr/transformation.py | 10 +- .../llms/vertex_ai/rag_engine/ingestion.py | 10 +- litellm/llms/vertex_ai/vertex_llm_base.py | 25 +- litellm/llms/watsonx/chat/transformation.py | 2 +- litellm/llms/xai/chat/transformation.py | 6 +- litellm/main.py | 18 +- litellm/ocr/main.py | 16 +- .../mcp_server/auth/user_api_key_auth_mcp.py | 100 ++--- litellm/proxy/_experimental/mcp_server/db.py | 2 +- .../mcp_server/discoverable_endpoints.py | 7 +- .../mcp_server/mcp_server_manager.py | 130 +++--- .../mcp_server/openapi_to_mcp_generator.py | 11 +- .../mcp_server/rest_endpoints.py | 26 +- .../mcp_server/semantic_tool_filter.py | 17 +- .../proxy/_experimental/mcp_server/server.py | 137 ++++--- .../_experimental/mcp_server/sse_transport.py | 22 +- .../_experimental/mcp_server/tool_registry.py | 4 +- .../_experimental/mcp_server/toolset_db.py | 2 +- .../mcp_server/ui_session_utils.py | 2 +- .../proxy/agent_endpoints/a2a_endpoints.py | 14 +- litellm/proxy/agent_endpoints/a2a_routing.py | 6 +- .../auth/agent_permission_handler.py | 14 +- litellm/proxy/agent_endpoints/endpoints.py | 28 +- .../agent_endpoints/model_list_helpers.py | 4 +- .../claude_code_marketplace.py | 26 +- .../proxy/anthropic_endpoints/endpoints.py | 4 +- litellm/proxy/auth/auth_checks.py | 47 ++- litellm/proxy/auth/auth_exception_handler.py | 4 +- litellm/proxy/auth/auth_utils.py | 23 +- litellm/proxy/auth/handle_jwt.py | 36 +- litellm/proxy/auth/litellm_license.py | 23 +- litellm/proxy/auth/model_checks.py | 8 +- litellm/proxy/auth/oauth2_proxy_hook.py | 2 +- litellm/proxy/auth/resolvers/store.py | 4 +- litellm/proxy/auth/route_checks.py | 5 +- litellm/proxy/auth/user_api_key_auth.py | 31 +- litellm/proxy/batches_endpoints/endpoints.py | 29 +- litellm/proxy/caching_routes.py | 4 +- litellm/proxy/client/cli/interface.py | 2 +- litellm/proxy/common_request_processing.py | 20 +- litellm/proxy/common_utils/callback_utils.py | 6 +- .../proxy/common_utils/custom_openapi_spec.py | 14 +- litellm/proxy/common_utils/debug_utils.py | 25 +- .../common_utils/encrypt_decrypt_utils.py | 4 +- .../expired_ui_session_key_cleanup_manager.py | 2 +- litellm/proxy/common_utils/get_routes.py | 2 +- .../proxy/common_utils/http_parsing_utils.py | 38 +- .../common_utils/key_rotation_manager.py | 8 +- .../proxy/common_utils/load_config_utils.py | 24 +- .../common_utils/openapi_schema_compat.py | 4 +- .../proxy/common_utils/performance_utils.py | 20 +- litellm/proxy/custom_prompt_management.py | 5 +- litellm/proxy/db/check_migration.py | 3 +- litellm/proxy/db/create_views.py | 2 +- litellm/proxy/db/db_spend_update_writer.py | 47 ++- .../db_transaction_queue/pod_lock_manager.py | 4 +- .../db_transaction_queue/spend_log_cleanup.py | 33 +- litellm/proxy/db/dynamo_db.py | 4 +- litellm/proxy/db/prisma_client.py | 23 +- .../example_config_yaml/custom_guardrail.py | 4 +- .../proxy/fine_tuning_endpoints/endpoints.py | 15 +- .../proxy/guardrails/guardrail_endpoints.py | 38 +- .../guardrails/guardrail_hooks/aim/aim.py | 4 +- .../guardrail_hooks/azure/prompt_shield.py | 4 +- .../guardrail_hooks/azure/text_moderation.py | 4 +- .../guardrail_hooks/bedrock_guardrails.py | 2 +- .../cato_networks/cato_networks.py | 4 +- .../crowdstrike_aidr/crowdstrike_aidr.py | 10 +- .../custom_code/custom_code_guardrail.py | 19 +- .../guardrail_hooks/custom_code/primitives.py | 20 +- .../hiddenlayer/hiddenlayer.py | 8 +- .../guardrails/guardrail_hooks/lasso/lasso.py | 27 +- .../litellm_content_filter/content_filter.py | 93 +++-- .../litellm_content_filter/patterns.py | 2 +- .../llm_as_a_judge/__init__.py | 6 +- .../mcp_end_user_permission.py | 11 +- .../guardrails/guardrail_hooks/noma/noma.py | 28 +- .../guardrails/guardrail_hooks/onyx/onyx.py | 10 +- .../guardrail_hooks/openai/moderations.py | 2 +- .../guardrail_hooks/pangea/pangea.py | 15 +- .../panw_prisma_airs/panw_prisma_airs.py | 82 ++-- .../guardrail_hooks/pillar/pillar.py | 34 +- .../guardrails/guardrail_hooks/presidio.py | 10 +- .../prompt_security/prompt_security.py | 8 +- .../guardrail_hooks/qualifire/qualifire.py | 8 +- .../semantic_guard/route_loader.py | 2 +- .../semantic_guard/semantic_guard.py | 9 +- .../guardrail_hooks/tool_permission.py | 16 +- .../zscaler_ai_guard/zscaler_ai_guard.py | 39 +- .../proxy/guardrails/guardrail_registry.py | 32 +- litellm/proxy/guardrails/init_guardrails.py | 6 +- .../health_endpoints/_health_endpoints.py | 20 +- litellm/proxy/hooks/azure_content_safety.py | 2 +- litellm/proxy/hooks/batch_rate_limiter.py | 16 +- litellm/proxy/hooks/batch_redis_get.py | 2 +- litellm/proxy/hooks/cache_control_check.py | 2 +- litellm/proxy/hooks/dynamic_rate_limiter.py | 10 +- .../proxy/hooks/dynamic_rate_limiter_v3.py | 24 +- .../proxy/hooks/key_management_event_hooks.py | 15 +- litellm/proxy/hooks/litellm_skills/main.py | 52 +-- litellm/proxy/hooks/max_budget_limiter.py | 2 +- .../proxy/hooks/mcp_semantic_filter/hook.py | 60 +-- .../proxy/hooks/model_max_budget_limiter.py | 4 +- .../proxy/hooks/parallel_request_limiter.py | 4 +- .../hooks/parallel_request_limiter_v3.py | 76 ++-- .../proxy/hooks/prompt_injection_detection.py | 2 +- .../proxy/hooks/proxy_track_cost_callback.py | 12 +- litellm/proxy/hooks/responses_id_security.py | 9 +- .../hooks/user_management_event_hooks.py | 2 +- litellm/proxy/image_endpoints/endpoints.py | 2 +- litellm/proxy/litellm_pre_call_utils.py | 54 +-- .../cache_settings_endpoints.py | 9 +- .../common_daily_activity.py | 4 +- .../management_endpoints/common_utils.py | 4 +- .../cost_tracking_settings.py | 20 +- .../customer_endpoints.py | 22 +- .../fallback_management_endpoints.py | 10 +- .../internal_user_endpoints.py | 36 +- .../key_management_endpoints.py | 56 +-- .../management_v1/budgets.py | 2 +- .../management_v1/spend_logs.py | 4 +- .../mcp_management_endpoints.py | 34 +- ...model_access_group_management_endpoints.py | 28 +- .../model_management_endpoints.py | 24 +- .../organization_endpoints.py | 8 +- .../policy_endpoints/endpoints.py | 2 +- .../router_settings_endpoints.py | 4 +- .../management_endpoints/scim/scim_v2.py | 38 +- .../sso/custom_microsoft_sso.py | 8 +- .../management_endpoints/sso/saml_sso.py | 4 +- .../tag_management_endpoints.py | 8 +- .../team_callback_endpoints.py | 6 +- .../management_endpoints/team_endpoints.py | 20 +- litellm/proxy/management_endpoints/ui_sso.py | 171 ++++---- .../proxy/management_helpers/audit_logs.py | 2 +- .../object_permission_utils.py | 6 +- litellm/proxy/ocr_endpoints/endpoints.py | 7 +- .../openai_files_endpoints/common_utils.py | 22 +- .../openai_files_endpoints/files_endpoints.py | 14 +- .../storage_backend_service.py | 8 +- .../llm_passthrough_endpoints.py | 16 +- .../anthropic_passthrough_logging_handler.py | 14 +- .../assembly_passthrough_logging_handler.py | 4 +- .../openai_passthrough_logging_handler.py | 12 +- ...tex_ai_live_passthrough_logging_handler.py | 12 +- .../vertex_passthrough_logging_handler.py | 14 +- .../pass_through_endpoints.py | 79 ++-- .../passthrough_endpoint_router.py | 8 +- .../streaming_handler.py | 6 +- .../policy_engine/attachment_registry.py | 34 +- .../policy_engine/condition_evaluator.py | 4 +- litellm/proxy/policy_engine/init_policies.py | 14 +- .../proxy/policy_engine/pipeline_executor.py | 11 +- .../proxy/policy_engine/policy_endpoints.py | 32 +- .../proxy/policy_engine/policy_registry.py | 46 ++- .../policy_engine/policy_resolve_endpoints.py | 4 +- .../proxy/policy_engine/policy_resolver.py | 16 +- .../proxy/policy_engine/policy_validator.py | 8 +- litellm/proxy/prisma_migration.py | 6 +- litellm/proxy/prometheus_cleanup.py | 8 +- litellm/proxy/prompts/init_prompts.py | 2 +- litellm/proxy/prompts/prompt_endpoints.py | 10 +- litellm/proxy/prompts/prompt_registry.py | 12 +- litellm/proxy/proxy_cli.py | 4 +- litellm/proxy/proxy_server.py | 386 ++++++++++-------- litellm/proxy/rag_endpoints/endpoints.py | 29 +- litellm/proxy/rerank_endpoints/endpoints.py | 2 +- .../proxy/response_api_endpoints/endpoints.py | 13 +- .../response_polling/background_streaming.py | 17 +- .../proxy/response_polling/polling_handler.py | 12 +- litellm/proxy/route_llm_request.py | 4 +- litellm/proxy/search_endpoints/endpoints.py | 11 +- .../search_tool_management.py | 30 +- .../search_endpoints/search_tool_registry.py | 12 +- .../spend_tracking/cloudzero_endpoints.py | 16 +- .../spend_management_endpoints.py | 12 +- .../proxy/spend_tracking/vantage_endpoints.py | 16 +- litellm/proxy/types_utils/utils.py | 13 +- .../proxy_setting_endpoints.py | 7 +- litellm/proxy/utils.py | 69 ++-- .../management_endpoints.py | 35 +- .../vector_store_files_endpoints/endpoints.py | 24 +- litellm/rag/ingestion/base_ingestion.py | 2 +- litellm/rag/ingestion/bedrock_ingestion.py | 58 +-- .../rag/ingestion/file_parsers/pdf_parser.py | 6 +- litellm/rag/ingestion/gemini_ingestion.py | 14 +- litellm/rag/ingestion/s3_vectors_ingestion.py | 62 +-- litellm/rag/ingestion/vertex_ai_ingestion.py | 26 +- litellm/repositories/config_repository.py | 2 +- litellm/rerank_api/main.py | 4 +- .../responses/mcp/chat_completions_handler.py | 4 +- .../mcp/litellm_proxy_mcp_handler.py | 18 +- .../responses/mcp/mcp_streaming_iterator.py | 22 +- litellm/responses/utils.py | 4 +- litellm/router.py | 236 ++++++----- .../auto_router/auto_router.py | 2 +- .../router_strategy/base_routing_strategy.py | 6 +- litellm/router_strategy/budget_limiter.py | 23 +- .../complexity_router/complexity_router.py | 32 +- litellm/router_strategy/lar1_routing.py | 8 +- litellm/router_strategy/lowest_cost.py | 10 +- litellm/router_strategy/lowest_latency.py | 6 +- litellm/router_strategy/lowest_tpm_rpm.py | 12 +- litellm/router_strategy/lowest_tpm_rpm_v2.py | 14 +- .../quality_router/quality_router.py | 16 +- litellm/router_strategy/simple_shuffle.py | 11 +- litellm/router_utils/batch_utils.py | 4 +- litellm/router_utils/cooldown_cache.py | 4 +- litellm/router_utils/cooldown_callbacks.py | 3 +- litellm/router_utils/cooldown_handlers.py | 8 +- .../router_utils/fallback_event_handlers.py | 6 +- litellm/router_utils/handle_error.py | 4 +- .../router_utils/pattern_match_deployments.py | 2 +- .../io_token_rate_limit_check.py | 32 +- .../pre_call_checks/model_rate_limit_check.py | 16 +- litellm/router_utils/search_api_router.py | 15 +- litellm/sandbox/main.py | 2 +- litellm/search/main.py | 4 +- .../cyberark_secret_manager.py | 22 +- .../get_azure_ad_token_provider.py | 2 +- .../hashicorp_secret_manager.py | 32 +- litellm/secret_managers/main.py | 5 +- .../guardrail_hooks/zscaler_ai_guard.py | 2 +- litellm/types/videos/utils.py | 4 +- litellm/utils.py | 103 +++-- .../vector_stores/vector_store_registry.py | 9 +- .../test_mcp_client.py | 8 +- .../integrations/newrelic/test_newrelic.py | 12 +- .../test_anthropic_cache_control_hook.py | 10 +- .../llms/azure/test_azure_common_utils.py | 2 +- .../mcp_server/test_mcp_server.py | 25 +- .../mcp_server/test_rest_endpoints.py | 14 +- .../proxy/auth/test_auth_checks.py | 8 +- .../proxy/auth/test_model_checks_fallbacks.py | 10 +- .../test_team_endpoints.py | 3 +- tests/test_litellm/test_logging.py | 65 +++ 419 files changed, 3919 insertions(+), 3140 deletions(-) diff --git a/litellm/_redis.py b/litellm/_redis.py index 693f9582705..e05e9d4eb20 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -651,7 +651,9 @@ def get_redis_async_client( if arg in args: url_kwargs[arg] = redis_kwargs[arg] else: - verbose_logger.debug(f"REDIS: ignoring argument: {arg}. Not an allowed async_redis.Redis.from_url arg.") + verbose_logger.debug( + "REDIS: ignoring argument: %s. Not an allowed async_redis.Redis.from_url arg.", arg + ) return async_redis.Redis.from_url(**url_kwargs) # Check for Redis Sentinel @@ -805,6 +807,6 @@ def _pretty_print_redis_config(redis_kwargs: dict) -> None: # Fallback to simple logging if rich is not available masker = SensitiveDataMasker() masked_redis_kwargs = masker.mask_dict(redis_kwargs) - verbose_logger.info(f"Redis configuration: {masked_redis_kwargs}") + verbose_logger.info("Redis configuration: %s", masked_redis_kwargs) except Exception as e: - verbose_logger.error(f"Error pretty printing Redis configuration: {e}") + verbose_logger.error("Error pretty printing Redis configuration: %s", e) diff --git a/litellm/a2a_protocol/card_resolver.py b/litellm/a2a_protocol/card_resolver.py index e4cce56d0e4..81a7813d14c 100644 --- a/litellm/a2a_protocol/card_resolver.py +++ b/litellm/a2a_protocol/card_resolver.py @@ -148,13 +148,13 @@ class LiteLLMA2ACardResolver(_A2ACardResolver): # type: ignore[misc] last_error = None for path in paths: try: - verbose_logger.debug(f"Attempting to fetch agent card from {self.base_url}{path}") + verbose_logger.debug("Attempting to fetch agent card from %s%s", self.base_url, path) return await super().get_agent_card( relative_card_path=path, http_kwargs=http_kwargs, ) except Exception as e: - verbose_logger.debug(f"Failed to fetch agent card from {self.base_url}{path}: {e}") + verbose_logger.debug("Failed to fetch agent card from %s%s: %s", self.base_url, path, e) last_error = e continue diff --git a/litellm/a2a_protocol/exception_mapping_utils.py b/litellm/a2a_protocol/exception_mapping_utils.py index 4d24dd4f1d8..16979667fe5 100644 --- a/litellm/a2a_protocol/exception_mapping_utils.py +++ b/litellm/a2a_protocol/exception_mapping_utils.py @@ -192,9 +192,11 @@ async def handle_a2a_localhost_retry( request_type = "streaming " if is_streaming else "" verbose_logger.warning( - f"A2A {request_type}request to '{error.localhost_url}' failed: {error.original_error}. " - f"Agent card contains localhost/internal URL. " - f"Retrying with base_url '{error.base_url}'." + "A2A %srequest to '%s' failed: %s. Agent card contains localhost/internal URL. Retrying with base_url '%s'.", + request_type, + error.localhost_url, + error.original_error, + error.base_url, ) # Fix the agent card URL diff --git a/litellm/a2a_protocol/litellm_completion_bridge/handler.py b/litellm/a2a_protocol/litellm_completion_bridge/handler.py index 1d46e5c700f..21366602d1a 100644 --- a/litellm/a2a_protocol/litellm_completion_bridge/handler.py +++ b/litellm/a2a_protocol/litellm_completion_bridge/handler.py @@ -76,7 +76,7 @@ class A2ACompletionBridgeHandler: ) if a2a_provider_config is not None: - verbose_logger.info(f"A2A: Using provider config for {custom_llm_provider}") + verbose_logger.info("A2A: Using provider config for %s", custom_llm_provider) return await a2a_provider_config.handle_non_streaming( request_id=request_id, @@ -103,7 +103,7 @@ class A2ACompletionBridgeHandler: else: full_model = model - verbose_logger.info(f"A2A completion bridge: model={full_model}, api_base={api_base}") + verbose_logger.info("A2A completion bridge: model=%s, api_base=%s", full_model, api_base) # Build completion params dict completion_params: dict[str, Any] = { @@ -143,7 +143,7 @@ class A2ACompletionBridgeHandler: request_id=request_id, ) - verbose_logger.info(f"A2A completion bridge completed: request_id={request_id}") + verbose_logger.info("A2A completion bridge completed: request_id=%s", request_id) return a2a_response @@ -185,7 +185,7 @@ class A2ACompletionBridgeHandler: ) if a2a_provider_config is not None: - verbose_logger.info(f"A2A: Using provider config for {custom_llm_provider} (streaming)") + verbose_logger.info("A2A: Using provider config for %s (streaming)", custom_llm_provider) async for chunk in a2a_provider_config.handle_streaming( request_id=request_id, @@ -221,7 +221,7 @@ class A2ACompletionBridgeHandler: else: full_model = model - verbose_logger.info(f"A2A completion bridge streaming: model={full_model}, api_base={api_base}") + verbose_logger.info("A2A completion bridge streaming: model=%s, api_base=%s", full_model, api_base) # Build completion params dict completion_params: dict[str, Any] = { @@ -300,7 +300,9 @@ class A2ACompletionBridgeHandler: ) yield completed_event - verbose_logger.info(f"A2A completion bridge streaming completed: request_id={request_id}, chunks={chunk_count}") + verbose_logger.info( + "A2A completion bridge streaming completed: request_id=%s, chunks=%s", request_id, chunk_count + ) # Convenience functions that delegate to the class methods diff --git a/litellm/a2a_protocol/litellm_completion_bridge/transformation.py b/litellm/a2a_protocol/litellm_completion_bridge/transformation.py index a63221b4a77..e216abb6c6d 100644 --- a/litellm/a2a_protocol/litellm_completion_bridge/transformation.py +++ b/litellm/a2a_protocol/litellm_completion_bridge/transformation.py @@ -109,7 +109,7 @@ class A2ACompletionBridgeTransformation: extra_body = {**extra_body, "metadata": merged_metadata} completion_params["extra_body"] = extra_body - verbose_logger.debug(f"A2A -> completion forward metadata keys={list(forward_metadata.keys())}") + verbose_logger.debug("A2A -> completion forward metadata keys=%s", list(forward_metadata.keys())) @staticmethod def a2a_message_to_openai_messages( @@ -145,7 +145,9 @@ class A2ACompletionBridgeTransformation: # once at run level via extra_body.metadata (LangGraph POST /runs/wait shape). openai_message: dict[str, Any] = {"role": openai_role, "content": content} - verbose_logger.debug(f"A2A -> OpenAI transform: role={role} -> {openai_role}, content_length={len(content)}") + verbose_logger.debug( + "A2A -> OpenAI transform: role=%s -> %s, content_length=%s", role, openai_role, len(content) + ) return [openai_message] @@ -186,7 +188,7 @@ class A2ACompletionBridgeTransformation: "result": a2a_message, } - verbose_logger.debug(f"OpenAI -> A2A transform: content_length={len(content)}") + verbose_logger.debug("OpenAI -> A2A transform: content_length=%s", len(content)) return a2a_response diff --git a/litellm/a2a_protocol/main.py b/litellm/a2a_protocol/main.py index 52d35a988c6..ec2d3ccf1f7 100644 --- a/litellm/a2a_protocol/main.py +++ b/litellm/a2a_protocol/main.py @@ -204,7 +204,7 @@ async def _send_message_via_completion_bridge( Requires request; api_base is optional for providers that derive endpoint from model. """ - verbose_logger.info(f"A2A using completion bridge: provider={custom_llm_provider}, api_base={api_base}") + verbose_logger.info("A2A using completion bridge: provider=%s, api_base=%s", custom_llm_provider, api_base) from litellm.a2a_protocol.litellm_completion_bridge.handler import ( A2ACompletionBridgeHandler, @@ -463,7 +463,7 @@ async def asend_message( agent_name = _get_a2a_model_info(a2a_client, kwargs) - verbose_logger.info(f"A2A send_message request_id={request.id}, agent={agent_name}") + verbose_logger.info("A2A send_message request_id=%s, agent=%s", request.id, agent_name) # Get agent card URL for localhost retry logic agent_card = _get_a2a_client_agent_card(a2a_client) @@ -478,7 +478,7 @@ async def asend_message( agent_name=agent_name, ) - verbose_logger.info(f"A2A send_message completed, request_id={request.id}") + verbose_logger.info("A2A send_message completed, request_id=%s", request.id) # Wrap in LiteLLM response type for _hidden_params support response = LiteLLMSendMessageResponse.from_a2a_response(a2a_response, request_id=str(request.id)) @@ -640,7 +640,7 @@ async def asend_message_streaming( raise ValueError("request is required for completion bridge") # api_base is optional for providers that derive endpoint from model (e.g., bedrock/agentcore) - verbose_logger.info(f"A2A streaming using completion bridge: provider={custom_llm_provider}") + verbose_logger.info("A2A streaming using completion bridge: provider=%s", custom_llm_provider) from litellm.a2a_protocol.litellm_completion_bridge.handler import ( A2ACompletionBridgeHandler, @@ -697,7 +697,7 @@ async def asend_message_streaming( proxy_server_request=proxy_server_request, ) - verbose_logger.info(f"A2A send_message_streaming request_id={request.id}, agent={agent_name}") + verbose_logger.info("A2A send_message_streaming request_id=%s, agent=%s", request.id, agent_name) agent_card = _get_a2a_client_agent_card(a2a_client) card_url = get_agent_card_url(agent_card) if agent_card else None @@ -759,7 +759,7 @@ async def create_a2a_client( "The 'a2a' package is required for A2A agent invocation. Install it with: pip install a2a-sdk" ) - verbose_logger.info(f"Creating A2A client for {base_url}") + verbose_logger.info("Creating A2A client for %s", base_url) # Use get_async_httpx_client with per-agent params so that different agents # (with different extra_headers) get separate cached clients. The params @@ -781,7 +781,7 @@ async def create_a2a_client( httpx_client = _async_handler.client if extra_headers: httpx_client.headers.update(extra_headers) - verbose_proxy_logger.debug(f"A2A client created with extra_headers={list(extra_headers.keys())}") + verbose_proxy_logger.debug("A2A client created with extra_headers=%s", list(extra_headers.keys())) a2a_client = await create_client( # pyright: ignore[reportOptionalCall] base_url, @@ -798,7 +798,7 @@ async def create_a2a_client( if agent_card is not None: a2a_client._litellm_agent_card = agent_card # type: ignore[attr-defined] - verbose_logger.info(f"A2A client created for {base_url}") + verbose_logger.info("A2A client created for %s", base_url) return a2a_client @@ -824,7 +824,7 @@ async def aget_agent_card( "The 'a2a' package is required for A2A agent invocation. Install it with: pip install a2a-sdk" ) - verbose_logger.info(f"Fetching agent card from {base_url}") + verbose_logger.info("Fetching agent card from %s", base_url) # Use LiteLLM's cached httpx client http_handler = get_async_httpx_client( @@ -839,5 +839,5 @@ async def aget_agent_card( ) agent_card = await resolver.get_agent_card() - verbose_logger.info(f"Fetched agent card: {agent_card.name if hasattr(agent_card, 'name') else 'unknown'}") + verbose_logger.info("Fetched agent card: %s", agent_card.name if hasattr(agent_card, "name") else "unknown") return agent_card diff --git a/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py b/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py index 56f5f806e7b..d19137ef4a9 100644 --- a/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py +++ b/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py @@ -53,7 +53,7 @@ class BedrockAgentCoreA2AHandler: agent_extra_headers=agent_extra_headers, ) - verbose_logger.info(f"BedrockAgentCore A2A: Sending non-streaming request to {url}") + verbose_logger.info("BedrockAgentCore A2A: Sending non-streaming request to %s", url) client = get_async_httpx_client( llm_provider=cast(Any, httpxSpecialProvider.A2AProvider), @@ -67,7 +67,7 @@ class BedrockAgentCoreA2AHandler: response_data = response.json() if "error" in response_data: - verbose_logger.warning(f"BedrockAgentCore A2A: Agent returned error: {response_data['error']}") + verbose_logger.warning("BedrockAgentCore A2A: Agent returned error: %s", response_data["error"]) return response_data @@ -100,7 +100,7 @@ class BedrockAgentCoreA2AHandler: agent_extra_headers=agent_extra_headers, ) - verbose_logger.info(f"BedrockAgentCore A2A: Sending streaming request to {url}") + verbose_logger.info("BedrockAgentCore A2A: Sending streaming request to %s", url) client = get_async_httpx_client( llm_provider=cast(Any, httpxSpecialProvider.A2AProvider), diff --git a/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py b/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py index f9343d2d3b4..0c1e01e7f9f 100644 --- a/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py +++ b/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py @@ -195,5 +195,5 @@ class BedrockAgentCoreA2ATransformation: event = json.loads(data_str) yield event except json.JSONDecodeError: - verbose_logger.debug(f"BedrockAgentCore A2A: Skipping non-JSON SSE line: {data_str[:100]}") + verbose_logger.debug("BedrockAgentCore A2A: Skipping non-JSON SSE line: %s", data_str[:100]) continue diff --git a/litellm/a2a_protocol/providers/pydantic_ai_agents/handler.py b/litellm/a2a_protocol/providers/pydantic_ai_agents/handler.py index 86cb2d47ad3..da1fc1eb657 100644 --- a/litellm/a2a_protocol/providers/pydantic_ai_agents/handler.py +++ b/litellm/a2a_protocol/providers/pydantic_ai_agents/handler.py @@ -47,7 +47,7 @@ class PydanticAIHandler: """ if api_base is None: raise ValueError("api_base is required for Pydantic AI agents") - verbose_logger.info(f"Pydantic AI: Routing to Pydantic AI agent at {api_base}") + verbose_logger.info("Pydantic AI: Routing to Pydantic AI agent at %s", api_base) # Send request directly to Pydantic AI agent response_data = await PydanticAITransformation.send_non_streaming_request( @@ -92,7 +92,7 @@ class PydanticAIHandler: """ if api_base is None: raise ValueError("api_base is required for Pydantic AI agents") - verbose_logger.info(f"Pydantic AI: Faking streaming for Pydantic AI agent at {api_base}") + verbose_logger.info("Pydantic AI: Faking streaming for Pydantic AI agent at %s", api_base) # Get raw task response first (not the transformed A2A format) raw_response = await PydanticAITransformation.send_and_get_raw_response( diff --git a/litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py b/litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py index 37127f2fcab..d8b22282d3a 100644 --- a/litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py +++ b/litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py @@ -118,7 +118,7 @@ class PydanticAITransformation: status = result.get("status", {}) state = status.get("state", "") - verbose_logger.debug(f"Pydantic AI: Poll attempt {attempt + 1}/{max_attempts}, state={state}") + verbose_logger.debug("Pydantic AI: Poll attempt %s/%s, state=%s", attempt + 1, max_attempts, state) if state == "completed": return poll_data @@ -173,7 +173,7 @@ class PydanticAITransformation: # FastA2A uses root endpoint (/) not /messages endpoint = api_base.rstrip("/") - verbose_logger.info(f"Pydantic AI: Sending non-streaming request to {endpoint}") + verbose_logger.info("Pydantic AI: Sending non-streaming request to %s", endpoint) # Send request to Pydantic AI agent using shared async HTTP client client = get_async_httpx_client( @@ -200,7 +200,7 @@ class PydanticAITransformation: # Need to poll for completion task_id = result.get("id") if task_id: - verbose_logger.info(f"Pydantic AI: Task {task_id} submitted, polling for completion...") + verbose_logger.info("Pydantic AI: Task %s submitted, polling for completion...", task_id) response_data = await PydanticAITransformation._poll_for_completion( client=client, endpoint=endpoint, @@ -209,7 +209,7 @@ class PydanticAITransformation: agent_extra_headers=agent_extra_headers, ) - verbose_logger.info(f"Pydantic AI: Received completed response for request_id={request_id}") + verbose_logger.info("Pydantic AI: Received completed response for request_id=%s", request_id) return response_data @@ -518,4 +518,4 @@ class PydanticAITransformation: } yield completed_event - verbose_logger.info(f"Pydantic AI: Fake streaming completed for request_id={request_id}") + verbose_logger.info("Pydantic AI: Fake streaming completed for request_id=%s", request_id) diff --git a/litellm/a2a_protocol/providers/watsonx_orchestrate/handler.py b/litellm/a2a_protocol/providers/watsonx_orchestrate/handler.py index efb2b38b912..2c7c04cec0b 100644 --- a/litellm/a2a_protocol/providers/watsonx_orchestrate/handler.py +++ b/litellm/a2a_protocol/providers/watsonx_orchestrate/handler.py @@ -135,7 +135,7 @@ class WatsonxOrchestrateHandler: response.raise_for_status() result: dict[str, Any] = response.json() status = result.get("status", "") - verbose_logger.debug(f"WXO: Poll {attempt + 1}/{max_attempts} run='{run_id}' status='{status}'") + verbose_logger.debug("WXO: Poll %s/%s run='%s' status='%s'", attempt + 1, max_attempts, run_id, status) if status in WatsonxOrchestrateTransformation.TERMINAL_STATES: return result @@ -297,8 +297,8 @@ class WatsonxOrchestrateHandler: response.raise_for_status() except httpx.TransportError as exc: verbose_logger.warning( - f"WXO: Streaming request failed before a run was submitted " - f"({exc!r}), falling back to non-streaming + fake streaming", + "WXO: Streaming request failed before a run was submitted (%r), falling back to non-streaming + fake streaming", + exc, exc_info=True, ) result = await WatsonxOrchestrateHandler.handle_non_streaming( diff --git a/litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py b/litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py index ab7b8abb3ba..18b0795aa8a 100644 --- a/litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py +++ b/litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py @@ -214,4 +214,4 @@ class WatsonxOrchestrateTransformation: }, } - verbose_logger.debug(f"WXO: Fake streaming completed for request_id={request_id}") + verbose_logger.debug("WXO: Fake streaming completed for request_id=%s", request_id) diff --git a/litellm/a2a_protocol/streaming_iterator.py b/litellm/a2a_protocol/streaming_iterator.py index 79056ca336f..f954cb187b5 100644 --- a/litellm/a2a_protocol/streaming_iterator.py +++ b/litellm/a2a_protocol/streaming_iterator.py @@ -138,13 +138,15 @@ class A2AStreamingIterator: ) verbose_logger.info( - f"A2A streaming completed: prompt_tokens={prompt_tokens}, " - f"completion_tokens={completion_tokens}, total_tokens={total_tokens}, " - f"response_cost={response_cost}" + "A2A streaming completed: prompt_tokens=%s, completion_tokens=%s, total_tokens=%s, response_cost=%s", + prompt_tokens, + completion_tokens, + total_tokens, + response_cost, ) except Exception as e: - verbose_logger.debug(f"Error in A2A streaming completion handler: {e}") + verbose_logger.debug("Error in A2A streaming completion handler: %s", e) def _build_logging_result(self, usage: litellm.Usage) -> dict[str, Any]: """Build a result dict for logging.""" diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index 542885b5130..063a83e6f38 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -51,7 +51,7 @@ class GetAnthropicBetaHeadersConfig: ) return content except Exception as e: - verbose_logger.error(f"Failed to load local beta headers config: {e}") + verbose_logger.error("Failed to load local beta headers config: %s", e) # Return empty config as fallback return { "anthropic": {}, @@ -246,7 +246,9 @@ def filter_and_transform_beta_headers( # Check if header is in the mapping if header not in provider_mapping: - verbose_logger.debug(f"Dropping unknown beta header '{header}' for provider '{provider}' (not in mapping)") + verbose_logger.debug( + "Dropping unknown beta header '%s' for provider '%s' (not in mapping)", header, provider + ) continue # Get the mapped header value @@ -254,7 +256,7 @@ def filter_and_transform_beta_headers( # Skip if header is unsupported (null value) if mapped_header is None: - verbose_logger.debug(f"Dropping unsupported beta header '{header}' for provider '{provider}'") + verbose_logger.debug("Dropping unsupported beta header '%s' for provider '%s'", header, provider) continue # Add the mapped header diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index 2e28aaa14df..4f90b50eaa2 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -258,10 +258,10 @@ async def _fetch_batch_output_file_content( if is_base64_unified_file_id: try: file_id = is_base64_unified_file_id.split("llm_output_file_id,")[1].split(";")[0] - verbose_logger.debug(f"Extracted LLM output file ID from unified file ID: {file_id}") + verbose_logger.debug("Extracted LLM output file ID from unified file ID: %s", file_id) except (IndexError, AttributeError) as e: verbose_logger.error( - f"Failed to extract LLM output file ID from unified file ID: {batch.output_file_id}, error: {e}" + "Failed to extract LLM output file ID from unified file ID: %s, error: %s", batch.output_file_id, e ) # Build kwargs for afile_content with credentials from litellm_params diff --git a/litellm/batches/main.py b/litellm/batches/main.py index 3a057d41744..33f2f4613bf 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -182,7 +182,7 @@ def create_batch( ) except Exception as e: verbose_logger.exception( - f"litellm.batches.main.py::create_batch() - Error inferring custom_llm_provider - {e}" + "litellm.batches.main.py::create_batch() - Error inferring custom_llm_provider - %s", e ) _is_async = kwargs.pop("acreate_batch", False) is True @@ -890,7 +890,7 @@ def cancel_batch( ) except Exception as e: verbose_logger.exception( - f"litellm.batches.main.py::cancel_batch() - Error inferring custom_llm_provider - {e}" + "litellm.batches.main.py::cancel_batch() - Error inferring custom_llm_provider - %s", e ) optional_params = GenericLiteLLMParams(**kwargs) litellm_params = get_litellm_params( diff --git a/litellm/caching/azure_blob_cache.py b/litellm/caching/azure_blob_cache.py index 80ad645ec7b..755b491f9a0 100644 --- a/litellm/caching/azure_blob_cache.py +++ b/litellm/caching/azure_blob_cache.py @@ -67,7 +67,10 @@ class AzureBlobCache(BaseCache): cached_response = json.loads(as_str) verbose_logger.debug( - f"Got Azure Blob Cache: key: {key}, cached_response {cached_response}. Type Response {type(cached_response)}" + "Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s", + key, + cached_response, + type(cached_response), ) return cached_response @@ -84,7 +87,10 @@ class AzureBlobCache(BaseCache): as_str = as_bytes.decode("utf-8") cached_response = json.loads(as_str) verbose_logger.debug( - f"Got Azure Blob Cache: key: {key}, cached_response {cached_response}. Type Response {type(cached_response)}" + "Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s", + key, + cached_response, + type(cached_response), ) return cached_response except ResourceNotFoundError: diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 9542be0999a..758a14afb17 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -676,7 +676,7 @@ class Cache: cache_key, cached_data, kwargs = self._add_cache_logic(result=result, **kwargs) self.cache.set_cache(cache_key, cached_data, **kwargs) except Exception as e: - verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {e}") + verbose_logger.exception("LiteLLM Cache: Excepton add_cache: %s", e) async def async_add_cache(self, result, dynamic_cache_object: BaseCache | None = None, **kwargs): """ @@ -695,7 +695,7 @@ class Cache: else: await self.cache.async_set_cache(cache_key, cached_data, **kwargs) except Exception as e: - verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {e}") + verbose_logger.exception("LiteLLM Cache: Excepton add_cache: %s", e) def _convert_to_cached_embedding( self, @@ -874,7 +874,7 @@ class Cache: else: await self.cache.async_set_cache_pipeline(cache_list=cache_list, **kwargs) except Exception as e: - verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {e}") + verbose_logger.exception("LiteLLM Cache: Excepton add_cache: %s", e) def should_use_cache(self, **kwargs): """ diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index aed38d6ef65..2655a5ad683 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -271,7 +271,7 @@ class LLMCachingHandler: embedding_all_elements_cache_hit=embedding_all_elements_cache_hit, ) - verbose_logger.debug(f"CACHE RESULT: {cached_result}") + verbose_logger.debug("CACHE RESULT: %s", cached_result) return CachingHandlerResponse( cached_result=cached_result, final_embedding_cached_response=final_embedding_cached_response, diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index b641c600a0e..a242f4a818e 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -147,7 +147,7 @@ class DualCache(BaseCache): return result except Exception as e: - verbose_logger.error(f"LiteLLM Cache: Excepton async add_cache: {e}") + verbose_logger.error("LiteLLM Cache: Excepton async add_cache: %s", e) raise e def get_cache( @@ -347,7 +347,7 @@ class DualCache(BaseCache): if self.redis_cache is not None and local_only is False: await self.redis_cache.async_set_cache(key, value, **kwargs) except Exception as e: - verbose_logger.exception(f"LiteLLM Cache: Excepton async add_cache: {e}") + verbose_logger.exception("LiteLLM Cache: Excepton async add_cache: %s", e) # async_batch_set_cache async def async_set_cache_pipeline(self, cache_list: list, local_only: bool = False, **kwargs): @@ -366,7 +366,7 @@ class DualCache(BaseCache): cache_list=cache_list, ttl=kwargs.pop("ttl", None), **kwargs ) except Exception as e: - verbose_logger.exception(f"LiteLLM Cache: Excepton async add_cache: {e}") + verbose_logger.exception("LiteLLM Cache: Excepton async add_cache: %s", e) async def async_increment_cache( self, diff --git a/litellm/caching/gcs_cache.py b/litellm/caching/gcs_cache.py index d74c68de770..1e1508669b3 100644 --- a/litellm/caching/gcs_cache.py +++ b/litellm/caching/gcs_cache.py @@ -71,12 +71,15 @@ class GCSCache(BaseCache): if response.status_code == 200: cached_response = json.loads(response.text) verbose_logger.debug( - f"Got GCS Cache: key: {key}, cached_response {cached_response}. Type Response {type(cached_response)}" + "Got GCS Cache: key: %s, cached_response %s. Type Response %s", + key, + cached_response, + type(cached_response), ) return cached_response return None except Exception as e: - verbose_logger.error(f"GCS Caching: get_cache() - Got exception from GCS: {e}") + verbose_logger.error("GCS Caching: get_cache() - Got exception from GCS: %s", e) async def async_get_cache(self, key, **kwargs): try: @@ -89,7 +92,7 @@ class GCSCache(BaseCache): return json.loads(response.text) return None except Exception as e: - verbose_logger.error(f"GCS Caching: async_get_cache() - Got exception from GCS: {e}") + verbose_logger.error("GCS Caching: async_get_cache() - Got exception from GCS: %s", e) def flush_cache(self): pass diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index e3c0e3616f0..2dfd123d46d 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -346,7 +346,8 @@ class RedisCache(BaseCache): verbose_logger.debug("Ignoring async redis ping. No running event loop.") else: verbose_logger.error( - f"Error connecting to Async Redis client - {e}", + "Error connecting to Async Redis client - %s", + e, extra={"error": str(e)}, ) self._handle_async_ping_error(e) @@ -1139,7 +1140,7 @@ class RedisCache(BaseCache): return decoded_results except Exception as e: - verbose_logger.error(f"Error occurred in batch get cache - {e}") + verbose_logger.error("Error occurred in batch get cache - %s", e) return key_value_dict @_redis_circuit_breaker_guard @@ -1257,7 +1258,7 @@ class RedisCache(BaseCache): parent_otel_span=parent_otel_span, ) ) - verbose_logger.error(f"Error occurred in async batch get cache - {e}") + verbose_logger.error("Error occurred in async batch get cache - %s", e) _record_swallowed_redis_failure(self._circuit_breaker, e) return key_value_dict @@ -1292,7 +1293,7 @@ class RedisCache(BaseCache): error=e, call_type=f"sync_ping <- {_get_call_stack_info()}", ) - verbose_logger.error(f"LiteLLM Redis Cache PING: - Got exception from REDIS : {e}") + verbose_logger.error("LiteLLM Redis Cache PING: - Got exception from REDIS : %s", e) raise e async def ping(self) -> bool: @@ -1326,7 +1327,7 @@ class RedisCache(BaseCache): call_type=f"async_ping <- {_get_call_stack_info()}", ) ) - verbose_logger.error(f"LiteLLM Redis Cache PING: - Got exception from REDIS : {e}") + verbose_logger.error("LiteLLM Redis Cache PING: - Got exception from REDIS : %s", e) raise e @_redis_circuit_breaker_guard @@ -1388,7 +1389,7 @@ class RedisCache(BaseCache): else: return {"status": "failed", "message": "Redis ping returned False"} except Exception as e: - verbose_logger.error(f"Redis connection test failed: {e}") + verbose_logger.error("Redis connection test failed: %s", e) return { "status": "failed", "message": f"Redis connection failed: {e}", @@ -1426,7 +1427,7 @@ class RedisCache(BaseCache): # Execute the pipeline and return results results = await pipe.execute() # only return float values - verbose_logger.debug(f"Increment ASYNC Redis Cache PIPELINE: results: {results}") + verbose_logger.debug("Increment ASYNC Redis Cache PIPELINE: results: %s", results) return [r for r in results if isinstance(r, float)] @_redis_circuit_breaker_guard @@ -1513,7 +1514,7 @@ class RedisCache(BaseCache): return None return ttl except Exception as e: - verbose_logger.debug(f"Redis TTL Error: {e}") + verbose_logger.debug("Redis TTL Error: %s", e) _record_swallowed_redis_failure(self._circuit_breaker, e) return None @@ -1565,7 +1566,7 @@ class RedisCache(BaseCache): call_type=f"async_rpush <- {_get_call_stack_info()}", ) ) - verbose_logger.error(f"LiteLLM Redis Cache RPUSH: - Got exception from REDIS : {e}") + verbose_logger.error("LiteLLM Redis Cache RPUSH: - Got exception from REDIS : %s", e) raise e async def _pipeline_rpush_helper( @@ -1711,7 +1712,7 @@ class RedisCache(BaseCache): call_type=f"async_lpop <- {_get_call_stack_info()}", ) ) - verbose_logger.error(f"LiteLLM Redis Cache LPOP: - Got exception from REDIS : {e}") + verbose_logger.error("LiteLLM Redis Cache LPOP: - Got exception from REDIS : %s", e) raise e async def _pipeline_lpop_helper( diff --git a/litellm/caching/redis_cluster_cache.py b/litellm/caching/redis_cluster_cache.py index 127a5c3bd29..926712e38ec 100644 --- a/litellm/caching/redis_cluster_cache.py +++ b/litellm/caching/redis_cluster_cache.py @@ -100,7 +100,7 @@ class RedisClusterCache(RedisCache): except Exception as e: from litellm._logging import verbose_logger - verbose_logger.error(f"Redis Cluster connection test failed: {e}") + verbose_logger.error("Redis Cluster connection test failed: %s", e) return { "status": "failed", "message": f"Redis Cluster connection failed: {e}", diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index f55274d446d..4fe42d1908e 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -138,7 +138,7 @@ class RedisSemanticCache(BaseCache): cache_vectorizer=cache_vectorizer, ) except Exception as e: - verbose_logger.error(f"Redis semantic-cache index build failed: {e}") + verbose_logger.error("Redis semantic-cache index build failed: %s", e) raise @classmethod diff --git a/litellm/caching/s3_cache.py b/litellm/caching/s3_cache.py index 5e185de7526..baad0e29c5e 100644 --- a/litellm/caching/s3_cache.py +++ b/litellm/caching/s3_cache.py @@ -104,12 +104,12 @@ class S3Cache(BaseCache): Compatible with Python 3.8+. """ try: - verbose_logger.debug(f"Set ASYNC S3 Cache: Key={key}. Value={value}") + verbose_logger.debug("Set ASYNC S3 Cache: Key=%s. Value=%s", key, value) loop = asyncio.get_event_loop() func = partial(self.set_cache, key, value, **kwargs) await loop.run_in_executor(None, func) except Exception as e: - verbose_logger.error(f"S3 Caching: async_set_cache() - Got exception from S3: {e}") + verbose_logger.error("S3 Caching: async_set_cache() - Got exception from S3: %s", e) def get_cache(self, key, **kwargs): import botocore @@ -138,17 +138,20 @@ class S3Cache(BaseCache): if not isinstance(cached_response, dict): cached_response = dict(cached_response) verbose_logger.debug( - f"Got S3 Cache: key: {key}, cached_response {cached_response}. Type Response {type(cached_response)}" + "Got S3 Cache: key: %s, cached_response %s. Type Response %s", + key, + cached_response, + type(cached_response), ) return cached_response except botocore.exceptions.ClientError as e: # type: ignore if e.response["Error"]["Code"] == "NoSuchKey": - verbose_logger.debug(f"S3 Cache: The specified key '{key}' does not exist in the S3 bucket.") + verbose_logger.debug("S3 Cache: The specified key '%s' does not exist in the S3 bucket.", key) return None except Exception as e: - verbose_logger.error(f"S3 Caching: get_cache() - Got exception from S3: {e}") + verbose_logger.error("S3 Caching: get_cache() - Got exception from S3: %s", e) async def async_get_cache(self, key, **kwargs): """ @@ -156,13 +159,13 @@ class S3Cache(BaseCache): Compatible with Python 3.8+. """ try: - verbose_logger.debug(f"Get ASYNC S3 Cache: key: {key}") + verbose_logger.debug("Get ASYNC S3 Cache: key: %s", key) loop = asyncio.get_event_loop() func = partial(self.get_cache, key, **kwargs) result = await loop.run_in_executor(None, func) return result except Exception as e: - verbose_logger.error(f"S3 Caching: async_get_cache() - Got exception from S3: {e}") + verbose_logger.error("S3 Caching: async_get_cache() - Got exception from S3: %s", e) return None def flush_cache(self): diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 3825854852d..16c90d5a20a 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -408,7 +408,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): self._map_optional_params_to_responses_api_request(optional_params, responses_api_request) stream = optional_params.get("stream") or litellm_params.get("stream", False) - verbose_logger.debug(f"Chat provider: Stream parameter: {stream}") + verbose_logger.debug("Chat provider: Stream parameter: %s", stream) # Ensure stream is properly set in the request if stream: @@ -418,7 +418,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): previous_response_id = optional_params.get("previous_response_id") if previous_response_id: # Use the existing session handler for responses API - verbose_logger.debug(f"Chat provider: Warning ignoring previous response ID: {previous_response_id}") + verbose_logger.debug("Chat provider: Warning ignoring previous response ID: %s", previous_response_id) # Convert back to responses API format for the actual request @@ -438,7 +438,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): "client": client, } - verbose_logger.debug(f"Chat provider: Final request model={api_model}, input_items={len(input_items)}") + verbose_logger.debug("Chat provider: Final request model=%s, input_items=%s", api_model, len(input_items)) self._merge_responses_api_request_into_request_data(request_data, responses_api_request, instructions) @@ -776,29 +776,29 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): """Convert chat completion content to responses API format""" from litellm.types.llms.openai import ChatCompletionImageObject - verbose_logger.debug(f"Chat provider: Converting content to responses format - input type: {type(content)}") + verbose_logger.debug("Chat provider: Converting content to responses format - input type: %s", type(content)) if content is None: return [self._convert_content_str_to_input_text("", role)] elif isinstance(content, str): result = [self._convert_content_str_to_input_text(content, role)] - verbose_logger.debug(f"Chat provider: String content -> {result}") + verbose_logger.debug("Chat provider: String content -> %s", result) return result elif isinstance(content, list): result = [] for i, item in enumerate(content): - verbose_logger.debug(f"Chat provider: Processing content item {i}: {type(item)} = {item}") + verbose_logger.debug("Chat provider: Processing content item %s: %s = %s", i, type(item), item) if isinstance(item, str): converted = self._convert_content_str_to_input_text(item, role) result.append(converted) - verbose_logger.debug(f"Chat provider: -> {converted}") + verbose_logger.debug("Chat provider: -> %s", converted) elif isinstance(item, dict): # Handle multimodal content original_type = item.get("type") if original_type == "text": converted = self._convert_content_str_to_input_text(item.get("text", ""), role) result.append(converted) - verbose_logger.debug(f"Chat provider: text -> {converted}") + verbose_logger.debug("Chat provider: text -> %s", converted) elif original_type == "image_url": # Map to responses API image format converted = cast( @@ -808,14 +808,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ), ) result.append(converted) - verbose_logger.debug(f"Chat provider: image_url -> {converted}") + verbose_logger.debug("Chat provider: image_url -> %s", converted) else: # Try to map other types to responses API format item_type = original_type or "input_text" if item_type == "image": converted = {"type": "input_image", **item} result.append(converted) - verbose_logger.debug(f"Chat provider: image -> {converted}") + verbose_logger.debug("Chat provider: image -> %s", converted) elif item_type == "file": # Map Chat Completion file to Responses API input_file # {"type": "file", "file": {"file_data": "...", "filename": "..."}} @@ -827,7 +827,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if key in file_data: converted[key] = file_data[key] result.append(converted) - verbose_logger.debug(f"Chat provider: file -> {converted}") + verbose_logger.debug("Chat provider: file -> %s", converted) elif item_type in [ "input_text", "input_image", @@ -839,17 +839,17 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ]: # Already in responses API format result.append(item) - verbose_logger.debug(f"Chat provider: passthrough -> {item}") + verbose_logger.debug("Chat provider: passthrough -> %s", item) else: # Default to input_text for unknown types converted = self._convert_content_str_to_input_text(str(item.get("text", item)), role) result.append(converted) - verbose_logger.debug(f"Chat provider: unknown({original_type}) -> {converted}") - verbose_logger.debug(f"Chat provider: Final converted content: {result}") + verbose_logger.debug("Chat provider: unknown(%s) -> %s", original_type, converted) + verbose_logger.debug("Chat provider: Final converted content: %s", result) return result else: result = [self._convert_content_str_to_input_text(str(content), role)] - verbose_logger.debug(f"Chat provider: Other content type -> {result}") + verbose_logger.debug("Chat provider: Other content type -> %s", result) return result def _convert_tools_to_responses_format(self, tools: list[dict[str, Any]]) -> list["ALL_RESPONSES_API_TOOL_PARAMS"]: @@ -1032,13 +1032,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): annotation_dict = annotation else: # Skip unsupported annotation types - verbose_logger.debug(f"Skipping unsupported annotation type: {type(annotation)}") + verbose_logger.debug("Skipping unsupported annotation type: %s", type(annotation)) continue result.append(annotation_dict) # type: ignore except Exception as e: # Skip malformed annotations - verbose_logger.debug(f"Skipping malformed annotation: {annotation}, error: {e}") + verbose_logger.debug("Skipping malformed annotation: %s, error: %s", annotation, e) continue return result if result else None @@ -1122,11 +1122,11 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): ): return ModelResponseStream(**parsed_chunk) - verbose_logger.debug(f"Chat provider: Processing event type: {event_type}") + verbose_logger.debug("Chat provider: Processing event type: %s", event_type) if event_type == "response.created": # Initial response creation event - verbose_logger.debug(f"Chat provider: response.created -> {parsed_chunk}") + verbose_logger.debug("Chat provider: response.created -> %s", parsed_chunk) return ModelResponseStream( choices=[ StreamingChoices( @@ -1345,7 +1345,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): else: pass # For any unhandled event types, create a minimal valid chunk or skip - verbose_logger.debug(f"Chat provider: Unhandled event type '{event_type}', creating empty chunk") + verbose_logger.debug("Chat provider: Unhandled event type '%s', creating empty chunk", event_type) # Return a minimal valid chunk for unknown events return ModelResponseStream( @@ -1368,7 +1368,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): Returns: ModelResponseStream: OpenAI-formatted streaming chunk """ - verbose_logger.debug(f"Chat provider: transform_streaming_response called with chunk: {chunk}") + verbose_logger.debug("Chat provider: transform_streaming_response called with chunk: %s", chunk) return self._with_stream_scoped_id( OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(chunk) ) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 25d448e48e9..bd03feb03c2 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -273,7 +273,7 @@ def _get_additional_costs( completion_tokens=completion_tokens, ) except Exception as e: - verbose_logger.debug(f"Error calculating additional costs: {e}") + verbose_logger.debug("Error calculating additional costs: %s", e) return None @@ -715,7 +715,7 @@ def _get_provider_for_cost_calc( _, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model) except Exception as e: verbose_logger.debug( - f"litellm.cost_calculator.py::_get_provider_for_cost_calc() - Error inferring custom_llm_provider - {e}" + "litellm.cost_calculator.py::_get_provider_for_cost_calc() - Error inferring custom_llm_provider - %s", e ) return None @@ -896,7 +896,7 @@ def _get_usage_object( elif isinstance(usage_obj, BaseModel): return Usage(**usage_obj.model_dump()) else: - verbose_logger.debug(f"Unknown usage object type: {type(usage_obj)}, usage_obj: {usage_obj}") + verbose_logger.debug("Unknown usage object type: %s, usage_obj: %s", type(usage_obj), usage_obj) return None @@ -994,16 +994,17 @@ def _apply_cost_margin( if custom_llm_provider and custom_llm_provider in litellm.cost_margin_config: margin_config = litellm.cost_margin_config[custom_llm_provider] if verbose_logger.isEnabledFor(logging.DEBUG): - verbose_logger.debug(f"Found provider-specific margin config for {custom_llm_provider}: {margin_config}") + verbose_logger.debug("Found provider-specific margin config for %s: %s", custom_llm_provider, margin_config) elif "global" in litellm.cost_margin_config: margin_config = litellm.cost_margin_config["global"] if verbose_logger.isEnabledFor(logging.DEBUG): - verbose_logger.debug(f"Using global margin config: {margin_config}") + verbose_logger.debug("Using global margin config: %s", margin_config) else: if verbose_logger.isEnabledFor(logging.DEBUG): verbose_logger.debug( - f"No margin config found. Provider: {custom_llm_provider}, " - f"Available configs: {list(litellm.cost_margin_config.keys())}" + "No margin config found. Provider: %s, Available configs: %s", + custom_llm_provider, + list(litellm.cost_margin_config.keys()), ) if margin_config is not None: @@ -1098,7 +1099,7 @@ def _store_cost_breakdown_in_logging_obj( ) except Exception as breakdown_error: - verbose_logger.debug(f"Error storing cost breakdown: {breakdown_error}") + verbose_logger.debug("Error storing cost breakdown: %s", breakdown_error) # Don't fail the main cost calculation if breakdown storage fails @@ -1225,7 +1226,7 @@ def completion_cost( for idx, model in enumerate(potential_model_names): try: if verbose_logger.isEnabledFor(logging.DEBUG): - verbose_logger.debug(f"selected model name for cost calculation: {model}") + verbose_logger.debug("selected model name for cost calculation: %s", model) if completion_response is not None and ( isinstance(completion_response, BaseModel) or isinstance(completion_response, dict) @@ -1321,7 +1322,8 @@ def completion_cost( ) # strip the llm provider from the model name -> for image gen cost calculation except Exception as e: verbose_logger.debug( - f"litellm.cost_calculator.py::completion_cost() - Error inferring custom_llm_provider - {e}" + "litellm.cost_calculator.py::completion_cost() - Error inferring custom_llm_provider - %s", + e, ) if CostCalculatorUtils._call_type_has_image_response(call_type) and isinstance( completion_response, ImageResponse @@ -1672,7 +1674,7 @@ def completion_cost( return _final_cost except Exception as e: verbose_logger.debug( - f"litellm.cost_calculator.py::completion_cost() - Error calculating cost for model={model} - {e}" + "litellm.cost_calculator.py::completion_cost() - Error calculating cost for model=%s - %s", model, e ) if idx == len(potential_model_names) - 1: raise e @@ -1888,7 +1890,7 @@ def vector_store_search_cost( ) if config is None: - verbose_logger.debug(f"Vector store search is not supported for {custom_llm_provider}") + verbose_logger.debug("Vector store search is not supported for %s", custom_llm_provider) return 0.0, 0.0 return config.calculate_vector_store_cost( @@ -1976,7 +1978,7 @@ def default_image_cost_calculator( # gpt-image-1 models use low, medium, high quality. If user did not specify quality, use medium fot gpt-image-1 model family model_name_with_v2_quality = f"{ImageGenerationRequestQuality.HIGH.value}/{base_model_name}" - verbose_logger.debug(f"Looking up cost for models: {model_name_with_quality}, {base_model_name}") + verbose_logger.debug("Looking up cost for models: %s, %s", model_name_with_quality, base_model_name) model_without_provider = f"{size_str}/{model.split('/')[-1]}" model_with_quality_without_provider = f"{quality}/{model_without_provider}" if quality else model_without_provider @@ -2046,7 +2048,7 @@ def default_video_cost_calculator( model_name_without_custom_llm_provider = model.replace(f"{custom_llm_provider}/", "") base_model_name = f"{custom_llm_provider}/{model_name_without_custom_llm_provider}" - verbose_logger.debug(f"Looking up cost for video model: {base_model_name}") + verbose_logger.debug("Looking up cost for video model: %s", base_model_name) model_without_provider = model.split("/")[-1] @@ -2082,7 +2084,8 @@ def default_video_cost_calculator( # If no cost information found, return 0 verbose_logger.info( - f"No cost information found for video model {model}. Please add pricing to model_prices_and_context_window.json" + "No cost information found for video model %s. Please add pricing to model_prices_and_context_window.json", + model, ) return 0.0 diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index 8815c38192b..c9d73363242 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -364,7 +364,7 @@ class MCPClient: try: await session_ctx.__aexit__(None, None, None) except BaseException as e: - verbose_logger.debug(f"Error during session context exit: {e}") + verbose_logger.debug("Error during session context exit: %s", e) except BaseException as e: in_flight_error = e raise @@ -372,7 +372,7 @@ class MCPClient: try: await transport_ctx.__aexit__(None, None, None) except BaseException as exit_error: - verbose_logger.debug(f"Error during transport context exit: {exit_error}") + verbose_logger.debug("Error during transport context exit: %s", exit_error) root_cause = _first_non_cancelled_cause(exit_error) if root_cause is not None and isinstance(in_flight_error, asyncio.CancelledError): raise root_cause from in_flight_error @@ -402,7 +402,7 @@ class MCPClient: try: await http_client.aclose() except BaseException as e: - verbose_logger.debug(f"Error during http_client cleanup: {e}") + verbose_logger.debug("Error during http_client cleanup: %s", e) def update_auth_value(self, mcp_auth_value: str | dict[str, str]): """ @@ -464,7 +464,7 @@ class MCPClient: """Create an httpx.AsyncClient with LiteLLM's SSL configuration.""" # Get unified SSL configuration using the same logic as http_handler.py ssl_config = get_ssl_configuration(self.ssl_verify) - verbose_logger.debug(f"MCP client using SSL configuration: {type(ssl_config).__name__}") + verbose_logger.debug("MCP client using SSL configuration: %s", type(ssl_config).__name__) # The MCP SDK's sse_client and streamable_http_client call this factory without # passing auth=, so the fallback is used: a v2-resolved auth if present, else the # SigV4 aws_auth. Both are None for the common case — no behavior change. @@ -490,7 +490,7 @@ class MCPClient: MCP client (triggering the upstream OAuth flow) rather than masking them as "connected, no tools". """ - verbose_logger.debug(f"MCP client listing tools from {self.server_url or 'stdio'}") + verbose_logger.debug("MCP client listing tools from %s", self.server_url or "stdio") async def _list_tools_operation(session: ClientSession): return await session.list_tools() @@ -499,7 +499,9 @@ class MCPClient: result = await self.run_with_session(_list_tools_operation, quiet_on_error=raise_on_error) tool_count = len(result.tools) tool_names = [tool.name for tool in result.tools] - verbose_logger.info(f"MCP client listed {tool_count} tools from {self.server_url or 'stdio'}: {tool_names}") + verbose_logger.info( + "MCP client listed %s tools from %s: %s", tool_count, self.server_url or "stdio", tool_names + ) return result.tools except asyncio.CancelledError: verbose_logger.warning("MCP client list_tools was cancelled") @@ -555,7 +557,7 @@ class MCPClient: an upstream 401 so it can re-mint the exchanged token and retry once; every other caller keeps the default and gets graceful ``isError`` degradation. """ - verbose_logger.info(f"MCP client calling tool '{call_tool_request_params.name}'") + verbose_logger.info("MCP client calling tool '%s'", call_tool_request_params.name) async def on_progress(progress: float, total: float | None, message: str | None): percentage = (progress / total * 100) if total else 0 @@ -568,7 +570,7 @@ class MCPClient: try: await host_progress_callback(progress, total) except Exception as e: - verbose_logger.warning(f"Failed to forward to Host: {e}") + verbose_logger.warning("Failed to forward to Host: %s", e) async def _call_tool_operation(session: ClientSession): verbose_logger.debug("MCP client sending tool call to session") @@ -580,16 +582,16 @@ class MCPClient: try: tool_result = await self.run_with_session(_call_tool_operation, quiet_on_error=raise_on_error) - verbose_logger.info(f"MCP client tool call '{call_tool_request_params.name}' completed successfully") + verbose_logger.info("MCP client tool call '%s' completed successfully", call_tool_request_params.name) return tool_result except asyncio.CancelledError: - verbose_logger.warning(f"MCP client tool call timed out after {self.timeout}s for {self.server_url}") + verbose_logger.warning("MCP client tool call timed out after %ss for %s", self.timeout, self.server_url) raise except Exception as e: import traceback error_trace = traceback.format_exc() - verbose_logger.debug(f"MCP client tool call traceback:\n{error_trace}") + verbose_logger.debug("MCP client tool call traceback:\n%s", error_trace) # Log detailed error information error_type = type(e).__name__ # When the caller opted into raise_on_error it owns the exception and logs it at the @@ -619,7 +621,7 @@ class MCPClient: async def list_prompts(self) -> list[Prompt]: """List available prompts from the server.""" - verbose_logger.debug(f"MCP client listing tools from {self.server_url or 'stdio'}") + verbose_logger.debug("MCP client listing tools from %s", self.server_url or "stdio") async def _list_prompts_operation(session: ClientSession): return await session.list_prompts() @@ -629,7 +631,7 @@ class MCPClient: prompt_count = len(result.prompts) prompt_names = [prompt.name for prompt in result.prompts] verbose_logger.info( - f"MCP client listed {prompt_count} tools from {self.server_url or 'stdio'}: {prompt_names}" + "MCP client listed %s tools from %s: %s", prompt_count, self.server_url or "stdio", prompt_names ) return result.prompts except asyncio.CancelledError: @@ -638,11 +640,11 @@ class MCPClient: except Exception as e: error_type = type(e).__name__ verbose_logger.error( - f"MCP client list_prompts failed - " - f"Error Type: {error_type}, " - f"Error: {e}, " - f"Server: {self.server_url or 'stdio'}, " - f"Transport: {self.transport_type}" + "MCP client list_prompts failed - Error Type: %s, Error: %s, Server: %s, Transport: %s", + error_type, + e, + self.server_url or "stdio", + self.transport_type, ) # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: @@ -655,7 +657,7 @@ class MCPClient: async def get_prompt(self, get_prompt_request_params: GetPromptRequestParams) -> GetPromptResult: """Fetch a prompt definition from the MCP server.""" - verbose_logger.info(f"MCP client fetching prompt '{get_prompt_request_params.name}'") + verbose_logger.info("MCP client fetching prompt '%s'", get_prompt_request_params.name) async def _get_prompt_operation(session: ClientSession): verbose_logger.debug("MCP client sending get_prompt request to session") @@ -666,7 +668,7 @@ class MCPClient: try: get_prompt_result = await self.run_with_session(_get_prompt_operation) - verbose_logger.info(f"MCP client get_prompt '{get_prompt_request_params.name}' completed successfully") + verbose_logger.info("MCP client get_prompt '%s' completed successfully", get_prompt_request_params.name) return get_prompt_result except asyncio.CancelledError: verbose_logger.warning("MCP client get_prompt was cancelled") @@ -675,16 +677,16 @@ class MCPClient: import traceback error_trace = traceback.format_exc() - verbose_logger.debug(f"MCP client get_prompt traceback:\n{error_trace}") + verbose_logger.debug("MCP client get_prompt traceback:\n%s", error_trace) # Log detailed error information error_type = type(e).__name__ verbose_logger.error( - f"MCP client get_prompt failed - " - f"Error Type: {error_type}, " - f"Error: {e}, " - f"Prompt: {get_prompt_request_params.name}, " - f"Server: {self.server_url or 'stdio'}, " - f"Transport: {self.transport_type}" + "MCP client get_prompt failed - Error Type: %s, Error: %s, Prompt: %s, Server: %s, Transport: %s", + error_type, + e, + get_prompt_request_params.name, + self.server_url or "stdio", + self.transport_type, ) # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: @@ -696,7 +698,7 @@ class MCPClient: async def list_resources(self) -> list[Resource]: """List available resources from the server.""" - verbose_logger.debug(f"MCP client listing resources from {self.server_url or 'stdio'}") + verbose_logger.debug("MCP client listing resources from %s", self.server_url or "stdio") async def _list_resources_operation(session: ClientSession): return await session.list_resources() @@ -706,7 +708,7 @@ class MCPClient: resource_count = len(result.resources) resource_names = [resource.name for resource in result.resources] verbose_logger.info( - f"MCP client listed {resource_count} resources from {self.server_url or 'stdio'}: {resource_names}" + "MCP client listed %s resources from %s: %s", resource_count, self.server_url or "stdio", resource_names ) return result.resources except asyncio.CancelledError: @@ -715,11 +717,11 @@ class MCPClient: except Exception as e: error_type = type(e).__name__ verbose_logger.error( - f"MCP client list_resources failed - " - f"Error Type: {error_type}, " - f"Error: {e}, " - f"Server: {self.server_url or 'stdio'}, " - f"Transport: {self.transport_type}" + "MCP client list_resources failed - Error Type: %s, Error: %s, Server: %s, Transport: %s", + error_type, + e, + self.server_url or "stdio", + self.transport_type, ) # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: @@ -732,7 +734,7 @@ class MCPClient: async def list_resource_templates(self) -> list[ResourceTemplate]: """List available resource templates from the server.""" - verbose_logger.debug(f"MCP client listing resource templates from {self.server_url or 'stdio'}") + verbose_logger.debug("MCP client listing resource templates from %s", self.server_url or "stdio") async def _list_resource_templates_operation(session: ClientSession): return await session.list_resource_templates() @@ -742,7 +744,10 @@ class MCPClient: resource_template_count = len(result.resourceTemplates) resource_template_names = [resourceTemplate.name for resourceTemplate in result.resourceTemplates] verbose_logger.info( - f"MCP client listed {resource_template_count} resource templates from {self.server_url or 'stdio'}: {resource_template_names}" + "MCP client listed %s resource templates from %s: %s", + resource_template_count, + self.server_url or "stdio", + resource_template_names, ) return result.resourceTemplates except asyncio.CancelledError: @@ -751,11 +756,11 @@ class MCPClient: except Exception as e: error_type = type(e).__name__ verbose_logger.error( - f"MCP client list_resource_templates failed - " - f"Error Type: {error_type}, " - f"Error: {e}, " - f"Server: {self.server_url or 'stdio'}, " - f"Transport: {self.transport_type}" + "MCP client list_resource_templates failed - Error Type: %s, Error: %s, Server: %s, Transport: %s", + error_type, + e, + self.server_url or "stdio", + self.transport_type, ) # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: @@ -768,7 +773,7 @@ class MCPClient: async def read_resource(self, url: AnyUrl) -> ReadResourceResult: """Fetch resource contents from the MCP server.""" - verbose_logger.info(f"MCP client fetching resource '{url}'") + verbose_logger.info("MCP client fetching resource '%s'", url) async def _read_resource_operation(session: ClientSession): verbose_logger.debug("MCP client sending read_resource request to session") @@ -776,7 +781,7 @@ class MCPClient: try: read_resource_result = await self.run_with_session(_read_resource_operation) - verbose_logger.info(f"MCP client read_resource '{url}' completed successfully") + verbose_logger.info("MCP client read_resource '%s' completed successfully", url) return read_resource_result except asyncio.CancelledError: verbose_logger.warning("MCP client read_resource was cancelled") @@ -785,16 +790,16 @@ class MCPClient: import traceback error_trace = traceback.format_exc() - verbose_logger.debug(f"MCP client read_resource traceback:\n{error_trace}") + verbose_logger.debug("MCP client read_resource traceback:\n%s", error_trace) # Log detailed error information error_type = type(e).__name__ verbose_logger.error( - f"MCP client read_resource failed - " - f"Error Type: {error_type}, " - f"Error: {e}, " - f"Url: {url}, " - f"Server: {self.server_url or 'stdio'}, " - f"Transport: {self.transport_type}" + "MCP client read_resource failed - Error Type: %s, Error: %s, Url: %s, Server: %s, Transport: %s", + error_type, + e, + url, + self.server_url or "stdio", + self.transport_type, ) # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: diff --git a/litellm/google_genai/adapters/transformation.py b/litellm/google_genai/adapters/transformation.py index 7c2800db07b..b13fb71690b 100644 --- a/litellm/google_genai/adapters/transformation.py +++ b/litellm/google_genai/adapters/transformation.py @@ -104,9 +104,10 @@ class GoogleGenAIStreamWrapper(AdapterCompletionStreamWrapper): except json.JSONDecodeError: # This can happen if the stream is abruptly cut off mid-argument string. verbose_logger.warning( - f"Could not parse tool call arguments at end of stream for index {tool_call_index}. " - f"Name: {tool_call_data['name']}. " - f"Partial args: {tool_call_data['arguments']}" + "Could not parse tool call arguments at end of stream for index %s. Name: %s. Partial args: %s", + tool_call_index, + tool_call_data["name"], + tool_call_data["arguments"], ) if parts: final_chunk = { @@ -662,7 +663,7 @@ class GoogleGenAIAdapter: # Optimization: Skip chunks that have no new data if not function_name and not args_chunk: - verbose_logger.debug(f"Skipping empty tool call chunk for index: {tool_call_index}") + verbose_logger.debug("Skipping empty tool call chunk for index: %s", tool_call_index) continue if function_name: diff --git a/litellm/integrations/SlackAlerting/batching_handler.py b/litellm/integrations/SlackAlerting/batching_handler.py index da905b606a5..12b1f772616 100644 --- a/litellm/integrations/SlackAlerting/batching_handler.py +++ b/litellm/integrations/SlackAlerting/batching_handler.py @@ -68,8 +68,8 @@ async def send_to_webhook(slackAlertingInstance: SlackAlertingType, item, count) data=json.dumps(payload), ) if response.status_code != 200: - verbose_proxy_logger.debug(f"Error sending slack alert to url={item['url']}. Error={response.text}") + verbose_proxy_logger.debug("Error sending slack alert to url=%s. Error=%s", item["url"], response.text) except Exception as e: - verbose_proxy_logger.debug(f"Error sending slack alert: {e}") + verbose_proxy_logger.debug("Error sending slack alert: %s", e) finally: _print_alerting_payload_warning(payload, slackAlertingInstance=slackAlertingInstance) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 114924e7359..0d842a5889a 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -1467,7 +1467,7 @@ Model Info: try: await self._flush_digest_buckets() except Exception as e: - verbose_proxy_logger.debug(f"Error flushing digest buckets: {e}") + verbose_proxy_logger.debug("Error flushing digest buckets: %s", e) await self.flush_queue() async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -1502,7 +1502,7 @@ Model Info: ) except Exception as e: verbose_proxy_logger.error( - f"[Non-Blocking Error] Slack Alerting: Got error in logging LLM deployment latency: {e}" + "[Non-Blocking Error] Slack Alerting: Got error in logging LLM deployment latency: %s", e ) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): @@ -1522,7 +1522,7 @@ Model Info: ) ) except Exception as e: - verbose_logger.debug(f"Exception raises -{e}") + verbose_logger.debug("Exception raises -%s", e) if isinstance(kwargs.get("exception", ""), APIError): if "outage_alerts" in self.alert_types: @@ -1662,9 +1662,9 @@ Model Info: ) except ValueError as ve: - verbose_proxy_logger.error(f"Invalid time range format: {ve}") + verbose_proxy_logger.error("Invalid time range format: %s", ve) except Exception as e: - verbose_proxy_logger.error(f"Error sending spend report: {e}") + verbose_proxy_logger.error("Error sending spend report: %s", e) async def send_monthly_spend_report(self): """ """ diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 751c8c01aae..f52b6bd8415 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -143,8 +143,8 @@ class AnthropicCacheControlHook(CustomPromptManagement): if limit_reached: verbose_logger.warning( - f"AnthropicCacheControlHook: Reached the Anthropic limit of " - f"{MAX_CACHE_CONTROL_BLOCKS} cache_control blocks. Skipping further injection." + "AnthropicCacheControlHook: Reached the Anthropic limit of %s cache_control blocks. Skipping further injection.", + MAX_CACHE_CONTROL_BLOCKS, ) return messages @@ -174,8 +174,10 @@ class AnthropicCacheControlHook(CustomPromptManagement): return [targetted_index] verbose_logger.warning( - f"AnthropicCacheControlHook: Provided index {original_index} is out of bounds for message list of length {len(messages)}. " - f"Targeted index was {targetted_index}. Skipping cache control injection for this point." + "AnthropicCacheControlHook: Provided index %s is out of bounds for message list of length %s. Targeted index was %s. Skipping cache control injection for this point.", + original_index, + len(messages), + targetted_index, ) return [] diff --git a/litellm/integrations/argilla.py b/litellm/integrations/argilla.py index d41291f9f98..b1cda6a5593 100644 --- a/litellm/integrations/argilla.py +++ b/litellm/integrations/argilla.py @@ -185,9 +185,9 @@ class ArgillaLogger(CustomBatchLogger): ) if response.status_code >= 300: - verbose_logger.error(f"Argilla Error: {response.status_code} - {response.text}") + verbose_logger.error("Argilla Error: %s - %s", response.status_code, response.text) else: - verbose_logger.debug(f"Batch of {len(self.log_queue)} runs successfully created") + verbose_logger.debug("Batch of %s runs successfully created", len(self.log_queue)) self.log_queue.clear() except Exception: @@ -204,7 +204,7 @@ class ArgillaLogger(CustomBatchLogger): random_sample = random.random() if random_sample > sampling_rate: verbose_logger.info( - f"Skipping Langsmith logging. Sampling rate={sampling_rate}, random_sample={random_sample}" + "Skipping Langsmith logging. Sampling rate=%s, random_sample=%s", sampling_rate, random_sample ) return # Skip logging verbose_logger.debug( @@ -217,7 +217,7 @@ class ArgillaLogger(CustomBatchLogger): return self.log_queue.append(data) - verbose_logger.debug(f"Langsmith, event added to queue. Will flush in {self.flush_interval} seconds...") + verbose_logger.debug("Langsmith, event added to queue. Will flush in %s seconds...", self.flush_interval) if len(self.log_queue) >= self.batch_size: self._send_batch() @@ -231,7 +231,7 @@ class ArgillaLogger(CustomBatchLogger): random_sample = random.random() if random_sample > sampling_rate: verbose_logger.info( - f"Skipping Langsmith logging. Sampling rate={sampling_rate}, random_sample={random_sample}" + "Skipping Langsmith logging. Sampling rate=%s, random_sample=%s", sampling_rate, random_sample ) return # Skip logging verbose_logger.debug( @@ -272,7 +272,7 @@ class ArgillaLogger(CustomBatchLogger): random_sample = random.random() if random_sample > sampling_rate: verbose_logger.info( - f"Skipping Langsmith logging. Sampling rate={sampling_rate}, random_sample={random_sample}" + "Skipping Langsmith logging. Sampling rate=%s, random_sample=%s", sampling_rate, random_sample ) return # Skip logging verbose_logger.info("Langsmith Failure Event Logging!") @@ -325,7 +325,7 @@ class ArgillaLogger(CustomBatchLogger): response.raise_for_status() if response.status_code >= 300: - verbose_logger.error(f"Argilla Error: {response.status_code} - {response.text}") + verbose_logger.error("Argilla Error: %s - %s", response.status_code, response.text) else: verbose_logger.debug("Batch of %s runs successfully created", len(self.log_queue)) except httpx.HTTPStatusError: diff --git a/litellm/integrations/arize/_utils.py b/litellm/integrations/arize/_utils.py index 032f490860d..fe5d235e51b 100644 --- a/litellm/integrations/arize/_utils.py +++ b/litellm/integrations/arize/_utils.py @@ -461,7 +461,7 @@ def set_attributes(span: "Span", kwargs, response_obj, attributes: type[BaseLLMO _set_response_attributes(span=span, response_obj=response_obj_for_attrs) except Exception as e: - verbose_logger.error(f"[Arize/Phoenix] Failed to set OpenInference span attributes: {e}") + verbose_logger.error("[Arize/Phoenix] Failed to set OpenInference span attributes: %s", e) if hasattr(span, "record_exception"): span.record_exception(e) diff --git a/litellm/integrations/arize/arize_phoenix.py b/litellm/integrations/arize/arize_phoenix.py index db698dd6b77..ae8a6994488 100644 --- a/litellm/integrations/arize/arize_phoenix.py +++ b/litellm/integrations/arize/arize_phoenix.py @@ -425,7 +425,7 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore endpoint = "http://localhost:6006/v1/traces" protocol = "otlp_http" verbose_logger.debug( - f"No PHOENIX_COLLECTOR_ENDPOINT found, using default local Phoenix endpoint: {endpoint}" + "No PHOENIX_COLLECTOR_ENDPOINT found, using default local Phoenix endpoint: %s", endpoint ) otlp_auth_headers = None diff --git a/litellm/integrations/arize/arize_phoenix_prompt_manager.py b/litellm/integrations/arize/arize_phoenix_prompt_manager.py index ca74835e167..9985bc20af0 100644 --- a/litellm/integrations/arize/arize_phoenix_prompt_manager.py +++ b/litellm/integrations/arize/arize_phoenix_prompt_manager.py @@ -339,7 +339,7 @@ class ArizePhoenixPromptManager(CustomPromptManagement): # Log error but don't fail the call import litellm - litellm._logging.verbose_proxy_logger.error(f"Error in Arize Phoenix prompt pre_call_hook: {e}") + litellm._logging.verbose_proxy_logger.error("Error in Arize Phoenix prompt pre_call_hook: %s", e) return messages, litellm_params def get_available_prompts(self) -> list[str]: diff --git a/litellm/integrations/azure_sentinel/azure_sentinel.py b/litellm/integrations/azure_sentinel/azure_sentinel.py index e0ed0cd7cf3..29bbac2912a 100644 --- a/litellm/integrations/azure_sentinel/azure_sentinel.py +++ b/litellm/integrations/azure_sentinel/azure_sentinel.py @@ -203,7 +203,7 @@ class AzureSentinelLogger(CustomBatchLogger): await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"Azure Sentinel Layer Error - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Azure Sentinel Layer Error - %s\n%s", e, traceback.format_exc()) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): """ @@ -233,7 +233,7 @@ class AzureSentinelLogger(CustomBatchLogger): await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"Azure Sentinel Layer Error - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Azure Sentinel Layer Error - %s\n%s", e, traceback.format_exc()) async def async_log_audit_log_event(self, audit_log: StandardAuditLogPayload) -> None: """ @@ -256,7 +256,7 @@ class AzureSentinelLogger(CustomBatchLogger): await self.async_send_audit_batch() except Exception as e: - verbose_logger.exception(f"Azure Sentinel Audit Log Layer Error - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Azure Sentinel Audit Log Layer Error - %s\n%s", e, traceback.format_exc()) async def async_send_batch(self): """ @@ -323,7 +323,7 @@ class AzureSentinelLogger(CustomBatchLogger): ) except Exception as e: - verbose_logger.exception(f"Azure Sentinel Error sending batch API - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Azure Sentinel Error sending batch API - %s\n%s", e, traceback.format_exc()) finally: log_queue.clear() diff --git a/litellm/integrations/azure_storage/azure_storage.py b/litellm/integrations/azure_storage/azure_storage.py index d2dd3d37dc7..142a0a56967 100644 --- a/litellm/integrations/azure_storage/azure_storage.py +++ b/litellm/integrations/azure_storage/azure_storage.py @@ -53,7 +53,9 @@ class AzureBlobStorageLogger(CustomBatchLogger): self.log_queue: list[StandardLoggingPayload] = [] super().__init__(**kwargs, flush_lock=self.flush_lock) except Exception as e: - verbose_logger.exception(f"AzureBlobStorageLogger: Got exception on init AzureBlobStorageLogger client {e}") + verbose_logger.exception( + "AzureBlobStorageLogger: Got exception on init AzureBlobStorageLogger client %s", e + ) raise e async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -77,7 +79,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): self.log_queue.append(standard_logging_payload) except Exception as e: - verbose_logger.exception(f"AzureBlobStorageLogger Layer Error - {e}") + verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): """ @@ -99,7 +101,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): self.log_queue.append(standard_logging_payload) except Exception as e: - verbose_logger.exception(f"AzureBlobStorageLogger Layer Error - {e}") + verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e) async def async_send_batch(self): """ @@ -122,7 +124,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): await self.async_upload_payload_to_azure_blob_storage(payload=payload) except Exception as e: - verbose_logger.exception(f"AzureBlobStorageLogger Error sending batch API - {e}") + verbose_logger.exception("AzureBlobStorageLogger Error sending batch API - %s", e) async def async_upload_payload_to_azure_blob_storage(self, payload: StandardLoggingPayload): """ @@ -148,16 +150,16 @@ class AzureBlobStorageLogger(CustomBatchLogger): await self._append_data(async_client, base_url, json_payload) await self._flush_data(async_client, base_url, len(payload_bytes)) - verbose_logger.debug(f"Successfully uploaded log to Azure Blob Storage: {filename}") + verbose_logger.debug("Successfully uploaded log to Azure Blob Storage: %s", filename) except Exception as e: - verbose_logger.exception(f"Error uploading to Azure Blob Storage: {e}") + verbose_logger.exception("Error uploading to Azure Blob Storage: %s", e) raise e async def _create_file(self, client: AsyncHTTPHandler, base_url: str): """Helper method to create the file resource""" try: - verbose_logger.debug(f"Creating file resource at: {base_url}") + verbose_logger.debug("Creating file resource at: %s", base_url) headers = { "x-ms-version": AZURE_STORAGE_MSFT_VERSION, "Content-Length": "0", @@ -167,13 +169,13 @@ class AzureBlobStorageLogger(CustomBatchLogger): response.raise_for_status() verbose_logger.debug("Successfully created file resource") except Exception as e: - verbose_logger.exception(f"Error creating file resource: {e}") + verbose_logger.exception("Error creating file resource: %s", e) raise async def _append_data(self, client: AsyncHTTPHandler, base_url: str, json_payload: str): """Helper method to append data to the file""" try: - verbose_logger.debug(f"Appending data to file: {base_url}") + verbose_logger.debug("Appending data to file: %s", base_url) headers = { "x-ms-version": AZURE_STORAGE_MSFT_VERSION, "Content-Type": "application/json", @@ -187,13 +189,13 @@ class AzureBlobStorageLogger(CustomBatchLogger): response.raise_for_status() verbose_logger.debug("Successfully appended data") except Exception as e: - verbose_logger.exception(f"Error appending data: {e}") + verbose_logger.exception("Error appending data: %s", e) raise async def _flush_data(self, client: AsyncHTTPHandler, base_url: str, position: int): """Helper method to flush the data""" try: - verbose_logger.debug(f"Flushing data at position {position}") + verbose_logger.debug("Flushing data at position %s", position) headers = { "x-ms-version": AZURE_STORAGE_MSFT_VERSION, "Content-Length": "0", @@ -203,7 +205,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): response.raise_for_status() verbose_logger.debug("Successfully flushed data") except Exception as e: - verbose_logger.exception(f"Error flushing data: {e}") + verbose_logger.exception("Error flushing data: %s", e) raise ####### Helper methods to managing Authentication to Azure Storage ####### @@ -227,7 +229,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): ) # Token typically expires in 1 hour self.token_expiry = datetime.now() + timedelta(hours=1) - verbose_logger.debug(f"New token will expire at {self.token_expiry}") + verbose_logger.debug("New token will expire at %s", self.token_expiry) def get_azure_ad_token_from_azure_storage( self, @@ -322,7 +324,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): # check if the directory exists if not await directory_client.exists(): await directory_client.create_directory() - verbose_logger.debug(f"Created directory: {today}") + verbose_logger.debug("Created directory: %s", today) # Create a file client file_name = f"{payload.get('id') or str(uuid.uuid4())}.json" @@ -340,7 +342,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): # Flush the content to finalize the file await file_client.flush_data(position=len(content), offset=0) - verbose_logger.debug(f"Successfully uploaded and wrote to {today}/{file_name}") + verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, file_name) except Exception as e: - verbose_logger.exception(f"Error occurred: {e}") + verbose_logger.exception("Error occurred: %s", e) diff --git a/litellm/integrations/bitbucket/bitbucket_prompt_manager.py b/litellm/integrations/bitbucket/bitbucket_prompt_manager.py index c76466b2f40..3a61c900600 100644 --- a/litellm/integrations/bitbucket/bitbucket_prompt_manager.py +++ b/litellm/integrations/bitbucket/bitbucket_prompt_manager.py @@ -320,7 +320,7 @@ class BitBucketPromptManager(CustomPromptManagement): # Log error but don't fail the call import litellm - litellm._logging.verbose_proxy_logger.error(f"Error in BitBucket prompt pre_call_hook: {e}") + litellm._logging.verbose_proxy_logger.error("Error in BitBucket prompt pre_call_hook: %s", e) return messages, litellm_params def _parse_prompt_to_messages(self, prompt_content: str) -> list[AllMessageValues]: diff --git a/litellm/integrations/braintrust_mock_client.py b/litellm/integrations/braintrust_mock_client.py index e2b732d6e9c..c775a8f3ab8 100644 --- a/litellm/integrations/braintrust_mock_client.py +++ b/litellm/integrations/braintrust_mock_client.py @@ -89,7 +89,7 @@ def _mock_http_handler_post( """Monkey-patched HTTPHandler.post that intercepts Braintrust calls with endpoint-specific responses.""" # Only mock Braintrust API calls if isinstance(url, str) and _is_braintrust_url(url): - verbose_logger.info(f"[BRAINTRUST MOCK] POST to {url}") + verbose_logger.info("[BRAINTRUST MOCK] POST to %s", url) time.sleep(_MOCK_LATENCY_SECONDS) # Return appropriate mock response based on endpoint if "/project" in url: diff --git a/litellm/integrations/cloudzero/cloudzero.py b/litellm/integrations/cloudzero/cloudzero.py index 52b41f74fce..5cab952cfea 100644 --- a/litellm/integrations/cloudzero/cloudzero.py +++ b/litellm/integrations/cloudzero/cloudzero.py @@ -38,7 +38,7 @@ class CloudZeroLogger(CustomLogger): self.connection_id = connection_id or os.getenv("CLOUDZERO_CONNECTION_ID") self.timezone = timezone or os.getenv("CLOUDZERO_TIMEZONE", "UTC") verbose_logger.debug( - f"CloudZero Logger initialized with connection ID: {self.connection_id}, timezone: {self.timezone}" + "CloudZero Logger initialized with connection ID: %s, timezone: %s", self.connection_id, self.timezone ) async def initialize_cloudzero_export_job(self): @@ -130,7 +130,7 @@ class CloudZeroLogger(CustomLogger): verbose_logger.debug("CloudZero Logger: No usage data found to export") return - verbose_logger.debug(f"CloudZero Logger: Processing {len(data)} records") + verbose_logger.debug("CloudZero Logger: Processing %s records", len(data)) # Transform data to CloudZero CBF format transformer = CBFTransformer() @@ -147,13 +147,13 @@ class CloudZeroLogger(CustomLogger): user_timezone=self.timezone, ) - verbose_logger.debug(f"CloudZero Logger: Transmitting {len(cbf_data)} records to CloudZero") + verbose_logger.debug("CloudZero Logger: Transmitting %s records to CloudZero", len(cbf_data)) streamer.send_batched(cbf_data, operation=operation) - verbose_logger.debug(f"CloudZero Logger: Successfully exported {len(cbf_data)} records to CloudZero") + verbose_logger.debug("CloudZero Logger: Successfully exported %s records to CloudZero", len(cbf_data)) except Exception as e: - verbose_logger.error(f"CloudZero Logger: Error exporting usage data: {e}") + verbose_logger.error("CloudZero Logger: Error exporting usage data: %s", e) raise async def dry_run_export_usage_data(self, limit: int | None = 10000): @@ -191,7 +191,7 @@ class CloudZeroLogger(CustomLogger): }, } - verbose_logger.debug(f"CloudZero Dry Run: Processing {len(data)} records...") + verbose_logger.debug("CloudZero Dry Run: Processing %s records...", len(data)) # Convert usage data to dict format for response usage_data_sample = data.head(50).to_dicts() # Return first 50 rows @@ -229,7 +229,7 @@ class CloudZeroLogger(CustomLogger): ) total_tokens = sum(record.get("usage/amount", 0) for record in cbf_data_dict) - verbose_logger.debug(f"CloudZero Logger: Dry run completed for {len(cbf_data)} records") + verbose_logger.debug("CloudZero Logger: Dry run completed for %s records", len(cbf_data)) return { "usage_data": usage_data_sample, @@ -244,8 +244,8 @@ class CloudZeroLogger(CustomLogger): } except Exception as e: - verbose_logger.error(f"CloudZero Logger: Error in dry run export: {e}") - verbose_logger.error(f"CloudZero Dry Run Error: {e}") + verbose_logger.error("CloudZero Logger: Error in dry run export: %s", e) + verbose_logger.error("CloudZero Dry Run Error: %s", e) raise def _display_cbf_data_on_screen(self, cbf_data): diff --git a/litellm/integrations/custom_batch_logger.py b/litellm/integrations/custom_batch_logger.py index 98a8e4ba739..7559bc83cfa 100644 --- a/litellm/integrations/custom_batch_logger.py +++ b/litellm/integrations/custom_batch_logger.py @@ -47,7 +47,7 @@ class CustomBatchLogger(CustomLogger): async def periodic_flush(self): while True: await asyncio.sleep(self.flush_interval) - verbose_logger.debug(f"CustomLogger periodic flush after {self.flush_interval} seconds") + verbose_logger.debug("CustomLogger periodic flush after %s seconds", self.flush_interval) await self.flush_queue() async def flush_queue(self): diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 743c539c36f..d9d65375ea8 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -864,7 +864,7 @@ class CustomGuardrail(CustomLogger): if premium_user is not True: verbose_logger.warning( - f"Trying to use premium guardrail without premium user {CommonProxyErrors.not_premium_user.value}" + "Trying to use premium guardrail without premium user %s", CommonProxyErrors.not_premium_user.value ) return False return True @@ -1028,7 +1028,7 @@ class CustomGuardrail(CustomLogger): else: guardrail_response = "allow" - verbose_logger.debug(f"Guardrail response: {response}") + verbose_logger.debug("Guardrail response: %s", response) self.add_standard_logging_guardrail_information_to_request_data( guardrail_json_response=guardrail_response, diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 9915224ba09..9df0cf6e84d 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -915,19 +915,19 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac for callback_obj in all_callbacks: if hasattr(callback_obj, "increment_callback_logging_failure"): - verbose_logger.debug(f"Incrementing callback failure metric for {callback_name}") + verbose_logger.debug("Incrementing callback failure metric for %s", callback_name) callback_obj.increment_callback_logging_failure(callback_name=callback_name) # type: ignore return verbose_logger.debug( - f"No callback with increment_callback_logging_failure method found for {callback_name}. " - "Ensure 'prometheus' is in your callbacks config." + "No callback with increment_callback_logging_failure method found for %s. Ensure 'prometheus' is in your callbacks config.", + callback_name, ) except Exception as e: from litellm._logging import verbose_logger - verbose_logger.debug(f"Error in handle_callback_failure for {callback_name}: {e}") + verbose_logger.debug("Error in handle_callback_failure for %s: %s", callback_name, e) async def _strip_base64_from_messages( self, @@ -946,7 +946,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac """ raw_messages: Any = payload.get("messages", []) messages: list[Any] = raw_messages if isinstance(raw_messages, list) else [] - verbose_logger.debug(f"[CustomLogger] Stripping base64 from {len(messages)} messages") + verbose_logger.debug("[CustomLogger] Stripping base64 from %s messages", len(messages)) if messages: payload["messages"] = self._process_messages(messages=messages, max_depth=max_depth) @@ -958,7 +958,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac if isinstance(content, list): total_items += len(content) - verbose_logger.debug(f"[CustomLogger] Completed base64 strip; retained {total_items} content items") + verbose_logger.debug("[CustomLogger] Completed base64 strip; retained %s content items", total_items) return payload def _strip_base64_from_messages_sync( @@ -978,7 +978,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac """ raw_messages: Any = payload.get("messages", []) messages: list[Any] = raw_messages if isinstance(raw_messages, list) else [] - verbose_logger.debug(f"[CustomLogger] Stripping base64 from {len(messages)} messages") + verbose_logger.debug("[CustomLogger] Stripping base64 from %s messages", len(messages)) if messages: payload["messages"] = self._process_messages(messages=messages, max_depth=max_depth) @@ -990,7 +990,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac if isinstance(content, list): total_items += len(content) - verbose_logger.debug(f"[CustomLogger] Completed base64 strip; retained {total_items} content items") + verbose_logger.debug("[CustomLogger] Completed base64 strip; retained %s content items", total_items) return payload def _redact_base64( @@ -1001,12 +1001,12 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac ) -> Any: """Recursively redact inline base64 from any nested structure with a max recursion depth limit.""" if depth > max_depth: - verbose_logger.warning(f"[CustomLogger] Max recursion depth {max_depth} reached while redacting base64") + verbose_logger.warning("[CustomLogger] Max recursion depth %s reached while redacting base64", max_depth) return "[MAX_DEPTH_REACHED]" if isinstance(value, str): if _BASE64_INLINE_PATTERN.search(value): - verbose_logger.debug(f"[CustomLogger] Redacted inline base64 string: {value[:40]}...") + verbose_logger.debug("[CustomLogger] Redacted inline base64 string: %s...", value[:40]) return _BASE64_INLINE_PATTERN.sub("[BASE64_REDACTED]", value) return value diff --git a/litellm/integrations/custom_secret_manager.py b/litellm/integrations/custom_secret_manager.py index 8cb7f02b798..e59842d409a 100644 --- a/litellm/integrations/custom_secret_manager.py +++ b/litellm/integrations/custom_secret_manager.py @@ -237,7 +237,7 @@ class CustomSecretManager(BaseSecretManager): Returns: True if the secret manager is healthy, False otherwise """ - verbose_logger.debug(f"Health check not implemented for {self.secret_manager_name}") + verbose_logger.debug("Health check not implemented for %s", self.secret_manager_name) return True def __repr__(self) -> str: diff --git a/litellm/integrations/datadog/datadog.py b/litellm/integrations/datadog/datadog.py index fa14e1fa459..ce6dda96820 100644 --- a/litellm/integrations/datadog/datadog.py +++ b/litellm/integrations/datadog/datadog.py @@ -171,7 +171,7 @@ class DataDogLogger( batch_size=_resolve_dd_batch_size(), ) except Exception as e: - verbose_logger.exception(f"Datadog: Got exception on init Datadog client {e}") + verbose_logger.exception("Datadog: Got exception on init Datadog client %s", e) raise e def _get_datadog_params(self) -> dict: @@ -210,7 +210,7 @@ class DataDogLogger( self.DD_API_KEY = dd_api_key or ( os.getenv("DD_API_KEY") if allow_env_credentials else None ) # Optional when using agent - verbose_logger.debug(f"Datadog: Using DD Agent at {self.intake_url}") + verbose_logger.debug("Datadog: Using DD Agent at %s", self.intake_url) def _configure_dd_direct_api( self, @@ -257,7 +257,7 @@ class DataDogLogger( await self._log_async_event(kwargs, response_obj, start_time, end_time) except Exception as e: - verbose_logger.exception(f"Datadog Layer Error - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Datadog Layer Error - %s\n%s", e, traceback.format_exc()) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): try: @@ -265,7 +265,7 @@ class DataDogLogger( await self._log_async_event(kwargs, response_obj, start_time, end_time) except Exception as e: - verbose_logger.exception(f"Datadog Layer Error - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Datadog Layer Error - %s\n%s", e, traceback.format_exc()) async def async_post_call_failure_hook( self, @@ -340,7 +340,7 @@ class DataDogLogger( if len(self.log_queue) >= self.batch_size: await self.flush_queue() except Exception as e: - verbose_logger.exception(f"Datadog: async_post_call_failure_hook - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Datadog: async_post_call_failure_hook - %s\n%s", e, traceback.format_exc()) return None async def async_send_batch(self): @@ -376,11 +376,11 @@ class DataDogLogger( self.log_queue = undelivered + self.log_queue if self.is_mock_mode: - verbose_logger.debug(f"[DATADOG MOCK] Batch of {len(batch_to_send)} events successfully mocked") + verbose_logger.debug("[DATADOG MOCK] Batch of %s events successfully mocked", len(batch_to_send)) except Exception as e: self.log_queue = batch_to_send + self.log_queue - verbose_logger.exception(f"Datadog Error sending batch API - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Datadog Error sending batch API - %s\n%s", e, traceback.format_exc()) async def _send_with_413_split(self, batch: list) -> list: """ @@ -411,7 +411,7 @@ class DataDogLogger( if isinstance(e, MaskedHTTPStatusError) and e.status_code == 413: response = e.response else: - verbose_logger.exception(f"Datadog Error sending batch API - {e}") + verbose_logger.exception("Datadog Error sending batch API - %s", e) return self._undelivered(chunk, pending) if response.status_code == 413: @@ -515,7 +515,7 @@ class DataDogLogger( ) except Exception as e: - verbose_logger.exception(f"Datadog Layer Error - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Datadog Layer Error - %s\n%s", e, traceback.format_exc()) async def _log_async_event(self, kwargs, response_obj, start_time, end_time): dd_payload = self.create_datadog_logging_payload( @@ -526,7 +526,7 @@ class DataDogLogger( ) self.log_queue.append(dd_payload) - verbose_logger.debug(f"Datadog, event added to queue. Will flush in {self.flush_interval} seconds...") + verbose_logger.debug("Datadog, event added to queue. Will flush in %s seconds...", self.flush_interval) if len(self.log_queue) >= self.batch_size: await self.flush_queue() @@ -653,7 +653,7 @@ class DataDogLogger( self.log_queue.append(_dd_payload) except Exception as e: - verbose_logger.exception(f"Datadog: Logger - Exception in async_service_failure_hook: {e}") + verbose_logger.exception("Datadog: Logger - Exception in async_service_failure_hook: %s", e) async def async_service_success_hook( self, @@ -692,7 +692,7 @@ class DataDogLogger( self.log_queue.append(_dd_payload) except Exception as e: - verbose_logger.exception(f"Datadog: Logger - Exception in async_service_failure_hook: {e}") + verbose_logger.exception("Datadog: Logger - Exception in async_service_failure_hook: %s", e) def _create_v0_logging_payload( self, diff --git a/litellm/integrations/datadog/datadog_cost_management.py b/litellm/integrations/datadog/datadog_cost_management.py index da45f94f02b..21a289877c2 100644 --- a/litellm/integrations/datadog/datadog_cost_management.py +++ b/litellm/integrations/datadog/datadog_cost_management.py @@ -84,7 +84,7 @@ class DatadogCostManagementLogger(CustomBatchLogger): await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"Datadog Cost Management: Error in async_log_success_event: {e}") + verbose_logger.exception("Datadog Cost Management: Error in async_log_success_event: %s", e) async def async_send_batch(self): if not self.log_queue: @@ -104,7 +104,7 @@ class DatadogCostManagementLogger(CustomBatchLogger): await self._upload_to_datadog(aggregated_entries) except Exception as e: self.log_queue = batch_to_send + self.log_queue - verbose_logger.exception(f"Datadog Cost Management: Error in async_send_batch: {e}") + verbose_logger.exception("Datadog Cost Management: Error in async_send_batch: %s", e) def _aggregate_costs(self, logs: list[StandardLoggingPayload]) -> list[DatadogFOCUSCostEntry]: """ @@ -159,7 +159,7 @@ class DatadogCostManagementLogger(CustomBatchLogger): aggregator[key]["BilledCost"] += cost except Exception as e: - verbose_logger.warning(f"Error processing log for cost aggregation: {e}") + verbose_logger.warning("Error processing log for cost aggregation: %s", e) continue return list(aggregator.values()) @@ -254,5 +254,5 @@ class DatadogCostManagementLogger(CustomBatchLogger): response.raise_for_status() verbose_logger.debug( - f"Datadog Cost Management: Uploaded {len(payload)} cost entries. Status: {response.status_code}" + "Datadog Cost Management: Uploaded %s cost entries. Status: %s", len(payload), response.status_code ) diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index 02e1affd361..8d7ed415315 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -89,7 +89,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): kwargs.update(dict_datadog_llm_obs_params) CustomBatchLogger.__init__(self, **kwargs, flush_lock=self.flush_lock) except Exception as e: - verbose_logger.exception(f"DataDogLLMObs: Error initializing - {e}") + verbose_logger.exception("DataDogLLMObs: Error initializing - %s", e) raise e def _configure_dd_agent(self, dd_agent_host: str): @@ -103,7 +103,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): agent_port = os.getenv("LITELLM_DD_LLM_OBS_PORT", "8126") self.DD_SITE = "localhost" # Not used for URL construction in agent mode self.intake_url = f"http://{dd_agent_host}:{agent_port}/api/intake/llm-obs/v1/trace/spans" - verbose_logger.debug(f"DataDogLLMObs: Using DD Agent at {self.intake_url}") + verbose_logger.debug("DataDogLLMObs: Using DD Agent at %s", self.intake_url) def _configure_dd_direct_api(self): """ @@ -137,34 +137,34 @@ class DataDogLLMObsLogger(CustomBatchLogger): async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): try: - verbose_logger.debug(f"DataDogLLMObs: Logging success event for model {kwargs.get('model', 'unknown')}") + verbose_logger.debug("DataDogLLMObs: Logging success event for model %s", kwargs.get("model", "unknown")) payload = self.create_llm_obs_payload(kwargs, start_time, end_time) - verbose_logger.debug(f"DataDogLLMObs: Payload: {payload}") + verbose_logger.debug("DataDogLLMObs: Payload: %s", payload) self.log_queue.append(payload) if len(self.log_queue) >= self.batch_size: await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"DataDogLLMObs: Error logging success event - {e}") + verbose_logger.exception("DataDogLLMObs: Error logging success event - %s", e) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): try: - verbose_logger.debug(f"DataDogLLMObs: Logging failure event for model {kwargs.get('model', 'unknown')}") + verbose_logger.debug("DataDogLLMObs: Logging failure event for model %s", kwargs.get("model", "unknown")) payload = self.create_llm_obs_payload(kwargs, start_time, end_time) - verbose_logger.debug(f"DataDogLLMObs: Payload: {payload}") + verbose_logger.debug("DataDogLLMObs: Payload: %s", payload) self.log_queue.append(payload) if len(self.log_queue) >= self.batch_size: await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"DataDogLLMObs: Error logging failure event - {e}") + verbose_logger.exception("DataDogLLMObs: Error logging failure event - %s", e) async def async_send_batch(self): try: if not self.log_queue: return - verbose_logger.debug(f"DataDogLLMObs: Flushing {len(self.log_queue)} events") + verbose_logger.debug("DataDogLLMObs: Flushing %s events", len(self.log_queue)) if self.is_mock_mode: verbose_logger.debug("[DATADOG MOCK] Mock mode enabled - API calls will be intercepted") @@ -207,14 +207,14 @@ class DataDogLLMObsLogger(CustomBatchLogger): ) if self.is_mock_mode: - verbose_logger.debug(f"[DATADOG MOCK] Batch of {len(self.log_queue)} events successfully mocked") + verbose_logger.debug("[DATADOG MOCK] Batch of %s events successfully mocked", len(self.log_queue)) else: - verbose_logger.debug(f"DataDogLLMObs: Successfully sent batch - status_code: {response.status_code}") + verbose_logger.debug("DataDogLLMObs: Successfully sent batch - status_code: %s", response.status_code) self.log_queue.clear() except httpx.HTTPStatusError as e: - verbose_logger.exception(f"DataDogLLMObs: Error sending batch - {e.response.text}") + verbose_logger.exception("DataDogLLMObs: Error sending batch - %s", e.response.text) except Exception as e: - verbose_logger.exception(f"DataDogLLMObs: Error sending batch - {e}") + verbose_logger.exception("DataDogLLMObs: Error sending batch - %s", e) def create_llm_obs_payload(self, kwargs: dict, start_time: datetime, end_time: datetime) -> LLMObsPayload: standard_logging_payload: StandardLoggingPayload | None = kwargs.get("standard_logging_object") @@ -613,7 +613,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): try: spend_metrics["user_api_key_spend"] = float(user_api_key_spend) except (ValueError, TypeError): - verbose_logger.debug(f"Invalid user_api_key_spend value: {user_api_key_spend}") + verbose_logger.debug("Invalid user_api_key_spend value: %s", user_api_key_spend) # API key budget reset datetime user_api_key_budget_reset_at = metadata.get("user_api_key_budget_reset_at") @@ -640,10 +640,10 @@ class DataDogLLMObsLogger(CustomBatchLogger): spend_metrics["user_api_key_budget_reset_at"] = iso_string # Debug logging to verify the conversion - verbose_logger.debug(f"Converted budget_reset_at to ISO format: {iso_string}") + verbose_logger.debug("Converted budget_reset_at to ISO format: %s", iso_string) except Exception as e: - verbose_logger.debug(f"Error processing budget reset datetime: {e}") - verbose_logger.debug(f"Original value: {user_api_key_budget_reset_at}") + verbose_logger.debug("Error processing budget reset datetime: %s", e) + verbose_logger.debug("Original value: %s", user_api_key_budget_reset_at) return spend_metrics @@ -707,7 +707,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): kv_pairs[f"tool_calls.{idx}.function.arguments"] = json.dumps(function_arguments) except (KeyError, TypeError, ValueError) as e: - verbose_logger.debug(f"DataDogLLMObs: Error processing tool call {idx}: {e}") + verbose_logger.debug("DataDogLLMObs: Error processing tool call %s: %s", idx, e) continue return kv_pairs @@ -747,6 +747,6 @@ class DataDogLLMObsLogger(CustomBatchLogger): tool_call_metadata[f"output_{key}"] = value except Exception as e: - verbose_logger.debug(f"DataDogLLMObs: Error extracting tool call metadata: {e}") + verbose_logger.debug("DataDogLLMObs: Error extracting tool call metadata: %s", e) return tool_call_metadata diff --git a/litellm/integrations/datadog/datadog_metrics.py b/litellm/integrations/datadog/datadog_metrics.py index 9fb86bfb125..c33c44e4249 100644 --- a/litellm/integrations/datadog/datadog_metrics.py +++ b/litellm/integrations/datadog/datadog_metrics.py @@ -180,7 +180,7 @@ class DatadogMetricsLogger(CustomBatchLogger): await self.flush_queue() except Exception as e: - verbose_logger.exception(f"Datadog Metrics: Error in async_log_success_event: {e}") + verbose_logger.exception("Datadog Metrics: Error in async_log_success_event: %s", e) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): try: @@ -202,7 +202,7 @@ class DatadogMetricsLogger(CustomBatchLogger): await self.flush_queue() except Exception as e: - verbose_logger.exception(f"Datadog Metrics: Error in async_log_failure_event: {e}") + verbose_logger.exception("Datadog Metrics: Error in async_log_failure_event: %s", e) async def async_send_batch(self): if not self.log_queue: @@ -214,7 +214,7 @@ class DatadogMetricsLogger(CustomBatchLogger): try: await self._upload_to_datadog(payload_data) except Exception as e: - verbose_logger.exception(f"Datadog Metrics: Error in async_send_batch: {e}") + verbose_logger.exception("Datadog Metrics: Error in async_send_batch: %s", e) raise async def _upload_to_datadog(self, payload: DatadogMetricsPayload): @@ -242,7 +242,7 @@ class DatadogMetricsLogger(CustomBatchLogger): response.raise_for_status() verbose_logger.debug( - f"Datadog Metrics: Uploaded {len(payload['series'])} metric points. Status: {response.status_code}" + "Datadog Metrics: Uploaded %s metric points. Status: %s", len(payload["series"]), response.status_code ) async def async_health_check(self) -> IntegrationHealthCheckStatus: diff --git a/litellm/integrations/deepeval/api.py b/litellm/integrations/deepeval/api.py index adca8928df4..512c74e035c 100644 --- a/litellm/integrations/deepeval/api.py +++ b/litellm/integrations/deepeval/api.py @@ -23,9 +23,9 @@ def log_retry_error(details): exception = details.get("exception") tries = details.get("tries") if exception: - logging.error(f"Confident AI Error: {exception}. Retrying: {tries} time(s)...") + logging.error("Confident AI Error: %s. Retrying: %s time(s)...", exception, tries) else: - logging.error(f"Retrying: {tries} time(s)...") + logging.error("Retrying: %s time(s)...", tries) class HttpMethods(Enum): diff --git a/litellm/integrations/gcs_bucket/gcs_bucket.py b/litellm/integrations/gcs_bucket/gcs_bucket.py index b5b3d4e81a3..e402e4962b7 100644 --- a/litellm/integrations/gcs_bucket/gcs_bucket.py +++ b/litellm/integrations/gcs_bucket/gcs_bucket.py @@ -76,7 +76,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): await self.log_queue.put(GCSLogQueueItem(payload=logging_payload, kwargs=kwargs, response_obj=response_obj)) except Exception as e: - verbose_logger.exception(f"GCS Bucket logging error: {e}") + verbose_logger.exception("GCS Bucket logging error: %s", e) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): try: @@ -95,7 +95,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): await self.log_queue.put(GCSLogQueueItem(payload=logging_payload, kwargs=kwargs, response_obj=response_obj)) except Exception as e: - verbose_logger.exception(f"GCS Bucket logging error: {e}") + verbose_logger.exception("GCS Bucket logging error: %s", e) def _drain_queue_batch(self) -> list[GCSLogQueueItem]: """ @@ -218,7 +218,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): except Exception as e: success_count = 0 error_count = len(items) - verbose_logger.exception(f"GCS Bucket error logging batch payload to GCS bucket: {e}") + verbose_logger.exception("GCS Bucket error logging batch payload to GCS bucket: %s", e) return (success_count, error_count) async def _send_individual_logs(self, items: list[GCSLogQueueItem]) -> None: @@ -255,7 +255,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): logging_payload=item["payload"], ) except Exception as e: - verbose_logger.exception(f"GCS Bucket error logging individual payload to GCS bucket: {e}") + verbose_logger.exception("GCS Bucket error logging individual payload to GCS bucket: %s", e) async def async_send_batch(self): """ @@ -336,7 +336,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): loaded_response = json.loads(response) return loaded_response except Exception as e: - verbose_logger.debug(f"Failed to fetch payload for date {date_str}: {e}") + verbose_logger.debug("Failed to fetch payload for date %s: %s", date_str, e) continue return None @@ -370,7 +370,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): """ while True: await asyncio.sleep(self.flush_interval) - verbose_logger.debug(f"GCS Bucket periodic flush after {self.flush_interval} seconds") + verbose_logger.debug("GCS Bucket periodic flush after %s seconds", self.flush_interval) await self.flush_queue() async def async_health_check(self) -> IntegrationHealthCheckStatus: diff --git a/litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py b/litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py index 86cf8617dd5..20e89c0647b 100644 --- a/litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py +++ b/litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py @@ -45,7 +45,7 @@ async def _mock_async_handler_get(self, url, params=None, headers=None, follow_r """Monkey-patched AsyncHTTPHandler.get that intercepts GCS calls.""" # Only mock GCS API calls if isinstance(url, str) and "storage.googleapis.com" in url: - verbose_logger.info(f"[GCS MOCK] GET to {url}") + verbose_logger.info("[GCS MOCK] GET to %s", url) await asyncio.sleep(_MOCK_LATENCY_SECONDS) # Return a minimal but valid StandardLoggingPayload JSON string as bytes # This matches what GCS returns when downloading with ?alt=media @@ -117,7 +117,7 @@ async def _mock_async_handler_delete( """Monkey-patched AsyncHTTPHandler.delete that intercepts GCS calls.""" # Only mock GCS API calls if isinstance(url, str) and "storage.googleapis.com" in url: - verbose_logger.info(f"[GCS MOCK] DELETE to {url}") + verbose_logger.info("[GCS MOCK] DELETE to %s", url) await asyncio.sleep(_MOCK_LATENCY_SECONDS) # DELETE returns 204 No Content with empty body (not JSON) return MockResponse( diff --git a/litellm/integrations/gcs_pubsub/pub_sub.py b/litellm/integrations/gcs_pubsub/pub_sub.py index b43e7626b77..d915c7341df 100644 --- a/litellm/integrations/gcs_pubsub/pub_sub.py +++ b/litellm/integrations/gcs_pubsub/pub_sub.py @@ -132,7 +132,7 @@ class GcsPubSubLogger(CustomBatchLogger): await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"PubSub Layer Error - {e}\n{traceback.format_exc()}") + verbose_logger.exception("PubSub Layer Error - %s\n%s", e, traceback.format_exc()) async def async_send_batch(self): """ @@ -142,13 +142,13 @@ class GcsPubSubLogger(CustomBatchLogger): if not self.log_queue: return - verbose_logger.debug(f"PubSub - about to flush {len(self.log_queue)} events") + verbose_logger.debug("PubSub - about to flush %s events", len(self.log_queue)) for message in self.log_queue: await self.publish_message(message) except Exception as e: - verbose_logger.exception(f"PubSub Error sending batch - {e}\n{traceback.format_exc()}") + verbose_logger.exception("PubSub Error sending batch - %s\n%s", e, traceback.format_exc()) finally: self.log_queue.clear() diff --git a/litellm/integrations/generic_api/generic_api_callback.py b/litellm/integrations/generic_api/generic_api_callback.py index c7f2661a5ad..3e02826ee5d 100644 --- a/litellm/integrations/generic_api/generic_api_callback.py +++ b/litellm/integrations/generic_api/generic_api_callback.py @@ -42,7 +42,7 @@ def load_compatible_callbacks() -> dict: with open(json_path, "r") as f: return json.load(f) except Exception as e: - verbose_logger.warning(f"Error loading generic_api_compatible_callbacks.json: {e}") + verbose_logger.warning("Error loading generic_api_compatible_callbacks.json: %s", e) return {} @@ -124,7 +124,7 @@ class GenericAPILogger(CustomBatchLogger): ######################################################### if callback_name: if is_callback_compatible(callback_name): - verbose_logger.debug(f"Loading configuration for callback: {callback_name}") + verbose_logger.debug("Loading configuration for callback: %s", callback_name) callback_config = get_callback_config(callback_name) # Use config from JSON if not explicitly provided @@ -145,7 +145,7 @@ class GenericAPILogger(CustomBatchLogger): log_format = callback_config["log_format"] else: verbose_logger.warning( - f"callback_name '{callback_name}' not found in generic_api_compatible_callbacks.json" + "callback_name '%s' not found in generic_api_compatible_callbacks.json", callback_name ) ######################################################### @@ -177,7 +177,12 @@ class GenericAPILogger(CustomBatchLogger): self.log_format: LOG_FORMAT_TYPES = log_format or "json_array" verbose_logger.debug( - f"in init GenericAPILogger, callback_name: {self.callback_name}, endpoint {self.endpoint}, headers {self.headers}, event_types: {self.event_types}, log_format: {self.log_format}" + "in init GenericAPILogger, callback_name: %s, endpoint %s, headers %s, event_types: %s, log_format: %s", + self.callback_name, + self.endpoint, + self.headers, + self.event_types, + self.log_format, ) ######################################################### @@ -214,7 +219,7 @@ class GenericAPILogger(CustomBatchLogger): key, value = item.split("=", 1) headers_dict[key.strip()] = value.strip() except Exception as e: - verbose_logger.warning(f"Error parsing headers from environment variables: {e}") + verbose_logger.warning("Error parsing headers from environment variables: %s", e) # 2. Update with litellm generic headers if available if litellm.generic_logger_headers: @@ -308,7 +313,7 @@ class GenericAPILogger(CustomBatchLogger): await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"Generic API Logger Error - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Generic API Logger Error - %s\n%s", e, traceback.format_exc()) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): """ @@ -339,7 +344,7 @@ class GenericAPILogger(CustomBatchLogger): await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"Generic API Logger Error - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Generic API Logger Error - %s\n%s", e, traceback.format_exc()) async def async_send_batch(self): """ @@ -355,7 +360,7 @@ class GenericAPILogger(CustomBatchLogger): return verbose_logger.debug( - f"Generic API Logger - about to flush {len(self.log_queue)} events in '{self.log_format}' format" + "Generic API Logger - about to flush %s events in '%s' format", len(self.log_queue), self.log_format ) if self.log_format == "single": @@ -371,11 +376,13 @@ class GenericAPILogger(CustomBatchLogger): # Log results for idx, result in enumerate(responses): if isinstance(result, Exception): - verbose_logger.exception(f"Generic API Logger - Error sending log {idx}: {result}") + verbose_logger.exception("Generic API Logger - Error sending log %s: %s", idx, result) else: # result is a Response object verbose_logger.debug( - f"Generic API Logger - sent log {idx}, status: {result.status_code}" # type: ignore + "Generic API Logger - sent log %s, status: %s", + idx, + result.status_code, # type: ignore ) else: # Format the payload based on log_format @@ -390,12 +397,14 @@ class GenericAPILogger(CustomBatchLogger): response = await self._post_with_retries(data=data) verbose_logger.debug( - f"Generic API Logger - sent batch to {self.endpoint}, " - f"status: {response.status_code}, format: {self.log_format}" + "Generic API Logger - sent batch to %s, status: %s, format: %s", + self.endpoint, + response.status_code, + self.log_format, ) except Exception as e: - verbose_logger.exception(f"Generic API Logger Error sending batch - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Generic API Logger Error sending batch - %s\n%s", e, traceback.format_exc()) finally: self.log_queue.clear() @@ -405,7 +414,7 @@ class GenericAPILogger(CustomBatchLogger): Returns a dict of the payload to send to the Generic API Endpoint """ - verbose_logger.debug(f"GenericAPILogger Logging - Enters logging function for model {kwargs}") + verbose_logger.debug("GenericAPILogger Logging - Enters logging function for model %s", kwargs) # construct payload to send custom logger # follows the same params as langfuse.py diff --git a/litellm/integrations/gitlab/gitlab_prompt_manager.py b/litellm/integrations/gitlab/gitlab_prompt_manager.py index 54e0a3ad02e..9fda4ebcc19 100644 --- a/litellm/integrations/gitlab/gitlab_prompt_manager.py +++ b/litellm/integrations/gitlab/gitlab_prompt_manager.py @@ -379,7 +379,7 @@ class GitLabPromptManager(CustomPromptManagement): except Exception as e: import litellm - litellm._logging.verbose_proxy_logger.error(f"Error in GitLab prompt pre_call_hook: {e}") + litellm._logging.verbose_proxy_logger.error("Error in GitLab prompt pre_call_hook: %s", e) return messages, litellm_params def _parse_prompt_to_messages(self, prompt_content: str) -> list[AllMessageValues]: diff --git a/litellm/integrations/lago.py b/litellm/integrations/lago.py index 3186f1bf58b..000d3e2c79a 100644 --- a/litellm/integrations/lago.py +++ b/litellm/integrations/lago.py @@ -117,7 +117,7 @@ class LagoLogger(CustomLogger): } } - verbose_logger.debug(f"\033[91mLogged Lago Object:\n{returned_val}\033[0m\n") + verbose_logger.debug("\x1b[91mLogged Lago Object:\n%s\x1b[0m\n", returned_val) return returned_val def log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -149,7 +149,7 @@ class LagoLogger(CustomLogger): except Exception as e: error_response = getattr(e, "response", None) if error_response is not None and hasattr(error_response, "text"): - verbose_logger.debug(f"\nError Message: {error_response.text}") + verbose_logger.debug("\nError Message: %s", error_response.text) raise e async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -184,8 +184,8 @@ class LagoLogger(CustomLogger): response.raise_for_status() - verbose_logger.debug(f"Logged Lago Object: {response.text}") + verbose_logger.debug("Logged Lago Object: %s", response.text) except Exception as e: if response is not None and hasattr(response, "text"): - verbose_logger.debug(f"\nError Message: {response.text}") + verbose_logger.debug("\nError Message: %s", response.text) raise e diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 2dab1874c01..d10c7a699f5 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -199,7 +199,7 @@ class LangFuseLogger: ) langfuse_client = Langfuse(**parameters) litellm.initialized_langfuse_clients += 1 - verbose_logger.debug(f"Created langfuse client number {litellm.initialized_langfuse_clients}") + verbose_logger.debug("Created langfuse client number %s", litellm.initialized_langfuse_clients) return langfuse_client @staticmethod @@ -226,9 +226,9 @@ class LangFuseLogger: if metadata_param_key.startswith("langfuse_"): trace_param_key = metadata_param_key.replace("langfuse_", "", 1) if trace_param_key in metadata: - verbose_logger.warning(f"Overwriting Langfuse `{trace_param_key}` from request header") + verbose_logger.warning("Overwriting Langfuse `%s` from request header", trace_param_key) else: - verbose_logger.debug(f"Found Langfuse `{trace_param_key}` in request header") + verbose_logger.debug("Found Langfuse `%s` in request header", trace_param_key) metadata[trace_param_key] = proxy_headers.get(metadata_param_key) return metadata @@ -256,7 +256,7 @@ class LangFuseLogger: Logs a success or error event on Langfuse """ try: - verbose_logger.debug(f"Langfuse Logging - Enters logging function for model {kwargs}") + verbose_logger.debug("Langfuse Logging - Enters logging function for model %s", kwargs) # set default values for input/output for langfuse logging input = None @@ -295,7 +295,7 @@ class LangFuseLogger: level=level, status_message=status_message, ) - verbose_logger.debug(f"OUTPUT IN LANGFUSE: {output}; original: {response_obj}") + verbose_logger.debug("OUTPUT IN LANGFUSE: %s; original: %s", output, response_obj) trace_id = None generation_id = None if self._is_langfuse_v2(): @@ -325,12 +325,12 @@ class LangFuseLogger: input=input, response_obj=response_obj, ) - verbose_logger.debug(f"Langfuse Layer Logging - final response object: {response_obj}") + verbose_logger.debug("Langfuse Layer Logging - final response object: %s", response_obj) verbose_logger.info("Langfuse Layer Logging - logging success") return {"trace_id": trace_id, "generation_id": generation_id} except Exception as e: - verbose_logger.exception(f"Langfuse Layer Error(): Exception occured - {e}") + verbose_logger.exception("Langfuse Layer Error(): Exception occured - %s", e) return {"trace_id": None, "generation_id": None} def _get_langfuse_input_output_content( @@ -625,7 +625,7 @@ class LangFuseLogger: trace_params["metadata"] = {"metadata_passed_to_litellm": metadata} cost = kwargs.get("response_cost", None) - verbose_logger.debug(f"trace: {cost}") + verbose_logger.debug("trace: %s", cost) clean_metadata["litellm_response_cost"] = cost if standard_logging_object is not None: @@ -780,12 +780,13 @@ class LangFuseLogger: if hasattr(generation_client, "trace_id") and generation_client.trace_id: if generation_client.trace_id != trace_id: verbose_logger.warning( - f"Langfuse trace_id mismatch: set {trace_id}, but langfuse returned {generation_client.trace_id}. " - "Using our intended trace_id for consistency." + "Langfuse trace_id mismatch: set %s, but langfuse returned %s. Using our intended trace_id for consistency.", + trace_id, + generation_client.trace_id, ) return trace_id, generation_id except Exception: - verbose_logger.error(f"Langfuse Layer Error - {traceback.format_exc()}") + verbose_logger.error("Langfuse Layer Error - %s", traceback.format_exc()) return None, None @staticmethod @@ -902,7 +903,7 @@ class LangFuseLogger: # For other types, try to apply the function directly return masking_function(data) except Exception as e: - verbose_logger.warning(f"Failed to apply masking function: {e}. Returning original data.") + verbose_logger.warning("Failed to apply masking function: %s. Returning original data.", e) return data @staticmethod @@ -966,7 +967,7 @@ class LangFuseLogger: end_time=guardrail_entry.get("end_time", None), # type: ignore ) - verbose_logger.debug(f"Logged guardrail information as span: {span}") + verbose_logger.debug("Logged guardrail information as span: %s", span) span.end() @@ -1035,7 +1036,7 @@ def _add_prompt_to_generation_params( try: generation_params["prompt"] = langfuse_client.get_prompt(prompt_management_metadata["prompt_id"]) except Exception as e: - verbose_logger.debug(f"[Non-blocking] Langfuse Logger: Error getting prompt client for logging: {e}") + verbose_logger.debug("[Non-blocking] Langfuse Logger: Error getting prompt client for logging: %s", e) else: generation_params["prompt"] = user_prompt diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index 143362f3468..d7e9460a580 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -315,10 +315,10 @@ class LangfuseOtelLogger(OpenTelemetry): if langfuse_host: normalized_host = langfuse_host if langfuse_host.startswith("http") else f"https://{langfuse_host}" endpoint = f"{normalized_host.rstrip('/')}/api/public/otel" - verbose_logger.debug(f"Using Langfuse OTEL endpoint from host: {endpoint}") + verbose_logger.debug("Using Langfuse OTEL endpoint from host: %s", endpoint) else: endpoint = LANGFUSE_CLOUD_US_ENDPOINT - verbose_logger.debug(f"Using Langfuse US cloud endpoint: {endpoint}") + verbose_logger.debug("Using Langfuse US cloud endpoint: %s", endpoint) auth_header = LangfuseOtelLogger._get_langfuse_authorization_header( public_key=public_key, secret_key=secret_key diff --git a/litellm/integrations/langfuse/langfuse_prompt_management.py b/litellm/integrations/langfuse/langfuse_prompt_management.py index 56383b45a8c..f7fc63c0866 100644 --- a/litellm/integrations/langfuse/langfuse_prompt_management.py +++ b/litellm/integrations/langfuse/langfuse_prompt_management.py @@ -317,7 +317,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge except Exception as e: from litellm._logging import verbose_logger - verbose_logger.exception(f"Langfuse Layer Error - Exception occurred while logging success event: {e}") + verbose_logger.exception("Langfuse Layer Error - Exception occurred while logging success event: %s", e) self.handle_callback_failure(callback_name="langfuse") async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): @@ -347,5 +347,5 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge except Exception as e: from litellm._logging import verbose_logger - verbose_logger.exception(f"Langfuse Layer Error - Exception occurred while logging failure event: {e}") + verbose_logger.exception("Langfuse Layer Error - Exception occurred while logging failure event: %s", e) self.handle_callback_failure(callback_name="langfuse") diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 1f5d3179fb3..6dd0863cc41 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -194,7 +194,7 @@ class LangsmithLogger(CustomBatchLogger): fields = self._extract_metadata_fields(metadata, credentials) verbose_logger.debug( - f"Langsmith Logging - project_name: {fields['project_name']}, run_name {fields['run_name']}" + "Langsmith Logging - project_name: %s, run_name %s", fields["project_name"], fields["run_name"] ) payload: StandardLoggingPayload | None = kwargs.get("standard_logging_object", None) @@ -244,7 +244,7 @@ class LangsmithLogger(CustomBatchLogger): random_sample = random.random() if random_sample > sampling_rate: verbose_logger.info( - f"Skipping Langsmith logging. Sampling rate={sampling_rate}, random_sample={random_sample}" + "Skipping Langsmith logging. Sampling rate=%s, random_sample=%s", sampling_rate, random_sample ) return # Skip logging verbose_logger.debug( @@ -267,7 +267,7 @@ class LangsmithLogger(CustomBatchLogger): credentials=credentials, ) ) - verbose_logger.debug(f"Langsmith, event added to queue. Will flush in {self.flush_interval} seconds...") + verbose_logger.debug("Langsmith, event added to queue. Will flush in %s seconds...", self.flush_interval) if len(self.log_queue) >= self.batch_size: self._send_batch() @@ -282,7 +282,7 @@ class LangsmithLogger(CustomBatchLogger): random_sample = random.random() if random_sample > sampling_rate: verbose_logger.info( - f"Skipping Langsmith logging. Sampling rate={sampling_rate}, random_sample={random_sample}" + "Skipping Langsmith logging. Sampling rate=%s, random_sample=%s", sampling_rate, random_sample ) return # Skip logging verbose_logger.debug( @@ -321,7 +321,7 @@ class LangsmithLogger(CustomBatchLogger): random_sample = random.random() if random_sample > sampling_rate: verbose_logger.info( - f"Skipping Langsmith logging. Sampling rate={sampling_rate}, random_sample={random_sample}" + "Skipping Langsmith logging. Sampling rate=%s, random_sample=%s", sampling_rate, random_sample ) return # Skip logging verbose_logger.info("Langsmith Failure Event Logging!") @@ -422,16 +422,16 @@ class LangsmithLogger(CustomBatchLogger): response.raise_for_status() if response.status_code >= 300: - verbose_logger.error(f"Langsmith Error: {response.status_code} - {response.text}") + verbose_logger.error("Langsmith Error: %s - %s", response.status_code, response.text) else: if self.is_mock_mode: - verbose_logger.debug(f"[LANGSMITH MOCK] Batch of {len(elements_to_log)} runs successfully mocked") + verbose_logger.debug("[LANGSMITH MOCK] Batch of %s runs successfully mocked", len(elements_to_log)) else: - verbose_logger.debug(f"Batch of {len(self.log_queue)} runs successfully created") + verbose_logger.debug("Batch of %s runs successfully created", len(self.log_queue)) except httpx.HTTPStatusError as e: - verbose_logger.exception(f"Langsmith HTTP Error: {e.response.status_code} - {e.response.text}") + verbose_logger.exception("Langsmith HTTP Error: %s - %s", e.response.status_code, e.response.text) except Exception: - verbose_logger.exception(f"Langsmith Layer Error - {traceback.format_exc()}") + verbose_logger.exception("Langsmith Layer Error - %s", traceback.format_exc()) def _group_batches_by_credentials(self) -> dict[CredentialsKey, BatchGroup]: """Groups queue objects by credentials using a proper key structure""" diff --git a/litellm/integrations/literal_ai.py b/litellm/integrations/literal_ai.py index a54fdcf4dbc..a17a35f6a0e 100644 --- a/litellm/integrations/literal_ai.py +++ b/litellm/integrations/literal_ai.py @@ -94,9 +94,9 @@ class LiteralAILogger(CustomBatchLogger): ) if response.status_code >= 300: - verbose_logger.error(f"Literal AI Error: {response.status_code} - {response.text}") + verbose_logger.error("Literal AI Error: %s - %s", response.status_code, response.text) else: - verbose_logger.debug(f"Batch of {len(self.log_queue)} runs successfully created") + verbose_logger.debug("Batch of %s runs successfully created", len(self.log_queue)) except Exception: verbose_logger.exception("Literal AI Layer Error") @@ -152,11 +152,11 @@ class LiteralAILogger(CustomBatchLogger): headers=self.headers, ) if response.status_code >= 300: - verbose_logger.error(f"Literal AI Error: {response.status_code} - {response.text}") + verbose_logger.error("Literal AI Error: %s - %s", response.status_code, response.text) else: - verbose_logger.debug(f"Batch of {len(self.log_queue)} runs successfully created") + verbose_logger.debug("Batch of %s runs successfully created", len(self.log_queue)) except httpx.HTTPStatusError as e: - verbose_logger.exception(f"Literal AI HTTP Error: {e.response.status_code} - {e.response.text}") + verbose_logger.exception("Literal AI HTTP Error: %s - %s", e.response.status_code, e.response.text) except Exception: verbose_logger.exception("Literal AI Layer Error") diff --git a/litellm/integrations/logfire_logger.py b/litellm/integrations/logfire_logger.py index c94fb832ccc..814e2b88ba1 100644 --- a/litellm/integrations/logfire_logger.py +++ b/litellm/integrations/logfire_logger.py @@ -90,7 +90,7 @@ class LogfireLogger: try: import logfire - verbose_logger.debug(f"logfire Logging - Enters logging function for model {kwargs}") + verbose_logger.debug("logfire Logging - Enters logging function for model %s", kwargs) if not response_obj: response_obj = {} @@ -159,4 +159,4 @@ class LogfireLogger: print_verbose(f"Logfire Layer Logging - final response object: {response_obj}") except Exception as e: - verbose_logger.debug(f"Logfire Layer Error - {e}\n{traceback.format_exc()}") + verbose_logger.debug("Logfire Layer Error - %s\n%s", e, traceback.format_exc()) diff --git a/litellm/integrations/mlflow.py b/litellm/integrations/mlflow.py index 7ce5b3d3eea..f41de320843 100644 --- a/litellm/integrations/mlflow.py +++ b/litellm/integrations/mlflow.py @@ -99,7 +99,7 @@ class MlflowLogger(CustomLogger): ) except Exception as e: - verbose_logger.debug(f"MLflow Logging Error - {e}", stack_info=True) + verbose_logger.debug("MLflow Logging Error - %s", e, stack_info=True) def _handle_stream_event(self, kwargs, response_obj, start_time, end_time): """ diff --git a/litellm/integrations/mock_client_factory.py b/litellm/integrations/mock_client_factory.py index 2aab1792618..f1bc2db16ef 100644 --- a/litellm/integrations/mock_client_factory.py +++ b/litellm/integrations/mock_client_factory.py @@ -144,7 +144,7 @@ def create_mock_client_factory(config: MockClientConfig): ): """Monkey-patched AsyncHTTPHandler.post that intercepts API calls.""" if isinstance(url, str) and _is_mock_url(url): - verbose_logger.info(f"[{config.name} MOCK] POST to {url}") + verbose_logger.info("[%s MOCK] POST to %s", config.name, url) await asyncio.sleep(_MOCK_LATENCY_SECONDS) return MockResponse( status_code=config.default_status_code, @@ -172,7 +172,7 @@ def create_mock_client_factory(config: MockClientConfig): def _mock_sync_client_post(self, url, **kwargs): """Monkey-patched httpx.Client.post that intercepts API calls.""" if _is_mock_url(url): - verbose_logger.info(f"[{config.name} MOCK] POST to {url} (sync)") + verbose_logger.info("[%s MOCK] POST to %s (sync)", config.name, url) return MockResponse( status_code=config.default_status_code, json_data=config.default_json_data, @@ -198,7 +198,7 @@ def create_mock_client_factory(config: MockClientConfig): ): """Monkey-patched HTTPHandler.post that intercepts API calls.""" if isinstance(url, str) and _is_mock_url(url): - verbose_logger.info(f"[{config.name} MOCK] POST to {url}") + verbose_logger.info("[%s MOCK] POST to %s", config.name, url) import time time.sleep(_MOCK_LATENCY_SECONDS) @@ -236,29 +236,29 @@ def create_mock_client_factory(config: MockClientConfig): if _mocks_initialized: return - verbose_logger.debug(f"[{config.name} MOCK] Initializing {config.name} mock client...") + verbose_logger.debug("[%s MOCK] Initializing %s mock client...", config.name, config.name) if config.patch_async_handler and _original_async_handler_post is None: from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler _original_async_handler_post = AsyncHTTPHandler.post AsyncHTTPHandler.post = _mock_async_handler_post # type: ignore - verbose_logger.debug(f"[{config.name} MOCK] Patched AsyncHTTPHandler.post") + verbose_logger.debug("[%s MOCK] Patched AsyncHTTPHandler.post", config.name) if config.patch_sync_client and _original_sync_client_post is None: _original_sync_client_post = httpx.Client.post httpx.Client.post = _mock_sync_client_post # type: ignore - verbose_logger.debug(f"[{config.name} MOCK] Patched httpx.Client.post") + verbose_logger.debug("[%s MOCK] Patched httpx.Client.post", config.name) if config.patch_http_handler and _original_http_handler_post is None: from litellm.llms.custom_httpx.http_handler import HTTPHandler _original_http_handler_post = HTTPHandler.post HTTPHandler.post = _mock_http_handler_post # type: ignore - verbose_logger.debug(f"[{config.name} MOCK] Patched HTTPHandler.post") + verbose_logger.debug("[%s MOCK] Patched HTTPHandler.post", config.name) verbose_logger.debug(f"[{config.name} MOCK] Mock latency set to {_MOCK_LATENCY_SECONDS * 1000:.0f}ms") - verbose_logger.debug(f"[{config.name} MOCK] {config.name} mock client initialization complete") + verbose_logger.debug("[%s MOCK] %s mock client initialization complete", config.name, config.name) _mocks_initialized = True @@ -274,7 +274,7 @@ def create_mock_client_factory(config: MockClientConfig): result = bool(result) if result is not None else False if result: - verbose_logger.info(f"{config.name} Mock Mode: ENABLED - API calls will be mocked") + verbose_logger.info("%s Mock Mode: ENABLED - API calls will be mocked", config.name) return result diff --git a/litellm/integrations/newrelic/newrelic.py b/litellm/integrations/newrelic/newrelic.py index 5511cb06174..bf8ab29d384 100644 --- a/litellm/integrations/newrelic/newrelic.py +++ b/litellm/integrations/newrelic/newrelic.py @@ -116,11 +116,12 @@ class NewRelicLogger(CustomLogger): self.enabled = True verbose_logger.info( - f"New Relic AI Monitoring initialized for app: {self.app_name}, " - f"content recording: {self.record_content}" + "New Relic AI Monitoring initialized for app: %s, content recording: %s", + self.app_name, + self.record_content, ) except Exception as e: - verbose_logger.error(f"Failed to initialize New Relic agent: {e}. Integration will be disabled.") + verbose_logger.error("Failed to initialize New Relic agent: %s. Integration will be disabled.", e) self.enabled = False def _get_newrelic_params(self) -> dict: @@ -170,9 +171,10 @@ class NewRelicLogger(CustomLogger): if value in ("0", "false", "no", "off"): return False verbose_logger.warning( - f"{var_name}={raw!r} is not a recognised boolean " - f"(accepts true/false, 1/0, yes/no, on/off). " - f"Falling back to default ({default})." + "%s=%r is not a recognised boolean (accepts true/false, 1/0, yes/no, on/off). Falling back to default (%s).", + var_name, + raw, + default, ) return default @@ -188,7 +190,7 @@ class NewRelicLogger(CustomLogger): return version("litellm") except Exception as e: - verbose_logger.warning(f"Unable to determine litellm version: {e}") + verbose_logger.warning("Unable to determine litellm version: %s", e) return "unknown" def _emit_supportability_metric(self): @@ -216,12 +218,12 @@ class NewRelicLogger(CustomLogger): if app and app.enabled: app.record_custom_metric(metric_name, 1) - verbose_logger.info(f"Emitted New Relic supportability metric: {metric_name}") + verbose_logger.info("Emitted New Relic supportability metric: %s", metric_name) else: verbose_logger.info("New Relic application is not enabled; skipping metric recording.") except Exception as e: - verbose_logger.warning(f"Failed to emit supportability metric: {e}") + verbose_logger.warning("Failed to emit supportability metric: %s", e) def _check_and_emit_periodic_metric(self): """ @@ -294,14 +296,13 @@ class NewRelicLogger(CustomLogger): trace_id = slo_trace_id except Exception as e: - verbose_logger.warning(f"Unable to parse New Relic trace context from upstream sources: {e}") + verbose_logger.warning("Unable to parse New Relic trace context from upstream sources: %s", e) if not trace_id: trace_id = uuid.uuid4().hex verbose_logger.debug( - f"New Relic trace_id not available from distributed tracing headers or " - f"StandardLoggingPayload. Generated trace_id={trace_id} for AI monitoring " - f"event grouping." + "New Relic trace_id not available from distributed tracing headers or StandardLoggingPayload. Generated trace_id=%s for AI monitoring event grouping.", + trace_id, ) return trace_id @@ -638,7 +639,7 @@ class NewRelicLogger(CustomLogger): verbose_logger.warning("New Relic application is not enabled; skipping summary event recording.") except Exception as e: - verbose_logger.warning(f"Failed to record New Relic summary event: {e}") + verbose_logger.warning("Failed to record New Relic summary event: %s", e) self.handle_callback_failure("newrelic") def _record_message_events( @@ -699,7 +700,7 @@ class NewRelicLogger(CustomLogger): app.record_custom_event("LlmChatCompletionMessage", event_data) except Exception as e: - verbose_logger.warning(f"Failed to record New Relic message events: {e}") + verbose_logger.warning("Failed to record New Relic message events: %s", e) self.handle_callback_failure("newrelic") def _record_error_metric(self): @@ -714,7 +715,7 @@ class NewRelicLogger(CustomLogger): if app and app.enabled: app.record_custom_metric("LLM/LiteLLM/Error", 1) except Exception as e: - verbose_logger.warning(f"Failed to record New Relic error metric: {e}") + verbose_logger.warning("Failed to record New Relic error metric: %s", e) self.handle_callback_failure("newrelic") def _process_success( @@ -846,7 +847,7 @@ class NewRelicLogger(CustomLogger): try: self._process_success(kwargs, response_obj, start_time, end_time) except Exception as e: - verbose_logger.warning(f"Error in New Relic log_success_event: {e}") + verbose_logger.warning("Error in New Relic log_success_event: %s", e) self.handle_callback_failure("newrelic") async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -859,7 +860,7 @@ class NewRelicLogger(CustomLogger): try: self._process_success(kwargs, response_obj, start_time, end_time) except Exception as e: - verbose_logger.warning(f"Error in New Relic async_log_success_event: {e}") + verbose_logger.warning("Error in New Relic async_log_success_event: %s", e) self.handle_callback_failure("newrelic") def log_failure_event(self, kwargs, response_obj, start_time, end_time): @@ -872,7 +873,7 @@ class NewRelicLogger(CustomLogger): self._record_error_metric() except Exception as e: - verbose_logger.warning(f"Error in New Relic log_failure_event: {e}") + verbose_logger.warning("Error in New Relic log_failure_event: %s", e) self.handle_callback_failure("newrelic") async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): @@ -885,5 +886,5 @@ class NewRelicLogger(CustomLogger): self._record_error_metric() except Exception as e: - verbose_logger.warning(f"Error in New Relic async_log_failure_event: {e}") + verbose_logger.warning("Error in New Relic async_log_failure_event: %s", e) self.handle_callback_failure("newrelic") diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 5342894a04f..94902460c81 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -2688,7 +2688,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): ) except json.JSONDecodeError: verbose_logger.debug( - f"litellm.integrations.opentelemetry.py::set_raw_request_attributes() - raw_response not json string - {_raw_response}" + "litellm.integrations.opentelemetry.py::set_raw_request_attributes() - raw_response not json string - %s", + _raw_response, ) self.safe_set_attribute( diff --git a/litellm/integrations/opik/opik.py b/litellm/integrations/opik/opik.py index e4d40a1af8f..afb6fef9d71 100644 --- a/litellm/integrations/opik/opik.py +++ b/litellm/integrations/opik/opik.py @@ -81,7 +81,7 @@ class OpikLogger(CustomBatchLogger): self.flush_lock: asyncio.Lock | None = asyncio.Lock() except Exception as e: verbose_logger.exception( - f"OpikLogger - Asynchronous processing not initialized as we are not running in an async context {e}" + "OpikLogger - Asynchronous processing not initialized as we are not running in an async context %s", e ) self.flush_lock = None @@ -154,14 +154,14 @@ class OpikLogger(CustomBatchLogger): self.log_queue.append(span_payload.__dict__) verbose_logger.debug( - f"OpikLogger added event to log_queue - Will flush in {self.flush_interval} seconds..." + "OpikLogger added event to log_queue - Will flush in %s seconds...", self.flush_interval ) if len(self.log_queue) >= self.batch_size: verbose_logger.debug("OpikLogger - Flushing batch") await self.flush_queue() except Exception as e: - verbose_logger.exception(f"OpikLogger failed to log success event - {e}\n{traceback.format_exc()}") + verbose_logger.exception("OpikLogger failed to log success event - %s\n%s", e, traceback.format_exc()) def _sync_send(self, url: str, headers: dict[str, str], batch: dict[str, Any]) -> None: try: @@ -174,7 +174,7 @@ class OpikLogger(CustomBatchLogger): if response.status_code != 204: raise Exception(f"Response from opik API status_code: {response.status_code}, text: {response.text}") except Exception as e: - verbose_logger.exception(f"OpikLogger failed to send batch - {e}\n{traceback.format_exc()}") + verbose_logger.exception("OpikLogger failed to send batch - %s\n%s", e, traceback.format_exc()) def log_success_event( self, @@ -245,7 +245,7 @@ class OpikLogger(CustomBatchLogger): batch={"spans": [span_payload.__dict__]}, ) except Exception as e: - verbose_logger.exception(f"OpikLogger failed to log success event - {e}\n{traceback.format_exc()}") + verbose_logger.exception("OpikLogger failed to log success event - %s\n%s", e, traceback.format_exc()) async def _submit_batch(self, url: str, headers: dict[str, str], batch: dict[str, Any]) -> None: try: @@ -257,11 +257,11 @@ class OpikLogger(CustomBatchLogger): response.raise_for_status() if response.status_code >= 300: - verbose_logger.error(f"OpikLogger - Error: {response.status_code} - {response.text}") + verbose_logger.error("OpikLogger - Error: %s - %s", response.status_code, response.text) else: - verbose_logger.info(f"OpikLogger - {len(self.log_queue)} Opik events submitted") + verbose_logger.info("OpikLogger - %s Opik events submitted", len(self.log_queue)) except Exception as e: - verbose_logger.exception(f"OpikLogger failed to send batch - {e}") + verbose_logger.exception("OpikLogger failed to send batch - %s", e) def _create_opik_headers(self) -> dict[str, str]: headers: dict[str, str] = {} @@ -283,7 +283,7 @@ class OpikLogger(CustomBatchLogger): # Send trace batch if len(traces) > 0: await self._submit_batch(url=self.trace_url, headers=self.headers, batch={"traces": traces}) - verbose_logger.info(f"Sent {len(traces)} traces") + verbose_logger.info("Sent %s traces", len(traces)) if len(spans) > 0: await self._submit_batch(url=self.span_url, headers=self.headers, batch={"spans": spans}) - verbose_logger.info(f"Sent {len(spans)} spans") + verbose_logger.info("Sent %s spans", len(spans)) diff --git a/litellm/integrations/opik/opik_payload_builder/extractors.py b/litellm/integrations/opik/opik_payload_builder/extractors.py index f95bd110cb3..ccc59cde751 100644 --- a/litellm/integrations/opik/opik_payload_builder/extractors.py +++ b/litellm/integrations/opik/opik_payload_builder/extractors.py @@ -66,7 +66,7 @@ def extract_opik_metadata( if requester_opik: opik_meta.update(requester_opik) - _logging.verbose_logger.debug(f"litellm_opik_metadata - {json.dumps(opik_meta, default=str)}") + _logging.verbose_logger.debug("litellm_opik_metadata - %s", json.dumps(opik_meta, default=str)) return opik_meta @@ -92,7 +92,7 @@ def extract_span_identifiers( try: return current_span_data.trace_id, current_span_data.id except AttributeError: - _logging.verbose_logger.warning(f"Unexpected current_span_data format: {type(current_span_data)}") + _logging.verbose_logger.warning("Unexpected current_span_data format: %s", type(current_span_data)) return None, None @@ -152,7 +152,7 @@ def apply_proxy_header_overrides( if isinstance(parsed_tags, list): tags.extend(parsed_tags) except (json.JSONDecodeError, TypeError): - _logging.verbose_logger.warning(f"Failed to parse tags from header: {value}") + _logging.verbose_logger.warning("Failed to parse tags from header: %s", value) return project_name, tags, thread_id diff --git a/litellm/integrations/opik/opik_payload_builder/payload_builders.py b/litellm/integrations/opik/opik_payload_builder/payload_builders.py index 517d5431b70..19e1dd1042c 100644 --- a/litellm/integrations/opik/opik_payload_builder/payload_builders.py +++ b/litellm/integrations/opik/opik_payload_builder/payload_builders.py @@ -61,7 +61,7 @@ def build_span_payload( created = response_obj.get("created", 0) span_name = f"{model}_{obj_type}_{created}" - _logging.verbose_logger.debug(f"OpikLogger creating span with id {span_id} for trace {trace_id}") + _logging.verbose_logger.debug("OpikLogger creating span with id %s for trace %s", span_id, trace_id) return types.SpanPayload( id=span_id, diff --git a/litellm/integrations/posthog.py b/litellm/integrations/posthog.py index 216edc44d3f..e6620535d81 100644 --- a/litellm/integrations/posthog.py +++ b/litellm/integrations/posthog.py @@ -72,7 +72,7 @@ class PostHogLogger(CustomBatchLogger): super().__init__(**kwargs, flush_lock=None, batch_size=POSTHOG_MAX_BATCH_SIZE) except Exception as e: - verbose_logger.exception(f"PostHog: Got exception on init PostHog client {e}") + verbose_logger.exception("PostHog: Got exception on init PostHog client %s", e) raise e def log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -107,7 +107,7 @@ class PostHogLogger(CustomBatchLogger): verbose_logger.debug("PostHog: Sync event successfully sent") except Exception as e: - verbose_logger.exception(f"PostHog Sync Layer Error - {e}") + verbose_logger.exception("PostHog Sync Layer Error - %s", e) async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): try: @@ -115,7 +115,7 @@ class PostHogLogger(CustomBatchLogger): self._ensure_async_setup() # Lazy initialization await self._log_async_event(kwargs, response_obj, start_time, end_time) except Exception as e: - verbose_logger.exception(f"PostHog Layer Error - {e}") + verbose_logger.exception("PostHog Layer Error - %s", e) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): try: @@ -123,7 +123,7 @@ class PostHogLogger(CustomBatchLogger): self._ensure_async_setup() # Lazy initialization await self._log_async_event(kwargs, response_obj, start_time, end_time) except Exception as e: - verbose_logger.exception(f"PostHog Layer Error - {e}") + verbose_logger.exception("PostHog Layer Error - %s", e) async def _log_async_event(self, kwargs, response_obj=None, start_time=0.0, end_time=0.0): # Note: response_obj, start_time, end_time not used - all data comes from kwargs @@ -132,7 +132,7 @@ class PostHogLogger(CustomBatchLogger): # Store event with its credentials for batch sending self.log_queue.append({"event": event_payload, "api_key": api_key, "api_url": api_url}) - verbose_logger.debug(f"PostHog, event added to queue. Will flush in {self.flush_interval} seconds...") + verbose_logger.debug("PostHog, event added to queue. Will flush in %s seconds...", self.flush_interval) if len(self.log_queue) >= self.batch_size: await self.flush_queue() @@ -328,7 +328,7 @@ class PostHogLogger(CustomBatchLogger): if not self.log_queue: return - verbose_logger.debug(f"PostHog: Sending batch of {len(self.log_queue)} events") + verbose_logger.debug("PostHog: Sending batch of %s events", len(self.log_queue)) if self.is_mock_mode: verbose_logger.debug("[POSTHOG MOCK] Mock mode enabled - API calls will be intercepted") @@ -363,11 +363,11 @@ class PostHogLogger(CustomBatchLogger): ) if self.is_mock_mode: - verbose_logger.debug(f"[POSTHOG MOCK] Batch of {len(self.log_queue)} events successfully mocked") + verbose_logger.debug("[POSTHOG MOCK] Batch of %s events successfully mocked", len(self.log_queue)) else: - verbose_logger.debug(f"PostHog: Batch of {len(self.log_queue)} events successfully sent") + verbose_logger.debug("PostHog: Batch of %s events successfully sent", len(self.log_queue)) except Exception as e: - verbose_logger.exception(f"PostHog Error sending batch API - {e}") + verbose_logger.exception("PostHog Error sending batch API - %s", e) def _ensure_async_setup(self): if not self._async_initialized: @@ -377,7 +377,7 @@ class PostHogLogger(CustomBatchLogger): self._async_initialized = True verbose_logger.debug("PostHog: Async components initialized") except Exception as e: - verbose_logger.error(f"PostHog: Failed to initialize async components: {e}") + verbose_logger.error("PostHog: Failed to initialize async components: %s", e) raise def _extract_metadata(self, kwargs: dict[str, Any]) -> dict[str, Any]: @@ -408,7 +408,7 @@ class PostHogLogger(CustomBatchLogger): if not self.log_queue: return - verbose_logger.debug(f"PostHog: Flushing {len(self.log_queue)} remaining events on exit") + verbose_logger.debug("PostHog: Flushing %s remaining events on exit", len(self.log_queue)) try: # Group events by credentials (same logic as async_send_batch) @@ -436,13 +436,13 @@ class PostHogLogger(CustomBatchLogger): response.raise_for_status() if response.status_code != 200: - verbose_logger.error(f"PostHog: Failed to flush on exit - status {response.status_code}") + verbose_logger.error("PostHog: Failed to flush on exit - status %s", response.status_code) if self.is_mock_mode: - verbose_logger.debug(f"[POSTHOG MOCK] Successfully flushed {len(self.log_queue)} events on exit") + verbose_logger.debug("[POSTHOG MOCK] Successfully flushed %s events on exit", len(self.log_queue)) else: - verbose_logger.debug(f"PostHog: Successfully flushed {len(self.log_queue)} events on exit") + verbose_logger.debug("PostHog: Successfully flushed %s events on exit", len(self.log_queue)) self.log_queue.clear() except Exception as e: - verbose_logger.error(f"PostHog: Error flushing events on exit: {e}") + verbose_logger.error("PostHog: Error flushing events on exit: %s", e) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index b7705a40e0c..78f4e213f50 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -697,7 +697,7 @@ class PrometheusLogger(CustomLogger): if not config: return {} - verbose_logger.debug(f"prometheus config: {config}") + verbose_logger.debug("prometheus config: %s", config) # Parse and validate all configuration groups parsed_configs = [] @@ -963,7 +963,10 @@ class PrometheusLogger(CustomLogger): except ImportError: # Fallback to simple logging if rich is not available verbose_logger.error( - f"Invalid labels for metric '{metric_name}': {invalid_labels}. Valid labels: {sorted(valid_labels)}" + "Invalid labels for metric '%s': %s. Valid labels: %s", + metric_name, + invalid_labels, + sorted(valid_labels), ) def _pretty_print_invalid_metric_error(self, invalid_metric_name: str, valid_metrics: tuple) -> None: @@ -1003,7 +1006,9 @@ class PrometheusLogger(CustomLogger): except ImportError: # Fallback to simple logging if rich is not available - verbose_logger.error(f"Invalid metric name: {invalid_metric_name}. Valid metrics: {sorted(valid_metrics)}") + verbose_logger.error( + "Invalid metric name: %s. Valid metrics: %s", invalid_metric_name, sorted(valid_metrics) + ) ######################################################### # End of pretty print functions @@ -1078,9 +1083,10 @@ class PrometheusLogger(CustomLogger): except ImportError: # Fallback to simple logging if rich is not available verbose_logger.info( - f"Enabled metrics: {sorted(self.enabled_metrics) if hasattr(self, 'enabled_metrics') else 'All metrics'}" + "Enabled metrics: %s", + sorted(self.enabled_metrics) if hasattr(self, "enabled_metrics") else "All metrics", ) - verbose_logger.info(f"Label filters: {label_filters}") + verbose_logger.info("Label filters: %s", label_filters) def _is_metric_enabled(self, metric_name: str) -> bool: """Check if a metric is enabled based on configuration""" @@ -1866,7 +1872,9 @@ class PrometheusLogger(CustomLogger): for i, r in enumerate(results): if isinstance(r, Exception): verbose_logger.debug( - f"[Non-Blocking] Prometheus: Budget metric lookup {['key', 'team', 'user', 'org'][i]} failed: {r}" + "[Non-Blocking] Prometheus: Budget metric lookup %s failed: %s", + ["key", "team", "user", "org"][i], + r, ) def _increment_top_level_request_and_spend_metrics( @@ -2132,7 +2140,7 @@ class PrometheusLogger(CustomLogger): response_cost=0, ) except Exception as e: - verbose_logger.exception(f"prometheus Layer Error(): Exception occured - {e}") + verbose_logger.exception("prometheus Layer Error(): Exception occured - %s", e) def _extract_status_code( self, @@ -2262,8 +2270,9 @@ class PrometheusLogger(CustomLogger): if self._is_invalid_api_key_request(status_code, exception=exception): verbose_logger.debug( - "Skipping Prometheus metrics for invalid API key request: " - f"status_code={status_code}, exception={type(exception).__name__ if exception else None}" + "Skipping Prometheus metrics for invalid API key request: status_code=%s, exception=%s", + status_code, + type(exception).__name__ if exception else None, ) return True @@ -2383,7 +2392,7 @@ class PrometheusLogger(CustomLogger): ) except Exception as e: - verbose_logger.exception(f"prometheus Layer Error(): Exception occured - {e}") + verbose_logger.exception("prometheus Layer Error(): Exception occured - %s", e) async def async_post_call_success_hook(self, data: dict, user_api_key_dict: UserAPIKeyAuth, response): """ @@ -2608,7 +2617,7 @@ class PrometheusLogger(CustomLogger): ) except Exception as e: - verbose_logger.debug(f"Prometheus Error: set_llm_deployment_failure_metrics. Exception occured - {e}") + verbose_logger.debug("Prometheus Error: set_llm_deployment_failure_metrics. Exception occured - %s", e) def _set_deployment_tpm_rpm_limit_metrics( self, @@ -2722,7 +2731,7 @@ class PrometheusLogger(CustomLogger): ) self.litellm_remaining_requests_metric.labels(**_labels).set(remaining_requests) except Exception as e: - verbose_logger.exception(f"Prometheus Error: _async_set_router_remaining_metrics. Exception occured - {e}") + verbose_logger.exception("Prometheus Error: _async_set_router_remaining_metrics. Exception occured - %s", e) def set_llm_deployment_success_metrics( self, @@ -2865,7 +2874,7 @@ class PrometheusLogger(CustomLogger): self.litellm_deployment_latency_per_output_token.labels(**_labels).observe(latency_per_token) except Exception as e: - verbose_logger.exception(f"Prometheus Error: set_llm_deployment_success_metrics. Exception occured - {e}") + verbose_logger.exception("Prometheus Error: set_llm_deployment_success_metrics. Exception occured - %s", e) return def _record_guardrail_metrics( @@ -2910,7 +2919,7 @@ class PrometheusLogger(CustomLogger): hook_type=hook_type, ).inc() except Exception as e: - verbose_logger.debug(f"Error recording guardrail metrics: {e}") + verbose_logger.debug("Error recording guardrail metrics: %s", e) ######################################## # Managed Batch Metric Recording Methods @@ -2933,7 +2942,7 @@ class PrometheusLogger(CustomLogger): api_key_alias=api_key_alias, ).inc() except Exception as e: - verbose_logger.warning(f"Error recording batch created metric: {e}") + verbose_logger.warning("Error recording batch created metric: %s", e) def record_managed_file_size( self, @@ -2954,7 +2963,7 @@ class PrometheusLogger(CustomLogger): user=user or "", ).set(size_bytes) except Exception as e: - verbose_logger.warning(f"Error recording file size metric: {e}") + verbose_logger.warning("Error recording file size metric: %s", e) def record_managed_batch_duration( self, @@ -2968,7 +2977,7 @@ class PrometheusLogger(CustomLogger): api_provider=api_provider or "", ).observe(duration_seconds) except Exception as e: - verbose_logger.warning(f"Error recording batch duration metric: {e}") + verbose_logger.warning("Error recording batch duration metric: %s", e) def record_managed_file_created( self, @@ -2987,14 +2996,14 @@ class PrometheusLogger(CustomLogger): api_key_alias=api_key_alias, ).inc() except Exception as e: - verbose_logger.warning(f"Error recording file created metric: {e}") + verbose_logger.warning("Error recording file created metric: %s", e) def record_managed_file_deleted(self, result: str): """Record a managed file deletion attempt. result is 'success' or 'blocked'.""" try: self.litellm_managed_file_deleted_total.labels(result=result).inc() except Exception as e: - verbose_logger.warning(f"Error recording file deleted metric: {e}") + verbose_logger.warning("Error recording file deleted metric: %s", e) def record_check_batch_cost_run( self, @@ -3021,7 +3030,7 @@ class PrometheusLogger(CustomLogger): api_provider=api_provider or "", ).inc() except Exception as e: - verbose_logger.warning(f"Error recording check batch cost metrics: {e}") + verbose_logger.warning("Error recording check batch cost metrics: %s", e) def record_check_batch_cost_error(self, error_type: str): try: @@ -3029,7 +3038,7 @@ class PrometheusLogger(CustomLogger): error_type=error_type, ).inc() except Exception as e: - verbose_logger.warning(f"Error recording check batch cost error metric: {e}") + verbose_logger.warning("Error recording check batch cost error metric: %s", e) @staticmethod def _get_exception_class_name(exception: Exception) -> str: @@ -3313,7 +3322,7 @@ class PrometheusLogger(CustomLogger): await set_metrics_function(data) except Exception as e: - verbose_logger.exception(f"Error initializing {data_type} budget metrics: {e}") + verbose_logger.exception("Error initializing %s budget metrics: %s", data_type, e) async def _initialize_team_budget_metrics(self): """ @@ -3493,18 +3502,18 @@ class PrometheusLogger(CustomLogger): # Get total user count total_users = await UserRepository(prisma_client).table.count() self.litellm_total_users_metric.set(total_users) - verbose_logger.debug(f"Prometheus: set litellm_total_users to {total_users}") + verbose_logger.debug("Prometheus: set litellm_total_users to %s", total_users) billable_users = await UserRepository(prisma_client).count_billable_users() self.litellm_active_users_metric.set(billable_users) - verbose_logger.debug(f"Prometheus: set litellm_active_users to {billable_users}") + verbose_logger.debug("Prometheus: set litellm_active_users to %s", billable_users) # Get total team count total_teams = await TeamRepository(prisma_client).table.count() self.litellm_teams_count_metric.set(total_teams) - verbose_logger.debug(f"Prometheus: set litellm_teams_count to {total_teams}") + verbose_logger.debug("Prometheus: set litellm_teams_count to %s", total_teams) except Exception as e: - verbose_logger.exception(f"Error initializing user/team count metrics: {e}") + verbose_logger.exception("Error initializing user/team count metrics: %s", e) async def _set_key_list_budget_metrics(self, keys: list[str | UserAPIKeyAuth]): """Helper function to set budget metrics for a list of keys""" @@ -3595,7 +3604,7 @@ class PrometheusLogger(CustomLogger): user_api_key_cache=user_api_key_cache, ) except Exception as e: - verbose_logger.debug(f"[Non-Blocking] Prometheus: Error getting team info: {e}") + verbose_logger.debug("[Non-Blocking] Prometheus: Error getting team info: %s", e) return team_object if team_info: @@ -3693,7 +3702,7 @@ class PrometheusLogger(CustomLogger): include_budget_table=True, ) except Exception as e: - verbose_logger.debug(f"[Non-Blocking] Prometheus: Error getting org info: {e}") + verbose_logger.debug("[Non-Blocking] Prometheus: Error getting org info: %s", e) return if org_info is None: @@ -3850,7 +3859,7 @@ class PrometheusLogger(CustomLogger): if key_object: user_api_key_dict.budget_reset_at = key_object.budget_reset_at except Exception as e: - verbose_logger.debug(f"[Non-Blocking] Prometheus: Error getting key info: {e}") + verbose_logger.debug("[Non-Blocking] Prometheus: Error getting key info: %s", e) return user_api_key_dict @@ -3915,7 +3924,7 @@ class PrometheusLogger(CustomLogger): check_db_only=False, ) except Exception as e: - verbose_logger.debug(f"[Non-Blocking] Prometheus: Error getting user info: {e}") + verbose_logger.debug("[Non-Blocking] Prometheus: Error getting user info: %s", e) return user_object if user_info: diff --git a/litellm/integrations/prometheus_services.py b/litellm/integrations/prometheus_services.py index 002d61265a4..446287441d7 100644 --- a/litellm/integrations/prometheus_services.py +++ b/litellm/integrations/prometheus_services.py @@ -92,7 +92,7 @@ class PrometheusServicesLogger: metrics = DEFAULT_SERVICE_CONFIGS.get(service, {}).get("metrics", []) if not metrics: - verbose_logger.debug(f"No metrics found for service {service}") + verbose_logger.debug("No metrics found for service %s", service) return DEFAULT_METRICS return metrics diff --git a/litellm/integrations/rubrik.py b/litellm/integrations/rubrik.py index 9942776bc00..e7fa1ca99b5 100644 --- a/litellm/integrations/rubrik.py +++ b/litellm/integrations/rubrik.py @@ -163,9 +163,9 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): parsed_rate = float(rbrk_sampling_rate.strip()) self.sampling_rate = max(0.0, min(1.0, parsed_rate)) if parsed_rate != self.sampling_rate: - verbose_logger.warning(f"RUBRIK_SAMPLING_RATE={parsed_rate} clamped to {self.sampling_rate}") + verbose_logger.warning("RUBRIK_SAMPLING_RATE=%s clamped to %s", parsed_rate, self.sampling_rate) except ValueError: - verbose_logger.warning(f"Invalid RUBRIK_SAMPLING_RATE: {rbrk_sampling_rate!r}, using 1.0") + verbose_logger.warning("Invalid RUBRIK_SAMPLING_RATE: %r, using 1.0", rbrk_sampling_rate) def _parse_batch_size(self) -> None: _batch_size = os.getenv("RUBRIK_BATCH_SIZE") @@ -173,11 +173,11 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): try: parsed_size = int(_batch_size) if parsed_size <= 0: - verbose_logger.warning(f"RUBRIK_BATCH_SIZE={_batch_size!r} must be > 0, using default") + verbose_logger.warning("RUBRIK_BATCH_SIZE=%r must be > 0, using default", _batch_size) else: self.batch_size = parsed_size except ValueError: - verbose_logger.warning(f"Invalid RUBRIK_BATCH_SIZE: {_batch_size!r}, using default") + verbose_logger.warning("Invalid RUBRIK_BATCH_SIZE: %r, using default", _batch_size) def _setup_clients(self, webhook_url: str) -> None: self.response_moderation_endpoint = f"{webhook_url}{_WEBHOOK_PATH_RESPONSE_MODERATION}" @@ -279,7 +279,9 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): return inputs except Exception as e: verbose_logger.error( - f"{label} hook failed: {e}. Returning original inputs unchanged.", + "%s hook failed: %s. Returning original inputs unchanged.", + label, + e, exc_info=True, ) return inputs @@ -388,8 +390,9 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): if logging_obj is None: verbose_logger.error( "Rubrik: moderation block fired with logging_obj=None for " - f"litellm_call_id={request_data.get('litellm_call_id')}; " - "cannot suppress success event or attach failure payload." + "litellm_call_id=%s; " + "cannot suppress success event or attach failure payload.", + request_data.get("litellm_call_id"), ) request_data["_rubrik_logging_obj"] = None return @@ -650,14 +653,15 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): payload["messages"] = (system_scaffold, messages) except Exception as e: verbose_logger.warning( - f"Rubrik: failed to prepend system prompt: {e}", + "Rubrik: failed to prepend system prompt: %s", + e, exc_info=True, ) async def _prepare_log_payload(self, kwargs: Mapping[str, Any], event_type: str) -> StandardLoggingPayload | None: """Shared logic for success logging (sampled).""" if random.random() > self.sampling_rate: - verbose_logger.debug(f"Skipping Rubrik {event_type} logging (sampling_rate={self.sampling_rate})") + verbose_logger.debug("Skipping Rubrik %s logging (sampling_rate=%s)", event_type, self.sampling_rate) return None # Deep-copy so mutations don't affect other callbacks sharing this object @@ -701,7 +705,9 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): await self._append_and_maybe_flush(payload) except Exception as e: verbose_logger.error( - f"Rubrik {event_type} logging hook failed: {e}. Skipping logging for this event.", + "Rubrik %s logging hook failed: %s. Skipping logging for this event.", + event_type, + e, exc_info=True, ) @@ -710,7 +716,8 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): # skip here to avoid double-logging the pre-block response. if kwargs.get("_rubrik_blocked"): verbose_logger.debug( - f"Rubrik: skipping success event for blocked request litellm_call_id={kwargs.get('litellm_call_id')}" + "Rubrik: skipping success event for blocked request litellm_call_id=%s", + kwargs.get("litellm_call_id"), ) return await self._enqueue_log_event(kwargs, "success") @@ -755,11 +762,14 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): # way we cannot build the payload. verbose_logger.warning( "Rubrik: block exception without stashed logging_obj. " - f"litellm_call_id={request_data.get('litellm_call_id')}, " - f"model={request_data.get('model')}, " - f"user_id={user_api_key_dict.user_id}, " - f"raising_guardrail=" - f"{getattr(original_exception, 'guardrail_name', None)}" + "litellm_call_id=%s, " + "model=%s, " + "user_id=%s, " + "raising_guardrail=%s", + request_data.get("litellm_call_id"), + request_data.get("model"), + user_api_key_dict.user_id, + getattr(original_exception, "guardrail_name", None), ) return @@ -786,8 +796,9 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): payload = self._prepare_block_failure_payload(logging_obj, exception, user_api_key_dict) except (AttributeError, ImportError, KeyError, TypeError) as e: verbose_logger.error( - f"Rubrik: failed to build blocked-tool payload for " - f"litellm_call_id={call_id}: {e}. Event will NOT be logged.", + "Rubrik: failed to build blocked-tool payload for litellm_call_id=%s: %s. Event will NOT be logged.", + call_id, + e, exc_info=True, ) return @@ -796,7 +807,9 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): await self._append_and_maybe_flush(payload) except Exception as e: verbose_logger.error( - f"Rubrik: failed to enqueue blocked-tool event for litellm_call_id={call_id}: {e}.", + "Rubrik: failed to enqueue blocked-tool event for litellm_call_id=%s: %s.", + call_id, + e, exc_info=True, ) @@ -853,8 +866,9 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): else: verbose_logger.debug( "Rubrik: standard_logging_object not yet on model_call_details " - f"for litellm_call_id={call_details.get('litellm_call_id')}; " - "using best-effort fallback payload." + "for litellm_call_id=%s; " + "using best-effort fallback payload.", + call_details.get("litellm_call_id"), ) payload = self._build_fallback_payload(call_details, user_api_key_dict) @@ -932,7 +946,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): ) response.raise_for_status() except httpx.HTTPStatusError as e: - verbose_logger.exception(f"Rubrik HTTP Error: {e.response.status_code} - {e.response.text}") + verbose_logger.exception("Rubrik HTTP Error: %s - %s", e.response.status_code, e.response.text) raise except Exception: verbose_logger.exception("Rubrik Layer Error") @@ -989,7 +1003,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): Exception: If the service is unavailable or returns an error. TypeError: If the response JSON is not a dict. """ - verbose_logger.debug(f"Sending request to {service_name}: {endpoint}") + verbose_logger.debug("Sending request to %s: %s", service_name, endpoint) http_response = await self.moderation_client.post( endpoint, json=payload, diff --git a/litellm/integrations/s3.py b/litellm/integrations/s3.py index c35cc88107f..a4bd488221c 100644 --- a/litellm/integrations/s3.py +++ b/litellm/integrations/s3.py @@ -31,7 +31,7 @@ class S3Logger: import boto3 try: - verbose_logger.debug(f"in init s3 logger - s3_callback_params {litellm.s3_callback_params}") + verbose_logger.debug("in init s3 logger - s3_callback_params %s", litellm.s3_callback_params) s3_use_team_prefix = False @@ -62,7 +62,7 @@ class S3Logger: self.s3_server_side_encryption, self.s3_sse_kms_key_id = resolve_sse_params( s3_server_side_encryption, s3_sse_kms_key_id ) - verbose_logger.debug(f"s3 logger using endpoint url {s3_endpoint_url}") + verbose_logger.debug("s3 logger using endpoint url %s", s3_endpoint_url) # Create an S3 client with custom endpoint URL self.s3_client = boto3.client( "s3", @@ -86,7 +86,7 @@ class S3Logger: def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose): try: - verbose_logger.debug(f"s3 Logging - Enters logging function for model {kwargs}") + verbose_logger.debug("s3 Logging - Enters logging function for model %s", kwargs) # construct payload to send to s3 # follows the same params as langfuse.py @@ -168,14 +168,14 @@ class S3Logger: print_verbose(f"s3 Layer Logging - final response object: {response_obj}") return response except Exception as e: - verbose_logger.exception(f"s3 Layer Error - {e}") + verbose_logger.exception("s3 Layer Error - %s", e) def _validated_sse_value(name: str, value: str | None) -> str | None: if value is None or isinstance(value, str): return value verbose_logger.warning( - f"s3 logging: ignoring {name} because it has invalid type {type(value).__name__}; expected a string" + "s3 logging: ignoring %s because it has invalid type %s; expected a string", name, type(value).__name__ ) return None @@ -191,8 +191,8 @@ def resolve_sse_params( return None, None if valid_key_id and not algorithm.startswith("aws:kms"): verbose_logger.warning( - f"s3 logging: ignoring s3_sse_kms_key_id because s3_server_side_encryption is {algorithm}; " - "set it to aws:kms to encrypt with the KMS key" + "s3 logging: ignoring s3_sse_kms_key_id because s3_server_side_encryption is %s; set it to aws:kms to encrypt with the KMS key", + algorithm, ) return algorithm, None return algorithm, valid_key_id diff --git a/litellm/integrations/s3_v2.py b/litellm/integrations/s3_v2.py index 44c6e42f9f0..c197b65696f 100644 --- a/litellm/integrations/s3_v2.py +++ b/litellm/integrations/s3_v2.py @@ -64,12 +64,12 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): _masker = SensitiveDataMasker() if s3_callback_params_override is not None: verbose_logger.debug( - f"in init s3 logger (audit override) - {_masker.mask_dict(dict(s3_callback_params_override))}" + "in init s3 logger (audit override) - %s", _masker.mask_dict(dict(s3_callback_params_override)) ) else: verbose_logger.debug( - f"in init s3 logger - s3_callback_params " - f"{_masker.mask_dict(dict(litellm.s3_callback_params or {}))}" + "in init s3 logger - s3_callback_params %s", + _masker.mask_dict(dict(litellm.s3_callback_params or {})), ) # Initialize S3 params first to get the correct s3_verify value @@ -98,11 +98,11 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): s3_server_side_encryption=s3_server_side_encryption, s3_sse_kms_key_id=s3_sse_kms_key_id, ) - verbose_logger.debug(f"s3 logger using endpoint url {s3_endpoint_url}") + verbose_logger.debug("s3 logger using endpoint url %s", s3_endpoint_url) # IMPORTANT # Create httpx client AFTER _init_s3_params so we have the correct s3_verify value - verbose_logger.debug(f"s3_v2 logger creating async httpx client with s3_verify={self.s3_verify}") + verbose_logger.debug("s3_v2 logger creating async httpx client with s3_verify=%s", self.s3_verify) self.async_httpx_client = get_async_httpx_client( llm_provider=httpxSpecialProvider.LoggingCallback, params={"ssl_verify": self.s3_verify}, @@ -111,7 +111,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): asyncio.create_task(self.periodic_flush()) self.flush_lock = asyncio.Lock() - verbose_logger.debug(f"s3 flush interval: {s3_flush_interval}, s3 batch size: {s3_batch_size}") + verbose_logger.debug("s3 flush interval: %s, s3 batch size: %s", s3_flush_interval, s3_batch_size) # Call CustomLogger's __init__ CustomBatchLogger.__init__( self, @@ -259,7 +259,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): async def _async_log_event_base(self, kwargs, response_obj, start_time, end_time): try: - verbose_logger.debug(f"s3 Logging - Enters logging function for model {kwargs}") + verbose_logger.debug("s3 Logging - Enters logging function for model %s", kwargs) s3_batch_logging_element = self.create_s3_batch_logging_element( start_time=start_time, @@ -284,7 +284,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): self.batch_size, ) except Exception as e: - verbose_logger.exception(f"s3 Layer Error - {e}") + verbose_logger.exception("s3 Layer Error - %s", e) self.handle_callback_failure(callback_name="S3Logger") async def async_upload_data_to_s3(self, batch_logging_element: s3BatchLoggingElement): @@ -313,8 +313,8 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): aws_sts_endpoint=self.s3_aws_sts_endpoint, ) - verbose_logger.debug(f"s3_v2 logger - uploading data to s3 - {batch_logging_element.s3_object_key}") - verbose_logger.debug(f"s3_v2 logger - s3_verify setting: {self.s3_verify}") + verbose_logger.debug("s3_v2 logger - uploading data to s3 - %s", batch_logging_element.s3_object_key) + verbose_logger.debug("s3_v2 logger - s3_verify setting: %s", self.s3_verify) # Prepare the URL url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}" @@ -374,16 +374,19 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): if response.status_code in (500, 503) and attempt < max_retries - 1: wait_time = 2**attempt # 1s, 2s verbose_logger.warning( - f"S3 upload returned {response.status_code}, retrying in {wait_time}s " - f"(attempt {attempt + 1}/{max_retries}) " - f"key={batch_logging_element.s3_object_key}" + "S3 upload returned %s, retrying in %ss (attempt %s/%s) key=%s", + response.status_code, + wait_time, + attempt + 1, + max_retries, + batch_logging_element.s3_object_key, ) await asyncio.sleep(wait_time) continue response.raise_for_status() break except Exception as e: - verbose_logger.exception(f"Error uploading to s3: {e}") + verbose_logger.exception("Error uploading to s3: %s", e) self.handle_callback_failure(callback_name="S3Logger") async def async_send_batch(self): @@ -395,7 +398,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): Raises: Does not raise an exception, will only verbose_logger.exception() """ - verbose_logger.debug(f"s3_v2 logger - sending batch of {len(self.log_queue)}") + verbose_logger.debug("s3_v2 logger - sending batch of %s", len(self.log_queue)) if not self.log_queue: return @@ -447,7 +450,10 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): s3_file_name = litellm.utils.get_logging_id(start_time, standard_logging_payload) or "" verbose_logger.debug( - f"Creating s3 file with prefix_components={prefix_components},prefix_path={prefix_path} and {s3_file_name}" + "Creating s3 file with prefix_components=%s,prefix_path=%s and %s", + prefix_components, + prefix_path, + s3_file_name, ) s3_object_key = get_s3_object_key( s3_path=cast(str | None, self.s3_path) or "", @@ -455,7 +461,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): start_time=start_time, s3_file_name=s3_file_name, ) - verbose_logger.debug(f"s3_object_key={s3_object_key}") + verbose_logger.debug("s3_object_key=%s", s3_object_key) s3_object_download_filename = ( f"time-{start_time.strftime('%Y-%m-%dT%H-%M-%S-%f')}_{standard_logging_payload['id']}.json" @@ -479,7 +485,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): except ImportError: raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") try: - verbose_logger.debug(f"s3_v2 logger - uploading data to s3 - {batch_logging_element.s3_object_key}") + verbose_logger.debug("s3_v2 logger - uploading data to s3 - %s", batch_logging_element.s3_object_key) credentials: Credentials = self.get_credentials( aws_access_key_id=self.s3_aws_access_key_id, aws_secret_access_key=self.s3_aws_secret_access_key, @@ -548,16 +554,19 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): if response.status_code in (500, 503) and attempt < max_retries - 1: wait_time = 2**attempt # 1s, 2s verbose_logger.warning( - f"S3 upload returned {response.status_code}, retrying in {wait_time}s " - f"(attempt {attempt + 1}/{max_retries}) " - f"key={batch_logging_element.s3_object_key}" + "S3 upload returned %s, retrying in %ss (attempt %s/%s) key=%s", + response.status_code, + wait_time, + attempt + 1, + max_retries, + batch_logging_element.s3_object_key, ) time.sleep(wait_time) continue response.raise_for_status() break except Exception as e: - verbose_logger.exception(f"Error uploading to s3: {e}") + verbose_logger.exception("Error uploading to s3: %s", e) self.handle_callback_failure(callback_name="S3Logger") async def _download_object_from_s3(self, s3_object_key: str) -> dict | None: @@ -596,7 +605,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): aws_sts_endpoint=self.s3_aws_sts_endpoint, ) - verbose_logger.debug(f"s3_v2 logger - downloading data from s3 - {s3_object_key}") + verbose_logger.debug("s3_v2 logger - downloading data from s3 - %s", s3_object_key) # Prepare the URL url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{s3_object_key}" @@ -642,7 +651,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): return response.json() except Exception as e: - verbose_logger.exception(f"Error downloading from S3: {e}") + verbose_logger.exception("Error downloading from S3: %s", e) return None async def get_proxy_server_request_from_cold_storage_with_object_key( @@ -666,5 +675,5 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): downloaded_object = await self._download_object_from_s3(object_key) return downloaded_object except Exception as e: - verbose_logger.exception(f"Error retrieving object {object_key} from cold storage: {e}") + verbose_logger.exception("Error retrieving object %s from cold storage: %s", object_key, e) return None diff --git a/litellm/integrations/sqs.py b/litellm/integrations/sqs.py index 56618b62368..267bc0def22 100644 --- a/litellm/integrations/sqs.py +++ b/litellm/integrations/sqs.py @@ -68,7 +68,7 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): **kwargs, ) -> None: try: - verbose_logger.debug(f"in init sqs logger - sqs_callback_params {litellm.aws_sqs_callback_params}") + verbose_logger.debug("in init sqs logger - sqs_callback_params %s", litellm.aws_sqs_callback_params) self.async_httpx_client = get_async_httpx_client( llm_provider=httpxSpecialProvider.LoggingCallback, @@ -100,7 +100,7 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): asyncio.create_task(self.periodic_flush()) self.flush_lock = asyncio.Lock() - verbose_logger.debug(f"sqs flush interval: {sqs_flush_interval}, sqs batch size: {sqs_batch_size}") + verbose_logger.debug("sqs flush interval: %s, sqs batch size: %s", sqs_flush_interval, sqs_batch_size) CustomBatchLogger.__init__( self, @@ -215,7 +215,7 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): self.batch_size, ) except Exception as e: - verbose_logger.exception(f"sqs Layer Error - {e}") + verbose_logger.exception("sqs Layer Error - %s", e) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): try: @@ -233,10 +233,10 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): ) except Exception as e: - verbose_logger.exception(f"Datadog Layer Error - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Datadog Layer Error - %s\n%s", e, traceback.format_exc()) async def async_send_batch(self) -> None: - verbose_logger.debug(f"sqs logger - sending batch of {len(self.log_queue)}") + verbose_logger.debug("sqs logger - sending batch of %s", len(self.log_queue)) if not self.log_queue: return @@ -305,7 +305,7 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): ) response.raise_for_status() except Exception as e: - verbose_logger.exception(f"Error sending to SQS: {e}") + verbose_logger.exception("Error sending to SQS: %s", e) async def async_health_check(self) -> IntegrationHealthCheckStatus: """ diff --git a/litellm/integrations/traceloop.py b/litellm/integrations/traceloop.py index 77f20972f7a..f5d28fd369f 100644 --- a/litellm/integrations/traceloop.py +++ b/litellm/integrations/traceloop.py @@ -15,7 +15,9 @@ class TraceloopLogger: from traceloop.sdk.tracing.tracing import TracerWrapper except ModuleNotFoundError as e: verbose_logger.error( - f"Traceloop not installed, try running 'pip install traceloop-sdk' to fix this error: {e}\n{traceback.format_exc()}" + "Traceloop not installed, try running 'pip install traceloop-sdk' to fix this error: %s\n%s", + e, + traceback.format_exc(), ) raise e diff --git a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py index 6eac7a27e73..2e533b488b6 100644 --- a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py +++ b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py @@ -124,7 +124,7 @@ class VectorStorePreCallHook(CustomLogger): }, ) - verbose_logger.debug(f"search_response: {search_response}") + verbose_logger.debug("search_response: %s", search_response) # Store search results for later use in citations all_search_results.append(search_response) @@ -137,7 +137,7 @@ class VectorStorePreCallHook(CustomLogger): # Get the number of results for logging num_results = 0 num_results = len(search_response.get("data", []) or []) - verbose_logger.debug(f"Vector store search completed. Added context from {num_results} results") + verbose_logger.debug("Vector store search completed. Added context from %s results", num_results) # Store search results as-is (already in OpenAI-compatible format) if litellm_logging_obj and all_search_results: @@ -146,7 +146,7 @@ class VectorStorePreCallHook(CustomLogger): return model, modified_messages, non_default_params except Exception as e: - verbose_logger.exception(f"Error in VectorStorePreCallHook: {e}") + verbose_logger.exception("Error in VectorStorePreCallHook: %s", e) # Return original parameters on error return model, messages, non_default_params @@ -243,14 +243,14 @@ class VectorStorePreCallHook(CustomLogger): verbose_logger.debug("No litellm_logging_obj in request_data") return None - verbose_logger.debug(f"model_call_details keys: {list(litellm_logging_obj.model_call_details.keys())}") + verbose_logger.debug("model_call_details keys: %s", list(litellm_logging_obj.model_call_details.keys())) # Get search results from model_call_details (already in OpenAI format) search_results: list[VectorStoreSearchResponse] | None = litellm_logging_obj.model_call_details.get( "search_results" ) - verbose_logger.debug(f"Search results found: {search_results is not None}") + verbose_logger.debug("Search results found: %s", search_results is not None) if not search_results: verbose_logger.debug("No search results found") @@ -269,13 +269,13 @@ class VectorStorePreCallHook(CustomLogger): # Set the provider_specific_fields setattr(choice.message, "provider_specific_fields", provider_fields) - verbose_logger.debug(f"Added {len(search_results)} search results to response") + verbose_logger.debug("Added %s search results to response", len(search_results)) # Return modified response return response except Exception as e: - verbose_logger.exception(f"Error adding search results to response: {e}") + verbose_logger.exception("Error adding search results to response: %s", e) # Don't fail the request if search results fail to be added return None @@ -297,7 +297,7 @@ class VectorStorePreCallHook(CustomLogger): # Get search results from model_call_details (already in OpenAI format) search_results: list[VectorStoreSearchResponse] | None = request_data.get("search_results") - verbose_logger.debug(f"Search results found for streaming chunk: {search_results is not None}") + verbose_logger.debug("Search results found for streaming chunk: %s", search_results is not None) if not search_results: verbose_logger.debug("No search results found for streaming chunk") @@ -316,12 +316,12 @@ class VectorStorePreCallHook(CustomLogger): # Set the provider_specific_fields choice.delta.provider_specific_fields = provider_fields - verbose_logger.debug(f"Added {len(search_results)} search results to streaming chunk") + verbose_logger.debug("Added %s search results to streaming chunk", len(search_results)) # Return modified chunk return response_chunk except Exception as e: - verbose_logger.exception(f"Error adding search results to streaming chunk: {e}") + verbose_logger.exception("Error adding search results to streaming chunk: %s", e) # Don't fail the request if search results fail to be added return response_chunk diff --git a/litellm/integrations/weave/weave_otel.py b/litellm/integrations/weave/weave_otel.py index 321dda2983d..ffa30582771 100644 --- a/litellm/integrations/weave/weave_otel.py +++ b/litellm/integrations/weave/weave_otel.py @@ -148,10 +148,10 @@ def get_weave_otel_config() -> WeaveOtelConfig: host = "https://" + host # Self-managed instances use a different path endpoint = host.rstrip("/") + WEAVE_OTEL_ENDPOINT - verbose_logger.debug(f"Using Weave OTEL endpoint from host: {endpoint}") + verbose_logger.debug("Using Weave OTEL endpoint from host: %s", endpoint) else: endpoint = WEAVE_BASE_URL + WEAVE_OTEL_ENDPOINT - verbose_logger.debug(f"Using Weave cloud endpoint: {endpoint}") + verbose_logger.debug("Using Weave cloud endpoint: %s", endpoint) # Weave uses Basic auth with format: api: auth_header = _get_weave_authorization_header(api_key=api_key) diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 718f7b8fcd7..effc53d1c08 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -155,8 +155,8 @@ class WebSearchInterceptionLogger(CustomLogger): ) if anthropic_config is not None and anthropic_config.handles_web_search_natively(): verbose_logger.debug( - f"WebSearchInterception: Skipping short-circuit for {provider_str} " - "(provider handles web search natively via the agentic loop)" + "WebSearchInterception: Skipping short-circuit for %s (provider handles web search natively via the agentic loop)", + provider_str, ) return None except (ValueError, Exception): @@ -176,7 +176,7 @@ class WebSearchInterceptionLogger(CustomLogger): return None verbose_logger.debug( - f"WebSearchInterception: Short-circuit search detected (provider={provider_str}, query='{query}')" + "WebSearchInterception: Short-circuit search detected (provider=%s, query='%s')", provider_str, query ) # Native clients (Claude Desktop / Cowork / Anthropic SDK) make a @@ -198,7 +198,7 @@ class WebSearchInterceptionLogger(CustomLogger): else: search_result_text, structured = await self._execute_search(query, kwargs=kwargs) except Exception as e: - verbose_logger.error(f"WebSearchInterception: Short-circuit search failed: {e}") + verbose_logger.error("WebSearchInterception: Short-circuit search failed: %s", e) search_result_text, structured = f"Search failed: {e}", None content: list[dict[str, object]] = [] @@ -235,9 +235,9 @@ class WebSearchInterceptionLogger(CustomLogger): } verbose_logger.debug( - "WebSearchInterception: Short-circuit search completed, " - f"returning synthetic response ({len(search_result_text)} chars, " - f"native_blocks={native_tool is not None})" + "WebSearchInterception: Short-circuit search completed, returning synthetic response (%s chars, native_blocks=%s)", + len(search_result_text), + native_tool is not None, ) return response @@ -294,8 +294,10 @@ class WebSearchInterceptionLogger(CustomLogger): converted_tool = get_litellm_web_search_tool_openai() converted_tools.append(converted_tool) verbose_logger.debug( - f"WebSearchInterception: Converted {tool.get('name', 'unknown')} " - f"(type={tool.get('type', 'none')}) to {LITELLM_WEB_SEARCH_TOOL_NAME}" + "WebSearchInterception: Converted %s (type=%s) to %s", + tool.get("name", "unknown"), + tool.get("type", "none"), + LITELLM_WEB_SEARCH_TOOL_NAME, ) else: # Keep other tools as-is @@ -419,14 +421,14 @@ class WebSearchInterceptionLogger(CustomLogger): custom_llm_provider = kwargs.get("litellm_params", {}).get("custom_llm_provider", "") verbose_logger.debug( - f"WebSearchInterception: Pre-request hook called" - f" - custom_llm_provider={custom_llm_provider}" - f" - enabled_providers={self.enabled_providers or 'ALL'}" + "WebSearchInterception: Pre-request hook called - custom_llm_provider=%s - enabled_providers=%s", + custom_llm_provider, + self.enabled_providers or "ALL", ) if self.enabled_providers is not None and custom_llm_provider not in self.enabled_providers: verbose_logger.debug( - f"WebSearchInterception: Skipping - provider {custom_llm_provider} not in {self.enabled_providers}" + "WebSearchInterception: Skipping - provider %s not in %s", custom_llm_provider, self.enabled_providers ) return None @@ -440,7 +442,7 @@ class WebSearchInterceptionLogger(CustomLogger): if not has_websearch: return None - verbose_logger.debug(f"WebSearchInterception: Pre-request hook triggered for provider={custom_llm_provider}") + verbose_logger.debug("WebSearchInterception: Pre-request hook triggered for provider=%s", custom_llm_provider) # If the client sent an Anthropic-native web_search_* tool, mark the # request so the agentic loop emits native web_search_tool_result @@ -457,15 +459,17 @@ class WebSearchInterceptionLogger(CustomLogger): standard_tool = get_litellm_web_search_tool() converted_tools.append(standard_tool) verbose_logger.debug( - f"WebSearchInterception: Converted {tool.get('name', 'unknown')} " - f"(type={tool.get('type', 'none')}) to {LITELLM_WEB_SEARCH_TOOL_NAME}" + "WebSearchInterception: Converted %s (type=%s) to %s", + tool.get("name", "unknown"), + tool.get("type", "none"), + LITELLM_WEB_SEARCH_TOOL_NAME, ) else: converted_tools.append(tool) kwargs["tools"] = converted_tools verbose_logger.debug( - f"WebSearchInterception: Tools after conversion: {[t.get('name') for t in converted_tools]}" + "WebSearchInterception: Tools after conversion: %s", [t.get("name") for t in converted_tools] ) if "tool_choice" in kwargs: @@ -511,15 +515,17 @@ class WebSearchInterceptionLogger(CustomLogger): kwargs=kwargs, ) - verbose_logger.debug(f"WebSearchInterception: Hook called! provider={custom_llm_provider}, stream={stream}") - verbose_logger.debug(f"WebSearchInterception: Response type: {type(response)}") + verbose_logger.debug("WebSearchInterception: Hook called! provider=%s, stream=%s", custom_llm_provider, stream) + verbose_logger.debug("WebSearchInterception: Response type: %s", type(response)) # Check if provider should be intercepted # Note: custom_llm_provider is already normalized by get_llm_provider() # (e.g., "bedrock/invoke/..." -> "bedrock") if self.enabled_providers is not None and custom_llm_provider not in self.enabled_providers: verbose_logger.debug( - f"WebSearchInterception: Skipping provider {custom_llm_provider} (not in enabled list: {self.enabled_providers})" + "WebSearchInterception: Skipping provider %s (not in enabled list: %s)", + custom_llm_provider, + self.enabled_providers, ) return False, {} @@ -541,7 +547,7 @@ class WebSearchInterceptionLogger(CustomLogger): return False, {} verbose_logger.debug( - f"WebSearchInterception: Detected {len(tool_calls)} WebSearch tool call(s), executing agentic loop" + "WebSearchInterception: Detected %s WebSearch tool call(s), executing agentic loop", len(tool_calls) ) # Extract thinking blocks from response content. @@ -576,7 +582,7 @@ class WebSearchInterceptionLogger(CustomLogger): if thinking_blocks: verbose_logger.debug( - f"WebSearchInterception: Extracted {len(thinking_blocks)} thinking block(s) from response" + "WebSearchInterception: Extracted %s thinking block(s) from response", len(thinking_blocks) ) # Return tools dict with tool calls and thinking blocks @@ -606,14 +612,16 @@ class WebSearchInterceptionLogger(CustomLogger): """ verbose_logger.debug( - f"WebSearchInterception: Chat completion hook called! provider={custom_llm_provider}, stream={stream}" + "WebSearchInterception: Chat completion hook called! provider=%s, stream=%s", custom_llm_provider, stream ) - verbose_logger.debug(f"WebSearchInterception: Response type: {type(response)}") + verbose_logger.debug("WebSearchInterception: Response type: %s", type(response)) # Check if provider should be intercepted if self.enabled_providers is not None and custom_llm_provider not in self.enabled_providers: verbose_logger.debug( - f"WebSearchInterception: Skipping provider {custom_llm_provider} (not in enabled list: {self.enabled_providers})" + "WebSearchInterception: Skipping provider %s (not in enabled list: %s)", + custom_llm_provider, + self.enabled_providers, ) return False, {} @@ -635,7 +643,7 @@ class WebSearchInterceptionLogger(CustomLogger): return False, {} verbose_logger.debug( - f"WebSearchInterception: Detected {len(tool_calls)} WebSearch tool call(s), executing agentic loop" + "WebSearchInterception: Detected %s WebSearch tool call(s), executing agentic loop", len(tool_calls) ) # Return tools dict with tool calls @@ -659,12 +667,14 @@ class WebSearchInterceptionLogger(CustomLogger): ) -> tuple[bool, dict]: """Check if WebSearch interception is needed for the Responses API.""" verbose_logger.debug( - f"WebSearchInterception: Responses hook called! provider={custom_llm_provider}, stream={stream}" + "WebSearchInterception: Responses hook called! provider=%s, stream=%s", custom_llm_provider, stream ) if self.enabled_providers is not None and custom_llm_provider not in self.enabled_providers: verbose_logger.debug( - f"WebSearchInterception: Skipping provider {custom_llm_provider} (not in enabled list: {self.enabled_providers})" + "WebSearchInterception: Skipping provider %s (not in enabled list: %s)", + custom_llm_provider, + self.enabled_providers, ) return False, {} @@ -684,7 +694,7 @@ class WebSearchInterceptionLogger(CustomLogger): return False, {} verbose_logger.debug( - f"WebSearchInterception: Detected {len(tool_calls)} WebSearch function_call(s), executing agentic loop" + "WebSearchInterception: Detected %s WebSearch function_call(s), executing agentic loop", len(tool_calls) ) tools_dict = { @@ -716,7 +726,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls = tools["tool_calls"] thinking_blocks = tools.get("thinking_blocks", []) - verbose_logger.debug(f"WebSearchInterception: Executing agentic loop for {len(tool_calls)} search(es)") + verbose_logger.debug("WebSearchInterception: Executing agentic loop for %s search(es)", len(tool_calls)) return await self._execute_agentic_loop( model=model, @@ -853,7 +863,8 @@ class WebSearchInterceptionLogger(CustomLogger): # Object refused write — fall through and leave the response # untouched rather than crash the request. verbose_logger.debug( - f"WebSearchInterception: could not inject native blocks into response of type {type(response).__name__}" + "WebSearchInterception: could not inject native blocks into response of type %s", + type(response).__name__, ) return response @@ -878,7 +889,7 @@ class WebSearchInterceptionLogger(CustomLogger): response_format = tools.get("response_format", "openai") verbose_logger.debug( - f"WebSearchInterception: Executing chat completion agentic loop for {len(tool_calls)} search(es)" + "WebSearchInterception: Executing chat completion agentic loop for %s search(es)", len(tool_calls) ) return await self._execute_chat_completion_agentic_loop( @@ -962,7 +973,7 @@ class WebSearchInterceptionLogger(CustomLogger): for tool_call in tool_calls ] - verbose_logger.debug(f"WebSearchInterception: Executing {len(search_tasks)} responses search(es) in parallel") + verbose_logger.debug("WebSearchInterception: Executing %s responses search(es) in parallel", len(search_tasks)) search_results = await asyncio.gather(*search_tasks, return_exceptions=True) search_texts = [self._extract_search_text(result) for result in search_results] @@ -1038,12 +1049,12 @@ class WebSearchInterceptionLogger(CustomLogger): @staticmethod def _extract_search_text(result: object) -> str: if isinstance(result, Exception): - verbose_logger.error(f"WebSearchInterception: Responses search failed with error: {result}") + verbose_logger.error("WebSearchInterception: Responses search failed with error: %s", result) return f"Search failed: {result}" if isinstance(result, tuple) and len(result) == 2: text_value, _ = result return text_value if isinstance(text_value, str) else str(text_value) - verbose_logger.debug(f"WebSearchInterception: Unexpected search result type {type(result)}") + verbose_logger.debug("WebSearchInterception: Unexpected search result type %s", type(result)) return str(result) @staticmethod @@ -1176,15 +1187,15 @@ class WebSearchInterceptionLogger(CustomLogger): for tool_call in tool_calls: query = tool_call["input"].get("query") if query: - verbose_logger.debug(f"WebSearchInterception: Queuing search for query='{query}'") + verbose_logger.debug("WebSearchInterception: Queuing search for query='%s'", query) search_tasks.append(self._execute_search(query, kwargs=kwargs)) else: - verbose_logger.debug(f"WebSearchInterception: Tool call {tool_call['id']} has no query") + verbose_logger.debug("WebSearchInterception: Tool call %s has no query", tool_call["id"]) # Add empty result for tools without query search_tasks.append(self._create_empty_search_result()) # Execute searches in parallel - verbose_logger.debug(f"WebSearchInterception: Executing {len(search_tasks)} search(es) in parallel") + verbose_logger.debug("WebSearchInterception: Executing %s search(es) in parallel", len(search_tasks)) search_results = await asyncio.gather(*search_tasks, return_exceptions=True) # Split the gathered (text, structured) tuples into two parallel lists. @@ -1194,7 +1205,7 @@ class WebSearchInterceptionLogger(CustomLogger): structured_results: list[SearchResponse | None] = [] for i, result in enumerate(search_results): if isinstance(result, Exception): - verbose_logger.error(f"WebSearchInterception: Search {i} failed with error: {result}") + verbose_logger.error("WebSearchInterception: Search %s failed with error: %s", i, result) final_search_results.append(f"Search failed: {result}") structured_results.append(None) elif isinstance(result, tuple) and len(result) == 2: @@ -1204,7 +1215,7 @@ class WebSearchInterceptionLogger(CustomLogger): else: # Defensive: legacy callers / unexpected shape — preserve text, # drop structure. - verbose_logger.debug(f"WebSearchInterception: Unexpected result type {type(result)} at index {i}") + verbose_logger.debug("WebSearchInterception: Unexpected result type %s at index %s", type(result), i) final_search_results.append(str(result)) structured_results.append(None) @@ -1224,7 +1235,7 @@ class WebSearchInterceptionLogger(CustomLogger): max_tokens = self._resolve_max_tokens(anthropic_messages_optional_request_params, kwargs) - verbose_logger.debug(f"WebSearchInterception: Using max_tokens={max_tokens} for follow-up request") + verbose_logger.debug("WebSearchInterception: Using max_tokens=%s for follow-up request", max_tokens) optional_params_without_max_tokens = { k: v for k, v in anthropic_messages_optional_request_params.items() if k != "max_tokens" @@ -1286,12 +1297,12 @@ class WebSearchInterceptionLogger(CustomLogger): if not search_provider: search_provider = "perplexity" verbose_logger.debug( - "WebSearchInterception: No search tools configured in router, " - f"using default provider '{search_provider}'" + "WebSearchInterception: No search tools configured in router, using default provider '%s'", + search_provider, ) verbose_logger.debug( - f"WebSearchInterception: Executing search for '{query}' using provider '{search_provider}'" + "WebSearchInterception: Executing search for '%s' using provider '%s'", query, search_provider ) search_kwargs = { key: value @@ -1304,11 +1315,11 @@ class WebSearchInterceptionLogger(CustomLogger): search_result_text = WebSearchTransformation.format_search_response(result) verbose_logger.debug( - f"WebSearchInterception: Search completed for '{query}', got {len(search_result_text)} chars" + "WebSearchInterception: Search completed for '%s', got %s chars", query, len(search_result_text) ) return search_result_text, result except Exception as e: - verbose_logger.error(f"WebSearchInterception: Search failed for '{query}': {e}") + verbose_logger.error("WebSearchInterception: Search failed for '%s': %s", query, e) raise async def _authorize_search_tool( @@ -1392,21 +1403,25 @@ class WebSearchInterceptionLogger(CustomLogger): if matching_tools: search_provider = (matching_tools[0].get("litellm_params", {}) or {}).get("search_provider") verbose_logger.debug( - f"WebSearchInterception: Found search tool '{self.search_tool_name}' " - f"from {source} with provider '{search_provider}'" + "WebSearchInterception: Found search tool '%s' from %s with provider '%s'", + self.search_tool_name, + source, + search_provider, ) return matching_tools[0] verbose_logger.debug( - f"WebSearchInterception: Search tool '{self.search_tool_name}' not found in {source}, " - "falling back to first available or perplexity" + "WebSearchInterception: Search tool '%s' not found in %s, falling back to first available or perplexity", + self.search_tool_name, + source, ) if search_tools: first_tool = search_tools[0] search_provider = (first_tool.get("litellm_params", {}) or {}).get("search_provider") verbose_logger.debug( - f"WebSearchInterception: Using first available search tool from {source} " - f"with provider '{search_provider}'" + "WebSearchInterception: Using first available search tool from %s with provider '%s'", + source, + search_provider, ) return first_tool @@ -1470,15 +1485,15 @@ class WebSearchInterceptionLogger(CustomLogger): query = args.get("query") if query: - verbose_logger.debug(f"WebSearchInterception: Queuing search for query='{query}'") + verbose_logger.debug("WebSearchInterception: Queuing search for query='%s'", query) search_tasks.append(self._execute_search(query, kwargs=kwargs)) else: - verbose_logger.debug(f"WebSearchInterception: Tool call {tool_call.get('id')} has no query") + verbose_logger.debug("WebSearchInterception: Tool call %s has no query", tool_call.get("id")) # Add empty result for tools without query search_tasks.append(self._create_empty_search_result()) # Execute searches in parallel - verbose_logger.debug(f"WebSearchInterception: Executing {len(search_tasks)} search(es) in parallel") + verbose_logger.debug("WebSearchInterception: Executing %s search(es) in parallel", len(search_tasks)) search_results = await asyncio.gather(*search_tasks, return_exceptions=True) # Chat-completion path only needs text — OpenAI tool_result format @@ -1486,13 +1501,13 @@ class WebSearchInterceptionLogger(CustomLogger): final_search_results: list[str] = [] for i, result in enumerate(search_results): if isinstance(result, Exception): - verbose_logger.error(f"WebSearchInterception: Search {i} failed with error: {result}") + verbose_logger.error("WebSearchInterception: Search %s failed with error: %s", i, result) final_search_results.append(f"Search failed: {result}") elif isinstance(result, tuple) and len(result) == 2: text_value, _ = result final_search_results.append(cast(str, text_value) if isinstance(text_value, str) else str(text_value)) else: - verbose_logger.debug(f"WebSearchInterception: Unexpected result type {type(result)} at index {i}") + verbose_logger.debug("WebSearchInterception: Unexpected result type %s at index %s", type(result), i) final_search_results.append(str(result)) # Build assistant and tool messages using transformation @@ -1517,7 +1532,7 @@ class WebSearchInterceptionLogger(CustomLogger): ] verbose_logger.debug("WebSearchInterception: Making follow-up chat completion request with search results") - verbose_logger.debug(f"WebSearchInterception: Follow-up messages count: {len(follow_up_messages)}") + verbose_logger.debug("WebSearchInterception: Follow-up messages count: %s", len(follow_up_messages)) # Remove internal parameters that shouldn't be passed to follow-up request internal_params = { diff --git a/litellm/integrations/websearch_interception/transformation.py b/litellm/integrations/websearch_interception/transformation.py index 9dd0c155142..3b9683366be 100644 --- a/litellm/integrations/websearch_interception/transformation.py +++ b/litellm/integrations/websearch_interception/transformation.py @@ -103,7 +103,7 @@ class WebSearchTransformation: parsed_input = json.loads(arguments) if arguments else {} except json.JSONDecodeError: verbose_logger.warning( - f"WebSearchInterception: Failed to parse function_call arguments: {arguments}" + "WebSearchInterception: Failed to parse function_call arguments: %s", arguments ) parsed_input = {} elif isinstance(arguments, dict): @@ -122,7 +122,7 @@ class WebSearchTransformation: "input": parsed_input, } ) - verbose_logger.debug(f"WebSearchInterception: Found {item_name} function_call with call_id={call_id}") + verbose_logger.debug("WebSearchInterception: Found %s function_call with call_id=%s", item_name, call_id) return len(tool_calls) > 0, tool_calls @@ -178,7 +178,7 @@ class WebSearchTransformation: "input": block_input, } tool_calls.append(tool_call) - verbose_logger.debug(f"WebSearchInterception: Found {block_name} tool_use with id={tool_call['id']}") + verbose_logger.debug("WebSearchInterception: Found %s tool_use with id=%s", block_name, tool_call["id"]) return len(tool_calls) > 0, tool_calls @@ -255,7 +255,7 @@ class WebSearchTransformation: arguments = json.loads(function_arguments) except json.JSONDecodeError: verbose_logger.warning( - f"WebSearchInterception: Failed to parse function arguments: {function_arguments}" + "WebSearchInterception: Failed to parse function arguments: %s", function_arguments ) arguments = {} else: @@ -273,7 +273,7 @@ class WebSearchTransformation: "input": arguments, # For compatibility with Anthropic format } tool_calls.append(tool_call_dict) - verbose_logger.debug(f"WebSearchInterception: Found {function_name} tool_call with id={tool_id}") + verbose_logger.debug("WebSearchInterception: Found %s tool_call with id=%s", function_name, tool_id) return len(tool_calls) > 0, tool_calls diff --git a/litellm/integrations/weights_biases.py b/litellm/integrations/weights_biases.py index 0fe2a70ab66..0c1d28c23ef 100644 --- a/litellm/integrations/weights_biases.py +++ b/litellm/integrations/weights_biases.py @@ -42,9 +42,9 @@ try: elif response["object"] == "chat.completion": return self._resolve_chat_completion(request, response, time_elapsed) else: - logger.debug(f"Unknown OpenAI response object: {response['object']}") + logger.debug("Unknown OpenAI response object: %s", response["object"]) except Exception as e: - logger.warning(f"Failed to resolve request/response: {e}") + logger.warning("Failed to resolve request/response: %s", e) return None @staticmethod diff --git a/litellm/interactions/streaming_iterator.py b/litellm/interactions/streaming_iterator.py index f8ac4f25ebc..0fa6a5a7f75 100644 --- a/litellm/interactions/streaming_iterator.py +++ b/litellm/interactions/streaming_iterator.py @@ -109,7 +109,7 @@ class BaseInteractionsAPIStreamingIterator: return None except json.JSONDecodeError: # If we can't parse the chunk, continue - verbose_logger.debug(f"Failed to parse streaming chunk: {stripped_chunk[:200]}...") + verbose_logger.debug("Failed to parse streaming chunk: %s...", stripped_chunk[:200]) return None def _handle_logging_completed_response(self): diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 101cbae23f9..75e9524e3f1 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -536,7 +536,7 @@ def _map_anthropic_exception( llm_provider="anthropic", ) if hasattr(original_exception, "status_code"): - verbose_logger.debug(f"status_code: {original_exception.status_code}") + verbose_logger.debug("status_code: %s", original_exception.status_code) if original_exception.status_code == 401: raise AuthenticationError( message=f"AnthropicException - {error_str}", @@ -1752,7 +1752,7 @@ def _map_aleph_alpha_exception( response=getattr(original_exception, "response", None), ) elif hasattr(original_exception, "status_code"): - verbose_logger.debug(f"status code: {original_exception.status_code}") + verbose_logger.debug("status code: %s", original_exception.status_code) if original_exception.status_code == 401: raise AuthenticationError( message=f"AlephAlphaException - {original_exception.message}", @@ -2526,7 +2526,9 @@ def exception_logging( model_call_details["exception"] = exception model_call_details["additional_args"] = additional_args # User Logging -> if you pass in a custom logging function or want to use sentry breadcrumbs - verbose_logger.debug(f"Logging Details: logger_fn - {logger_fn} | callable(logger_fn) - {callable(logger_fn)}") + verbose_logger.debug( + "Logging Details: logger_fn - %s | callable(logger_fn) - %s", logger_fn, callable(logger_fn) + ) if logger_fn and callable(logger_fn): try: logger_fn( @@ -2534,11 +2536,11 @@ def exception_logging( ) # Expectation: any logger function passed in by the user should accept a dict object except Exception: verbose_logger.debug( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {traceback.format_exc()}" + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", traceback.format_exc() ) except Exception: verbose_logger.debug( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {traceback.format_exc()}" + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", traceback.format_exc() ) diff --git a/litellm/litellm_core_utils/fallback_utils.py b/litellm/litellm_core_utils/fallback_utils.py index 4e7ce828a58..a40970a1234 100644 --- a/litellm/litellm_core_utils/fallback_utils.py +++ b/litellm/litellm_core_utils/fallback_utils.py @@ -70,7 +70,7 @@ async def async_completion_with_fallbacks(**kwargs): ) except Exception as e: - verbose_logger.exception(f"Fallback attempt failed for model {model}: {e}") + verbose_logger.exception("Fallback attempt failed for model %s: %s", model, e) most_recent_exception_str = str(e) continue diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index f38b2859259..6b05591fb85 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -202,7 +202,7 @@ try: EnterpriseStandardLoggingPayloadSetup ) except Exception as e: - verbose_logger.debug(f"[Non-Blocking] Unable to import GenericAPILogger - LiteLLM Enterprise Feature - {e}") + verbose_logger.debug("[Non-Blocking] Unable to import GenericAPILogger - LiteLLM Enterprise Feature - %s", e) GenericAPILogger = CustomLogger # type: ignore ResendEmailLogger = CustomLogger # type: ignore SendGridEmailLogger = CustomLogger # type: ignore @@ -546,7 +546,7 @@ class Logging(LiteLLMLoggingBaseClass): self.litellm_request_debug = litellm_params.get("litellm_request_debug", False) self.logger_fn = litellm_params.get("logger_fn", None) if _is_debugging_on() or self.litellm_request_debug: - verbose_logger.debug(f"self.optional_params: {self.optional_params}") + verbose_logger.debug("self.optional_params: %s", self.optional_params) self.model_call_details.update( { @@ -981,7 +981,7 @@ class Logging(LiteLLMLoggingBaseClass): ) # Expectation: any logger function passed in by the user should accept a dict object except Exception as e: verbose_logger.exception( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {e}" + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e ) self.model_call_details["api_call_start_time"] = datetime.datetime.now() @@ -1001,7 +1001,7 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.debug("reaches supabase for logging!") model = self.model_call_details["model"] messages = self.model_call_details["input"] - verbose_logger.debug(f"supabaseClient: {supabaseClient}") + verbose_logger.debug("supabaseClient: %s", supabaseClient) supabaseClient.input_log_event( model=model, messages=messages, @@ -1041,15 +1041,15 @@ class Logging(LiteLLMLoggingBaseClass): callback_func=callback, ) except Exception as e: - verbose_logger.exception(f"litellm.Logging.pre_call(): Exception occured - {e}") + verbose_logger.exception("litellm.Logging.pre_call(): Exception occured - %s", e) verbose_logger.debug( - f"LiteLLM.Logging: is sentry capture exception initialized {capture_exception}" + "LiteLLM.Logging: is sentry capture exception initialized %s", capture_exception ) if capture_exception: # log this error to sentry for debugging capture_exception(e) except Exception as e: - verbose_logger.exception(f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {e}") - verbose_logger.error(f"LiteLLM.Logging: is sentry capture exception initialized {capture_exception}") + verbose_logger.exception("LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e) + verbose_logger.error("LiteLLM.Logging: is sentry capture exception initialized %s", capture_exception) if capture_exception: # log this error to sentry for debugging capture_exception(e) @@ -1091,10 +1091,10 @@ class Logging(LiteLLMLoggingBaseClass): ) if self.litellm_request_debug: verbose_logger.warning( - f"\033[92m{curl_command}\033[0m\n" + "\x1b[92m%s\x1b[0m\n", curl_command ) # .warning ensures this shows up in all environments else: - verbose_logger.debug(f"\033[92m{curl_command}\033[0m\n") + verbose_logger.debug("\x1b[92m%s\x1b[0m\n", curl_command) def _get_request_body(self, data: dict) -> str: return str(data) @@ -1164,7 +1164,7 @@ class Logging(LiteLLMLoggingBaseClass): ) # Expectation: any logger function passed in by the user should accept a dict object except Exception as e: verbose_logger.exception( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {e}" + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e ) original_response = redact_message_input_output_from_logging( model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), @@ -1201,15 +1201,16 @@ class Logging(LiteLLMLoggingBaseClass): ) except Exception as e: verbose_logger.exception( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while post-call logging with integrations {e}" + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while post-call logging with integrations %s", + e, ) verbose_logger.debug( - f"LiteLLM.Logging: is sentry capture exception initialized {capture_exception}" + "LiteLLM.Logging: is sentry capture exception initialized %s", capture_exception ) if capture_exception: # log this error to sentry for debugging capture_exception(e) except Exception as e: - verbose_logger.exception(f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {e}") + verbose_logger.exception("LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e) async def async_post_mcp_tool_call_hook( self, @@ -1249,7 +1250,7 @@ class Logging(LiteLLMLoggingBaseClass): if response is not None: response_obj = self._parse_post_mcp_call_hook_response(response=response) except Exception as e: - verbose_logger.exception(f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {e}") + verbose_logger.exception("LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e) return response_obj def _parse_post_mcp_call_hook_response(self, response: MCPPostCallResponseObject | None) -> Any: @@ -1439,14 +1440,14 @@ class Logging(LiteLLMLoggingBaseClass): error_str=str(e), traceback_str=_get_traceback_str_for_error(str(e)), ) - verbose_logger.debug(f"response_cost_failure_debug_information: {debug_info}") + verbose_logger.debug("response_cost_failure_debug_information: %s", debug_info) self.model_call_details["response_cost_failure_debug_information"] = debug_info return None try: response_cost = litellm.response_cost_calculator(**response_cost_calculator_kwargs) - verbose_logger.debug(f"response_cost: {response_cost}") + verbose_logger.debug("response_cost: %s", response_cost) additional_response_cost: object = self.model_call_details.get("additional_response_cost") if isinstance(additional_response_cost, (int, float)) and additional_response_cost > 0: return (response_cost or 0.0) + additional_response_cost @@ -1462,7 +1463,7 @@ class Logging(LiteLLMLoggingBaseClass): call_type=response_cost_calculator_kwargs["call_type"], custom_pricing=response_cost_calculator_kwargs["custom_pricing"], ) - verbose_logger.debug(f"response_cost_failure_debug_information: {debug_info}") + verbose_logger.debug("response_cost_failure_debug_information: %s", debug_info) self.model_call_details["response_cost_failure_debug_information"] = debug_info return None @@ -1497,7 +1498,7 @@ class Logging(LiteLLMLoggingBaseClass): raw_response=httpx.Response(status_code=200, headers={}), ) except Exception as e: # noqa: BLE001 - cost normalization must never break the response path - verbose_logger.debug(f"generate_content response cost normalization failed: {e}") + verbose_logger.debug("generate_content response cost normalization failed: %s", e) return None async def _response_cost_calculator_async( @@ -1666,7 +1667,7 @@ class Logging(LiteLLMLoggingBaseClass): # proxy cost tracking cal backs should run if not (isinstance(callback, CustomLogger) and "_PROXY_" in callback.__class__.__name__): - verbose_logger.debug(f"no-log request, skipping logging for {event_hook} event") + verbose_logger.debug("no-log request, skipping logging for %s event", event_hook) return False # Check for dynamically disabled callbacks via headers @@ -1676,7 +1677,7 @@ class Logging(LiteLLMLoggingBaseClass): standard_callback_dynamic_params=self.standard_callback_dynamic_params, ): verbose_logger.debug( - f"Callback {callback} disabled via x-litellm-disable-callbacks header for {event_hook} event" + "Callback %s disabled via x-litellm-disable-callbacks header for %s event", callback, event_hook ) return False @@ -1989,7 +1990,7 @@ class Logging(LiteLLMLoggingBaseClass): await self.async_success_handler(result=complete_streaming_response) def success_handler(self, result=None, start_time=None, end_time=None, cache_hit=None, **kwargs): - verbose_logger.debug(f"Logging Details LiteLLM-Success Call: Cache_hit={cache_hit}") + verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit) if not self.should_run_logging(event_type="sync_success"): # prevent double logging return start_time, end_time, result = self._success_handler_helper_fn( @@ -2210,7 +2211,8 @@ class Logging(LiteLLMLoggingBaseClass): # this only logs streaming once, complete_streaming_response exists i.e when stream ends if self.stream: verbose_logger.debug( - f"is complete_streaming_response in kwargs: {kwargs.get('complete_streaming_response', None)}" + "is complete_streaming_response in kwargs: %s", + kwargs.get("complete_streaming_response", None), ) if complete_streaming_response is None: continue @@ -2247,7 +2249,8 @@ class Logging(LiteLLMLoggingBaseClass): # this only logs streaming once, complete_streaming_response exists i.e when stream ends if self.stream: verbose_logger.debug( - f"is complete_streaming_response in kwargs: {kwargs.get('complete_streaming_response', None)}" + "is complete_streaming_response in kwargs: %s", + kwargs.get("complete_streaming_response", None), ) if complete_streaming_response is None: continue @@ -2389,7 +2392,8 @@ class Logging(LiteLLMLoggingBaseClass): pass except Exception as e: verbose_logger.exception( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging {e}", + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging %s", + e, ) async def async_success_handler(self, result=None, start_time=None, end_time=None, cache_hit=None, **kwargs): @@ -2483,10 +2487,10 @@ class Logging(LiteLLMLoggingBaseClass): result=complete_streaming_response ) - verbose_logger.debug(f"Model={self.model}; cost={self.model_call_details['response_cost']}") + verbose_logger.debug("Model=%s; cost=%s", self.model, self.model_call_details["response_cost"]) except litellm.NotFoundError: verbose_logger.warning( - f"Model={self.model} not found in completion cost map. Setting 'response_cost' to None" + "Model=%s not found in completion cost map. Setting 'response_cost' to None", self.model ) self.model_call_details["response_cost"] = None @@ -2681,7 +2685,8 @@ class Logging(LiteLLMLoggingBaseClass): ) except Exception: verbose_logger.error( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging {traceback.format_exc()}" + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging %s", + traceback.format_exc(), ) self._handle_callback_failure(callback=callback) @@ -2705,7 +2710,7 @@ class Logging(LiteLLMLoggingBaseClass): break # Only increment once except Exception as e: - verbose_logger.debug(f"Error in _handle_callback_failure: {e}") + verbose_logger.debug("Error in _handle_callback_failure: %s", e) def _failure_handler_helper_fn(self, exception, traceback_exception, start_time=None, end_time=None): if start_time is None: @@ -2784,7 +2789,7 @@ class Logging(LiteLLMLoggingBaseClass): ) # type: ignore def failure_handler(self, exception, traceback_exception, start_time=None, end_time=None): - verbose_logger.debug(f"Logging Details LiteLLM-Failure Call: {litellm.failure_callback}") + verbose_logger.debug("Logging Details LiteLLM-Failure Call: %s", litellm.failure_callback) if not self.should_run_logging(event_type="sync_failure"): # prevent double logging return litellm_params = self.model_call_details.get("litellm_params", {}) @@ -2949,7 +2954,7 @@ class Logging(LiteLLMLoggingBaseClass): capture_exception(e) except Exception as e: verbose_logger.exception( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure logging {e}" + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure logging %s", e ) async def async_failure_handler(self, exception, traceback_exception, start_time=None, end_time=None): @@ -3005,8 +3010,9 @@ class Logging(LiteLLMLoggingBaseClass): ) except Exception as e: verbose_logger.exception( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure \ - logging {e}\nCallback={callback}" + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure logging %s\nCallback=%s", + e, + callback, ) # Track callback logging failures in Prometheus self._handle_callback_failure(callback=callback) @@ -3134,7 +3140,7 @@ class Logging(LiteLLMLoggingBaseClass): """ filtered = [cb for cb in callbacks if not self._is_internal_litellm_proxy_callback(cb)] - verbose_logger.debug(f"Filtered callbacks: {filtered}") + verbose_logger.debug("Filtered callbacks: %s", filtered) return filtered def _get_callback_name(self, cb) -> str: @@ -4149,7 +4155,7 @@ def _init_custom_logger_compatible_class( return newrelic_logger # type: ignore return None except Exception as e: - verbose_logger.exception(f"[Non-Blocking Error] Error initializing custom logger: {e}") + verbose_logger.exception("[Non-Blocking Error] Error initializing custom logger: %s", e) return None return None @@ -4433,7 +4439,7 @@ def get_custom_logger_compatible_class( return None except Exception as e: - verbose_logger.exception(f"[Non-Blocking Error] Error getting custom logger: {e}") + verbose_logger.exception("[Non-Blocking Error] Error getting custom logger: %s", e) return None @@ -4783,7 +4789,8 @@ class StandardLoggingPayloadSetup: ) except Exception: verbose_logger.debug( # keep in debug otherwise it will trigger on every call - f"Model={model_cost_name} is not mapped in model cost map. Defaulting to None model_cost_information for standard_logging_payload" + "Model=%s is not mapped in model cost map. Defaulting to None model_cost_information for standard_logging_payload", + model_cost_name, ) model_cost_information = StandardLoggingModelInformation( model_map_key=model_cost_name, model_map_value=None @@ -5437,7 +5444,7 @@ def get_standard_logging_object_payload( return payload except Exception as e: - verbose_logger.exception(f"Error creating standard logging object - {e}") + verbose_logger.exception("Error creating standard logging object - %s", e) return None diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 2eaac7cb1ca..51c4e0200c3 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -150,7 +150,8 @@ def _generic_cost_per_character( prompt_cost = prompt_characters * custom_prompt_cost except Exception as e: verbose_logger.exception( - f"litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - {e}\nDefaulting to None" + "litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - %s\nDefaulting to None", + e, ) prompt_cost = None @@ -165,7 +166,8 @@ def _generic_cost_per_character( completion_cost = completion_characters * custom_completion_cost except Exception as e: verbose_logger.exception( - f"litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - {e}\nDefaulting to None" + "litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - %s\nDefaulting to None", + e, ) completion_cost = None @@ -388,7 +390,8 @@ def _get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: floa return float(cost_per_unit) except ValueError: verbose_logger.exception( - f"litellm.litellm_core_utils.llm_cost_calc.utils.py::calculate_cost_per_component(): Exception occured - {cost_per_unit}\nDefaulting to 0.0" + "litellm.litellm_core_utils.llm_cost_calc.utils.py::calculate_cost_per_component(): Exception occured - %s\nDefaulting to 0.0", + cost_per_unit, ) # If the service tier key doesn't exist or is None, try to fall back to the standard key @@ -408,7 +411,8 @@ def _get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: floa return float(fallback_cost) except ValueError: verbose_logger.exception( - f"litellm.litellm_core_utils.llm_cost_calc.utils.py::_get_cost_per_unit(): Exception occured - {fallback_cost}\nDefaulting to 0.0" + "litellm.litellm_core_utils.llm_cost_calc.utils.py::_get_cost_per_unit(): Exception occured - %s\nDefaulting to 0.0", + fallback_cost, ) break # Only try the first matching suffix diff --git a/litellm/litellm_core_utils/llm_response_utils/get_api_base.py b/litellm/litellm_core_utils/llm_response_utils/get_api_base.py index 1982e40448d..8eb8815b584 100644 --- a/litellm/litellm_core_utils/llm_response_utils/get_api_base.py +++ b/litellm/litellm_core_utils/llm_response_utils/get_api_base.py @@ -53,7 +53,7 @@ def get_api_base(model: str, optional_params: dict | LiteLLM_Params) -> str | No api_key=_optional_params.api_key, ) except Exception as e: - verbose_logger.debug(f"Error occurred in getting api base - {e}") + verbose_logger.debug("Error occurred in getting api base - %s", e) custom_llm_provider = None dynamic_api_base = None diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py index be732adfbe1..8bbe9ff4d25 100644 --- a/litellm/litellm_core_utils/logging_callback_manager.py +++ b/litellm/litellm_core_utils/logging_callback_manager.py @@ -146,7 +146,7 @@ class LoggingCallbackManager: if callback not in parent_list: parent_list.append(callback) else: - verbose_logger.debug(f"Callback {callback} already exists in {parent_list}, not adding again..") + verbose_logger.debug("Callback %s already exists in %s, not adding again..", callback, parent_list) def _check_callback_list_size(self, parent_list: list[CustomLogger | Callable | str]) -> bool: """ @@ -155,7 +155,9 @@ class LoggingCallbackManager: """ if len(parent_list) >= MAX_CALLBACKS: verbose_logger.warning( - f"Cannot add callback - would exceed MAX_CALLBACKS limit of {MAX_CALLBACKS}. Current callbacks: {len(parent_list)}" + "Cannot add callback - would exceed MAX_CALLBACKS limit of %s. Current callbacks: %s", + MAX_CALLBACKS, + len(parent_list), ) return False return True @@ -281,7 +283,7 @@ class LoggingCallbackManager: parent_list.append(callback) else: verbose_logger.debug( - f"Callback function {callback.__name__} already exists in {parent_list}, not adding again.." + "Callback function %s already exists in %s, not adding again..", callback.__name__, parent_list ) def _add_custom_logger_to_list( @@ -301,7 +303,10 @@ class LoggingCallbackManager: and self._get_custom_logger_key(existing_logger) == custom_logger_key ): verbose_logger.debug( - f"Custom logger of type {custom_logger_type_name}, key: {custom_logger_key} already exists in {parent_list}, not adding again.." + "Custom logger of type %s, key: %s already exists in %s, not adding again..", + custom_logger_type_name, + custom_logger_key, + parent_list, ) return parent_list.append(custom_logger) diff --git a/litellm/litellm_core_utils/logging_utils.py b/litellm/litellm_core_utils/logging_utils.py index 9340554b6d9..99a9ae4861a 100644 --- a/litellm/litellm_core_utils/logging_utils.py +++ b/litellm/litellm_core_utils/logging_utils.py @@ -178,7 +178,7 @@ def _get_parent_otel_span_from_logging_obj( return _get_parent_otel_span_from_kwargs(logging_obj.model_call_details) except Exception as e: - verbose_logger.exception(f"Error in _get_parent_otel_span_from_logging_obj: {e}") + verbose_logger.exception("Error in _get_parent_otel_span_from_logging_obj: %s", e) return None @@ -265,7 +265,7 @@ def _set_duration_in_model_call_details( else: verbose_logger.debug("`logging_obj` not found - unable to track `llm_api_duration_ms") except Exception as e: - verbose_logger.warning(f"Error setting `llm_api_duration_ms`: {e}") + verbose_logger.warning("Error setting `llm_api_duration_ms`: %s", e) def track_llm_api_timing(): @@ -321,7 +321,7 @@ def track_llm_api_timing(): ) ) except Exception as e: - verbose_logger.debug(f"Error in service logging: {e}") + verbose_logger.debug("Error in service logging: %s", e) @functools.wraps(func) def sync_wrapper(*args, **kwargs): @@ -366,7 +366,7 @@ def track_llm_api_timing(): parent_otel_span=parent_otel_span, ) except Exception as e: - verbose_logger.debug(f"Error in service logging: {e}") + verbose_logger.debug("Error in service logging: %s", e) # Check if the function is async or sync if inspect.iscoroutinefunction(func): diff --git a/litellm/litellm_core_utils/logging_worker.py b/litellm/litellm_core_utils/logging_worker.py index b0d1de32c3c..ff588243621 100644 --- a/litellm/litellm_core_utils/logging_worker.py +++ b/litellm/litellm_core_utils/logging_worker.py @@ -100,7 +100,7 @@ class LoggingWorker: timeout=self.timeout, ) except Exception as e: - verbose_logger.exception(f"LoggingWorker error: {e}") + verbose_logger.exception("LoggingWorker error: %s", e) finally: self._queue.task_done() finally: @@ -297,7 +297,7 @@ class LoggingWorker: if extracted_tasks: await self._process_extracted_tasks(extracted_tasks) except Exception as e: - verbose_logger.exception(f"LoggingWorker error during aggressive clear: {e}") + verbose_logger.exception("LoggingWorker error during aggressive clear: %s", e) finally: # Always reset the flag even if an error occurs self._aggressive_clear_in_progress = False @@ -383,7 +383,7 @@ class LoggingWorker: for _ in range(MAX_ITERATIONS_TO_CLEAR_QUEUE): # Check if we've exceeded the maximum time if asyncio.get_event_loop().time() - start_time >= MAX_TIME_TO_CLEAR_QUEUE: - verbose_logger.warning(f"clear_queue exceeded max_time of {MAX_TIME_TO_CLEAR_QUEUE}s, stopping early") + verbose_logger.warning("clear_queue exceeded max_time of %ss, stopping early", MAX_TIME_TO_CLEAR_QUEUE) break try: diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 8aa4f60b7c5..838261dedc8 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1413,7 +1413,7 @@ def convert_to_gemini_tool_call_result( inline_data_list.append(BlobType(data=mime_rest[1], mime_type=clean_mime)) content_str = "" except Exception as e: - verbose_logger.warning(f"Failed to parse data URL in tool response: {e}") + verbose_logger.warning("Failed to parse data URL in tool response: %s", e) elif isinstance(message["content"], list): content_list = message["content"] for content in content_list: @@ -1432,7 +1432,7 @@ def convert_to_gemini_tool_call_result( ) ) except Exception as e: - verbose_logger.warning(f"Failed to process Anthropic image block in tool response: {e}") + verbose_logger.warning("Failed to process Anthropic image block in tool response: %s", e) elif content_type in ("input_image", "image_url"): # Extract image for inline_data (for Computer Use screenshots and tool results) image_url_data = content.get("image_url", "") @@ -1449,7 +1449,7 @@ def convert_to_gemini_tool_call_result( ) ) except Exception as e: - verbose_logger.warning(f"Failed to process image in tool response: {e}") + verbose_logger.warning("Failed to process image in tool response: %s", e) elif content_type in ("file", "input_file"): # Extract file for inline_data (for tool results with PDF, audio, video, etc.) file_data = content.get("file_data", "") @@ -1474,7 +1474,7 @@ def convert_to_gemini_tool_call_result( ) ) except Exception as e: - verbose_logger.warning(f"Failed to process file in tool response: {e}") + verbose_logger.warning("Failed to process file in tool response: %s", e) name: str | None = message.get("name", "") # type: ignore # Recover name from last message with tool calls @@ -1997,7 +1997,7 @@ def _sanitize_empty_text_content( message = cast(AllMessageValues, dict(message)) # Make a copy message["content"] = _EMPTY_TEXT_PLACEHOLDER verbose_logger.debug( - f"_sanitize_empty_text_content: Replaced empty text content in {message.get('role')} message" + "_sanitize_empty_text_content: Replaced empty text content in %s message", message.get("role") ) return message @@ -2022,7 +2022,7 @@ def _sanitize_empty_text_content( message = cast(AllMessageValues, dict(message)) # Make a copy message["content"] = new_blocks # type: ignore verbose_logger.debug( - f"_sanitize_empty_text_content: Replaced empty text block(s) in {message.get('role')} message" + "_sanitize_empty_text_content: Replaced empty text block(s) in %s message", message.get("role") ) return message @@ -2086,7 +2086,8 @@ def _add_missing_tool_results( if missing_tool_call_ids: verbose_logger.debug( - f"_add_missing_tool_results: Found {len(missing_tool_call_ids)} orphaned tool calls. Adding dummy tool results." + "_add_missing_tool_results: Found %s orphaned tool calls. Adding dummy tool results.", + len(missing_tool_call_ids), ) result_messages.append(current_message) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index c401741f9dc..fd109d7cf3c 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -178,7 +178,7 @@ class RealTimeStreaming: # Catch-all base object so unknown/new event names never raise. typed_obj = OpenAIRealtimeStreamResponseBaseObject(**message_obj) # type: ignore except Exception as e: - verbose_logger.debug(f"Error parsing message for logging: {e}") + verbose_logger.debug("Error parsing message for logging: %s", e) self.messages.append(message_obj) # type: ignore[arg-type] return self.messages.append(typed_obj) @@ -213,7 +213,7 @@ class RealTimeStreaming: if tools and isinstance(tools, list): self.session_tools = tools # GA: session.type is required; log it for traceability but no action needed - verbose_logger.debug(f"Realtime session.type: {session.get('type')}") + verbose_logger.debug("Realtime session.type: %s", session.get("type")) if session.get("type") == "transcription": self._is_transcription_session = True except (json.JSONDecodeError, AttributeError, TypeError): @@ -981,7 +981,7 @@ class RealTimeStreaming: try: await self._handle_provider_config_message(raw_response) except Exception as e: - verbose_logger.exception(f"Error processing backend message, skipping: {e}") + verbose_logger.exception("Error processing backend message, skipping: %s", e) continue else: event = self._parse_backend_event(raw_response) @@ -1008,9 +1008,9 @@ class RealTimeStreaming: await self.websocket.send_text(json.dumps(translated)) except websockets.exceptions.ConnectionClosed as e: # type: ignore - verbose_logger.exception(f"Connection closed in backend to client send messages - {e}") + verbose_logger.exception("Connection closed in backend to client send messages - %s", e) except Exception as e: - verbose_logger.exception(f"Error in backend to client send messages: {e}") + verbose_logger.exception("Error in backend to client send messages: %s", e) finally: await self.log_messages() @@ -1404,7 +1404,7 @@ class RealTimeStreaming: self._guardrail_turn_detection_update_sent = True except Exception as e: - verbose_logger.debug(f"Error in client ack messages: {e}") + verbose_logger.debug("Error in client ack messages: %s", e) async def bidirectional_forward(self): forward_task = asyncio.create_task(self.backend_to_client_send_messages()) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 25155068baa..5cdb5877915 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -618,7 +618,7 @@ class CustomStreamWrapper: else: return "" except Exception as e: - verbose_logger.exception(f"litellm.CustomStreamWrapper.handle_baseten_chunk(): Exception occured - {e}") + verbose_logger.exception("litellm.CustomStreamWrapper.handle_baseten_chunk(): Exception occured - %s", e) return "" def handle_triton_stream(self, chunk): @@ -1430,7 +1430,7 @@ class CustomStreamWrapper: model_response.choices[0].delta = Delta(**_json_delta) except Exception as e: verbose_logger.exception( - f"litellm.CustomStreamWrapper.chunk_creator(): Exception occured - {e}" + "litellm.CustomStreamWrapper.chunk_creator(): Exception occured - %s", e ) model_response.choices[0].delta = Delta() elif self._has_any_special_delta_attributes(delta): @@ -1538,7 +1538,7 @@ class CustomStreamWrapper: except Exception as e: from litellm._logging import verbose_logger - verbose_logger.exception(f"Error in post-call streaming deployment hook: {e}") + verbose_logger.exception("Error in post-call streaming deployment hook: %s", e) return chunk def _add_mcp_list_tools_to_first_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream: @@ -1578,7 +1578,7 @@ class CustomStreamWrapper: except Exception as e: from litellm._logging import verbose_logger - verbose_logger.exception(f"Error adding MCP list tools to first chunk: {e}") + verbose_logger.exception("Error adding MCP list tools to first chunk: %s", e) return chunk @@ -1615,7 +1615,7 @@ class CustomStreamWrapper: except Exception as e: from litellm._logging import verbose_logger - verbose_logger.exception(f"Error adding MCP metadata to final chunk: {e}") + verbose_logger.exception("Error adding MCP metadata to final chunk: %s", e) return chunk diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index ff94965f628..d65f70d5a7a 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -80,17 +80,20 @@ def get_modified_max_tokens( ) # give at least a 10 token buffer. token counting can be imprecise. input_tokens += int(token_buffer) - verbose_logger.debug(f"max_output_tokens: {max_output_tokens}, user_max_tokens: {user_max_tokens}") + verbose_logger.debug("max_output_tokens: %s, user_max_tokens: %s", max_output_tokens, user_max_tokens) ## CASE 1: model input + output can't exceed X - happens when max input = max output, e.g. gpt-3.5-turbo if _model_info["max_input_tokens"] == max_output_tokens: - verbose_logger.debug(f"input_tokens: {input_tokens}, max_output_tokens: {max_output_tokens}") + verbose_logger.debug("input_tokens: %s, max_output_tokens: %s", input_tokens, max_output_tokens) if input_tokens > max_output_tokens: pass # allow call to fail normally - don't set max_tokens to negative. elif ( user_max_tokens + input_tokens > max_output_tokens ): # we can still modify to keep it positive but below the limit verbose_logger.debug( - f"MODIFYING MAX TOKENS - user_max_tokens={user_max_tokens}, input_tokens={input_tokens}, max_output_tokens={max_output_tokens}" + "MODIFYING MAX TOKENS - user_max_tokens=%s, input_tokens=%s, max_output_tokens=%s", + user_max_tokens, + input_tokens, + max_output_tokens, ) user_max_tokens = int(max_output_tokens - input_tokens) ## CASE 2: user_max_tokens> model max output tokens @@ -98,13 +101,17 @@ def get_modified_max_tokens( user_max_tokens = max_output_tokens verbose_logger.debug( - f"litellm.litellm_core_utils.token_counter.py::get_modified_max_tokens() - user_max_tokens: {user_max_tokens}" + "litellm.litellm_core_utils.token_counter.py::get_modified_max_tokens() - user_max_tokens: %s", + user_max_tokens, ) return user_max_tokens except Exception as e: verbose_logger.debug( - f"litellm.litellm_core_utils.token_counter.py::get_modified_max_tokens() - Error while checking max token limit: {e}\nmodel={model}, base_model={base_model}" + "litellm.litellm_core_utils.token_counter.py::get_modified_max_tokens() - Error while checking max token limit: %s\nmodel=%s, base_model=%s", + e, + model, + base_model, ) return user_max_tokens @@ -280,7 +287,7 @@ def calculate_img_tokens( int: The number of tokens for the image. """ if use_default_image_token_count: - verbose_logger.debug(f"Using default image token count: {DEFAULT_IMAGE_TOKEN_COUNT}") + verbose_logger.debug("Using default image token count: %s", DEFAULT_IMAGE_TOKEN_COUNT) return DEFAULT_IMAGE_TOKEN_COUNT if mode == "low" or mode == "auto": return base_tokens @@ -367,7 +374,7 @@ def token_counter( if litellm.disable_token_counter is True: return 0 - verbose_logger.debug(f"messages in token_counter: {messages}, text in token_counter: {text}") + verbose_logger.debug("messages in token_counter: %s, text in token_counter: %s", messages, text) if text is not None and messages is not None: raise ValueError("text and messages cannot both be set") if use_default_image_token_count is None: diff --git a/litellm/llms/__init__.py b/litellm/llms/__init__.py index 60715ac9bbf..a35fe5b2093 100644 --- a/litellm/llms/__init__.py +++ b/litellm/llms/__init__.py @@ -92,7 +92,7 @@ def discover_guardrail_translation_mappings() -> dict[CallTypes, type["BaseTrans try: # Import the module - verbose_logger.debug(f"Discovering guardrail translations in: {module_path}") + verbose_logger.debug("Discovering guardrail translations in: %s", module_path) module = importlib.import_module(module_path) @@ -102,14 +102,14 @@ def discover_guardrail_translation_mappings() -> dict[CallTypes, type["BaseTrans if isinstance(mappings, dict): discovered_mappings.update(mappings) verbose_logger.debug( - f"Found guardrail_translation_mappings in {module_path}: {list(mappings.keys())}" + "Found guardrail_translation_mappings in %s: %s", module_path, list(mappings.keys()) ) except ImportError as e: - verbose_logger.error(f"Could not import {module_path}: {e}") + verbose_logger.error("Could not import %s: %s", module_path, e) continue except Exception as e: - verbose_logger.error(f"Error processing {module_path}: {e}") + verbose_logger.error("Error processing %s: %s", module_path, e) continue try: @@ -126,11 +126,13 @@ def discover_guardrail_translation_mappings() -> dict[CallTypes, type["BaseTrans verbose_logger.debug("MCP guardrail translation mappings not available; skipping") verbose_logger.debug( - f"Discovered {len(discovered_mappings)} guardrail translation mappings: {list(discovered_mappings.keys())}" + "Discovered %s guardrail translation mappings: %s", + len(discovered_mappings), + list(discovered_mappings.keys()), ) except Exception as e: - verbose_logger.error(f"Error discovering guardrail translation mappings: {e}") + verbose_logger.error("Error discovering guardrail translation mappings: %s", e) return discovered_mappings diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 0fb7d7802a0..c752d02f662 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -825,10 +825,10 @@ class AnthropicMessagesHandler(BaseTranslation): if delta.get("type") == "text_delta": text += delta.get("text", "") except json.JSONDecodeError: - verbose_proxy_logger.warning(f"Failed to parse JSON from SSE data: {data_line}") + verbose_proxy_logger.warning("Failed to parse JSON from SSE data: %s", data_line) except Exception as e: - verbose_proxy_logger.error(f"Error extracting text from SSE: {e}") + verbose_proxy_logger.error("Error extracting text from SSE: %s", e) return text @@ -889,10 +889,10 @@ class AnthropicMessagesHandler(BaseTranslation): if stop_reason is not None: return True except json.JSONDecodeError: - verbose_proxy_logger.warning(f"Failed to parse JSON from SSE data: {data_line}") + verbose_proxy_logger.warning("Failed to parse JSON from SSE data: %s", data_line) except Exception as e: - verbose_proxy_logger.error(f"Error checking streaming end in SSE: {e}") + verbose_proxy_logger.error("Error checking streaming end in SSE: %s", e) # Handle already-parsed dict format elif isinstance(response, dict): diff --git a/litellm/llms/anthropic/count_tokens/handler.py b/litellm/llms/anthropic/count_tokens/handler.py index 0c3d0e931a2..3762b2b2f2f 100644 --- a/litellm/llms/anthropic/count_tokens/handler.py +++ b/litellm/llms/anthropic/count_tokens/handler.py @@ -54,7 +54,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): # Validate the request self.validate_request(model, messages) - verbose_logger.debug(f"Processing Anthropic CountTokens request for model: {model}") + verbose_logger.debug("Processing Anthropic CountTokens request for model: %s", model) # Transform request to Anthropic format request_body = self.transform_request_to_count_tokens( @@ -64,12 +64,12 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): system=system, ) - verbose_logger.debug(f"Transformed request: {request_body}") + verbose_logger.debug("Transformed request: %s", request_body) # Get endpoint URL endpoint_url = api_base or self.get_anthropic_count_tokens_endpoint() - verbose_logger.debug(f"Making request to: {endpoint_url}") + verbose_logger.debug("Making request to: %s", endpoint_url) # Get required headers headers = self.get_required_headers(api_key) @@ -87,11 +87,11 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): timeout=request_timeout, ) - verbose_logger.debug(f"Response status: {response.status_code}") + verbose_logger.debug("Response status: %s", response.status_code) if response.status_code != 200: error_text = response.text - verbose_logger.error(f"Anthropic API error: {error_text}") + verbose_logger.error("Anthropic API error: %s", error_text) raise AnthropicError( status_code=response.status_code, message=error_text, @@ -99,7 +99,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): anthropic_response = response.json() - verbose_logger.debug(f"Anthropic response: {anthropic_response}") + verbose_logger.debug("Anthropic response: %s", anthropic_response) # Return Anthropic response directly - no transformation needed return anthropic_response @@ -109,13 +109,13 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): raise except httpx.HTTPStatusError as e: # HTTP errors - preserve the actual status code - verbose_logger.error(f"HTTP error in CountTokens handler: {e}") + verbose_logger.error("HTTP error in CountTokens handler: %s", e) raise AnthropicError( status_code=e.response.status_code, message=e.response.text, ) except Exception as e: - verbose_logger.error(f"Error in CountTokens handler: {e}") + verbose_logger.error("Error in CountTokens handler: %s", e) raise AnthropicError( status_code=500, message=f"CountTokens processing error: {e}", diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index 8cc9d2ec0a9..5a2b6044528 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -81,7 +81,7 @@ class AnthropicTokenCounter(BaseTokenCounter): original_response=result, ) except AnthropicError as e: - verbose_logger.warning(f"Anthropic CountTokens API error: status={e.status_code}, message={e.message}") + verbose_logger.warning("Anthropic CountTokens API error: status=%s, message=%s", e.status_code, e.message) return TokenCountResponse( total_tokens=0, request_model=request_model, @@ -92,7 +92,7 @@ class AnthropicTokenCounter(BaseTokenCounter): status_code=e.status_code, ) except Exception as e: - verbose_logger.warning(f"Error calling Anthropic CountTokens API: {e}") + verbose_logger.warning("Error calling Anthropic CountTokens API: %s", e) return TokenCountResponse( total_tokens=0, request_model=request_model, diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index bb61043742d..5de40cc34b5 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -669,7 +669,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): return {"type": "message_stop"} raise StopIteration except Exception as e: - verbose_logger.error(f"Anthropic Adapter - {e}\n{traceback.format_exc()}") + verbose_logger.error("Anthropic Adapter - %s\n%s", e, traceback.format_exc()) raise StopIteration async def __anext__(self): diff --git a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/clear_tool_uses.py b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/clear_tool_uses.py index abfc45859ef..b432b122504 100644 --- a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/clear_tool_uses.py +++ b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/clear_tool_uses.py @@ -68,8 +68,8 @@ def _trigger_met( messages=messages, tools=cast(Any, tools), ) - verbose_logger.debug(f"context_management polyfill: current_tokens: {current_tokens}") - verbose_logger.debug(f"context_management polyfill: threshold: {threshold}") + verbose_logger.debug("context_management polyfill: current_tokens: %s", current_tokens) + verbose_logger.debug("context_management polyfill: threshold: %s", threshold) return current_tokens > threshold, current_tokens diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py index 887240bb1cc..74ee291bf9f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py @@ -164,8 +164,9 @@ async def anthropic_messages_with_mcp( response = await litellm.anthropic_messages(messages=list(working_messages), stream=False, **base_call_args) else: verbose_logger.warning( - f"MCP tool loop hit its {MAX_MCP_TOOL_USE_ITERATIONS} iteration cap for model {model}; " - "returning the last response" + "MCP tool loop hit its %s iteration cap for model %s; returning the last response", + MAX_MCP_TOOL_USE_ITERATIONS, + model, ) if stream: diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index f698c78604d..251c4e0f61a 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -300,7 +300,7 @@ class AnthropicResponsesStreamWrapper: except StopAsyncIteration: pass except Exception as e: - verbose_logger.error(f"AnthropicResponsesStreamWrapper error: {e}\n{traceback.format_exc()}") + verbose_logger.error("AnthropicResponsesStreamWrapper error: %s\n%s", e, traceback.format_exc()) # Drain any remaining queued chunks if self._chunk_queue: diff --git a/litellm/llms/anthropic/files/handler.py b/litellm/llms/anthropic/files/handler.py index b911347b2ff..c05392e0d7c 100644 --- a/litellm/llms/anthropic/files/handler.py +++ b/litellm/llms/anthropic/files/handler.py @@ -270,7 +270,7 @@ class AnthropicFilesHandler: transformed_content += "\n" # Add trailing newline for JSONL format return transformed_content.encode("utf-8") except Exception as e: - verbose_logger.error(f"Error transforming Anthropic batch results to OpenAI format: {e}") + verbose_logger.error("Error transforming Anthropic batch results to OpenAI format: %s", e) # Return original content if transformation fails return anthropic_content @@ -330,7 +330,7 @@ class AnthropicFilesHandler: return openai_body except Exception as e: - verbose_logger.error(f"Error transforming Anthropic message to OpenAI format: {e}") + verbose_logger.error("Error transforming Anthropic message to OpenAI format: %s", e) # Return a basic error response if transformation fails error_response: OpenAIChatCompletionResponse = { "id": anthropic_message.get("id", ""), diff --git a/litellm/llms/azure/chat/o_series_transformation.py b/litellm/llms/azure/chat/o_series_transformation.py index 2b8c2e88e51..86944c3b2c2 100644 --- a/litellm/llms/azure/chat/o_series_transformation.py +++ b/litellm/llms/azure/chat/o_series_transformation.py @@ -91,7 +91,7 @@ class AzureOpenAIO1Config(OpenAIOSeriesConfig): ): # allow user to override default with model_info={"supports_native_streaming": true} return False except Exception as e: - verbose_logger.debug(f"Error getting model info in AzureOpenAIO1Config: {e}") + verbose_logger.debug("Error getting model info in AzureOpenAIO1Config: %s", e) return True def is_o_series_model(self, model: str) -> bool: diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index 8db422e00ff..45f683d5962 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -333,7 +333,8 @@ def get_azure_ad_token( verbose_logger.debug("Azure AD Token Provider could not be used.") except Exception as e: verbose_logger.error( - f"Error calling Azure AD token provider: {e}. Follow docs - https://docs.litellm.ai/docs/providers/azure/#azure-ad-token-refresh---defaultazurecredential" + "Error calling Azure AD token provider: %s. Follow docs - https://docs.litellm.ai/docs/providers/azure/#azure-ad-token-refresh---defaultazurecredential", + e, ) raise e @@ -351,7 +352,7 @@ def get_azure_ad_token( try: token = azure_ad_token_provider() if not isinstance(token, str): - verbose_logger.error(f"Azure AD token provider returned non-string value: {type(token)}") + verbose_logger.error("Azure AD token provider returned non-string value: %s", type(token)) raise TypeError(f"Azure AD token must be a string, got {type(token)}") else: azure_ad_token = token @@ -359,7 +360,7 @@ def get_azure_ad_token( # Re-raise TypeError directly raise except Exception as e: - verbose_logger.error(f"Error calling Azure AD token provider: {e}") + verbose_logger.error("Error calling Azure AD token provider: %s", e) raise RuntimeError(f"Failed to get Azure AD token: {e}") from e return azure_ad_token @@ -393,7 +394,7 @@ class BaseAzureLLM(BaseOpenAILLM): verbose_logger.debug("Successfully obtained Azure AD token provider using DefaultAzureCredential") return azure_ad_token_provider except Exception as e: - verbose_logger.debug(f"DefaultAzureCredential failed: {e}") + verbose_logger.debug("DefaultAzureCredential failed: %s", e) return None def get_azure_openai_client( @@ -481,7 +482,7 @@ class BaseAzureLLM(BaseOpenAILLM): if "http_client" in azure_client_params: v1_params["http_client"] = azure_client_params["http_client"] - verbose_logger.debug(f"Using Azure v1 API with base_url: {v1_params['base_url']}") + verbose_logger.debug("Using Azure v1 API with base_url: %s", v1_params["base_url"]) if _is_async is True: openai_client = AsyncOpenAI(**v1_params) # type: ignore @@ -582,7 +583,7 @@ class BaseAzureLLM(BaseOpenAILLM): # only show first 5 chars of api_key _api_key = _api_key[:8] + "*" * 15 verbose_logger.debug( - f"Initializing Azure OpenAI Client for {model_name}, Api Base: {api_base}, Api Key:{_api_key}" + "Initializing Azure OpenAI Client for %s, Api Base: %s, Api Key:%s", model_name, api_base, _api_key ) azure_client_params = { "api_key": api_key, diff --git a/litellm/llms/azure/cost_calculation.py b/litellm/llms/azure/cost_calculation.py index 6fddb8523e7..ab4307f9172 100644 --- a/litellm/llms/azure/cost_calculation.py +++ b/litellm/llms/azure/cost_calculation.py @@ -35,7 +35,10 @@ def cost_per_token( and response_time_ms is not None ): verbose_logger.debug( - f"For model={model} - output_cost_per_second: {model_info.get('output_cost_per_second')}; response time: {response_time_ms}" + "For model=%s - output_cost_per_second: %s; response time: %s", + model, + model_info.get("output_cost_per_second"), + response_time_ms, ) ## COST PER SECOND ## prompt_cost = 0.0 diff --git a/litellm/llms/azure/image_generation/__init__.py b/litellm/llms/azure/image_generation/__init__.py index 64636bc689d..a2a905f2287 100644 --- a/litellm/llms/azure/image_generation/__init__.py +++ b/litellm/llms/azure/image_generation/__init__.py @@ -29,6 +29,6 @@ def get_azure_image_generation_config(model: str) -> BaseImageGenerationConfig: return AzureFoundryMAIImageGenerationConfig() else: verbose_logger.debug( - f"Using AzureGPTImageGenerationConfig for model: {model}. This follows the gpt-image model format." + "Using AzureGPTImageGenerationConfig for model: %s. This follows the gpt-image model format.", model ) return AzureGPTImageGenerationConfig() diff --git a/litellm/llms/azure/responses/o_series_transformation.py b/litellm/llms/azure/responses/o_series_transformation.py index 7a88c42cb14..121ceda7fa1 100644 --- a/litellm/llms/azure/responses/o_series_transformation.py +++ b/litellm/llms/azure/responses/o_series_transformation.py @@ -68,7 +68,7 @@ class AzureOpenAIOSeriesResponsesAPIConfig(AzureOpenAIResponsesAPIConfig): # If drop_params is enabled, remove temperature parameter for O-series models if drop_params and "temperature" in mapped_params: verbose_logger.debug( - f"Dropping unsupported parameter 'temperature' for Azure OpenAI O-series responses API model {model}" + "Dropping unsupported parameter 'temperature' for Azure OpenAI O-series responses API model %s", model ) mapped_params.pop("temperature", None) diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 860b1b1dd5c..51b63b6299c 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -74,7 +74,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): return dict_reasoning_item except Exception as e: - verbose_logger.debug(f"Failed to create ResponseReasoningItem, falling back to manual filtering: {e}") + verbose_logger.debug("Failed to create ResponseReasoningItem, falling back to manual filtering: %s", e) # Fallback: manually filter out known None fields filtered_item = { k: v @@ -252,7 +252,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): delete_url = self._construct_url_for_response_id_in_path(api_base=api_base, response_id=response_id) data: dict = {} - verbose_logger.debug(f"delete response url={delete_url}") + verbose_logger.debug("delete response url=%s", delete_url) return delete_url, data ######################################################### @@ -273,7 +273,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): """ get_url = self._construct_url_for_response_id_in_path(api_base=api_base, response_id=response_id) data: dict = {} - verbose_logger.debug(f"get response url={get_url}") + verbose_logger.debug("get response url=%s", get_url) return get_url, data def transform_list_input_items_request( @@ -302,7 +302,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): params["limit"] = limit if order is not None: params["order"] = order - verbose_logger.debug(f"list input items url={url}") + verbose_logger.debug("list input items url=%s", url) return url, params ######################################################### @@ -329,7 +329,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): ) data: dict = {} - verbose_logger.debug(f"cancel response url={cancel_url}") + verbose_logger.debug("cancel response url=%s", cancel_url) return cancel_url, data def transform_cancel_response_api_response( diff --git a/litellm/llms/azure_ai/agents/handler.py b/litellm/llms/azure_ai/agents/handler.py index 7023dbca0b8..81532aed208 100644 --- a/litellm/llms/azure_ai/agents/handler.py +++ b/litellm/llms/azure_ai/agents/handler.py @@ -193,7 +193,7 @@ class AzureAIAgentsHandler: ), ) except Exception as e: - verbose_logger.warning(f"Failed to calculate token usage: {e}") + verbose_logger.warning("Failed to calculate token usage: %s", e) return model_response @@ -226,7 +226,7 @@ class AzureAIAgentsHandler: thread_id = optional_params.get("thread_id") api_base = api_base.rstrip("/") - verbose_logger.debug(f"Azure AI Agents completion - api_base: {api_base}, agent_id: {agent_id}") + verbose_logger.debug("Azure AI Agents completion - api_base: %s, agent_id: %s", api_base, agent_id) return headers, api_version, agent_id, thread_id, api_base @@ -305,11 +305,11 @@ class AzureAIAgentsHandler: # Step 1: Create thread if not provided if not thread_id: - verbose_logger.debug(f"Creating thread at: {self._build_thread_url(api_base, api_version)}") + verbose_logger.debug("Creating thread at: %s", self._build_thread_url(api_base, api_version)) response = make_request("POST", self._build_thread_url(api_base, api_version), {}) self._check_response(response, [200, 201], "Failed to create thread") thread_id = response.json()["id"] - verbose_logger.debug(f"Created thread: {thread_id}") + verbose_logger.debug("Created thread: %s", thread_id) # At this point thread_id is guaranteed to be a string assert thread_id is not None @@ -329,7 +329,7 @@ class AzureAIAgentsHandler: response = make_request("POST", self._build_runs_url(api_base, thread_id, api_version), run_payload) self._check_response(response, [200, 201], "Failed to create run") run_id = response.json()["id"] - verbose_logger.debug(f"Created run: {run_id}") + verbose_logger.debug("Created run: %s", run_id) # Step 4: Poll for completion status_url = self._build_run_status_url(api_base, thread_id, run_id, api_version) @@ -338,7 +338,7 @@ class AzureAIAgentsHandler: self._check_response(response, [200], "Failed to get run status") status = response.json().get("status") - verbose_logger.debug(f"Run status: {status}") + verbose_logger.debug("Run status: %s", status) if status == "completed": break @@ -428,11 +428,11 @@ class AzureAIAgentsHandler: # Step 1: Create thread if not provided if not thread_id: - verbose_logger.debug(f"Creating thread at: {self._build_thread_url(api_base, api_version)}") + verbose_logger.debug("Creating thread at: %s", self._build_thread_url(api_base, api_version)) response = await make_request("POST", self._build_thread_url(api_base, api_version), {}) self._check_response(response, [200, 201], "Failed to create thread") thread_id = response.json()["id"] - verbose_logger.debug(f"Created thread: {thread_id}") + verbose_logger.debug("Created thread: %s", thread_id) # At this point thread_id is guaranteed to be a string assert thread_id is not None @@ -452,7 +452,7 @@ class AzureAIAgentsHandler: response = await make_request("POST", self._build_runs_url(api_base, thread_id, api_version), run_payload) self._check_response(response, [200, 201], "Failed to create run") run_id = response.json()["id"] - verbose_logger.debug(f"Created run: {run_id}") + verbose_logger.debug("Created run: %s", run_id) # Step 4: Poll for completion status_url = self._build_run_status_url(api_base, thread_id, run_id, api_version) @@ -461,7 +461,7 @@ class AzureAIAgentsHandler: self._check_response(response, [200], "Failed to get run status") status = response.json().get("status") - verbose_logger.debug(f"Run status: {status}") + verbose_logger.debug("Run status: %s", status) if status == "completed": break @@ -526,7 +526,7 @@ class AzureAIAgentsHandler: payload["instructions"] = optional_params["instructions"] url = self._build_create_thread_and_run_url(api_base, api_version) - verbose_logger.debug(f"Azure AI Agents streaming - URL: {url}") + verbose_logger.debug("Azure AI Agents streaming - URL: %s", url) # Use LiteLLM's async HTTP client for streaming client = get_async_httpx_client( @@ -607,7 +607,7 @@ class AzureAIAgentsHandler: # Extract thread_id from thread.created event if current_event == "thread.created" and "id" in data: thread_id = data["id"] - verbose_logger.debug(f"Stream created thread: {thread_id}") + verbose_logger.debug("Stream created thread: %s", thread_id) # Extract annotations from completed message if current_event == "thread.message.completed": diff --git a/litellm/llms/azure_ai/agents/transformation.py b/litellm/llms/azure_ai/agents/transformation.py index fb18b0cb651..7b6c8f0eefb 100644 --- a/litellm/llms/azure_ai/agents/transformation.py +++ b/litellm/llms/azure_ai/agents/transformation.py @@ -229,7 +229,7 @@ class AzureAIAgentsConfig(BaseConfig): if "instructions" in optional_params: payload["instructions"] = optional_params["instructions"] - verbose_logger.debug(f"Azure AI Agents request payload: {payload}") + verbose_logger.debug("Azure AI Agents request payload: %s", payload) return payload def validate_environment( diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py index 65d8c0182ee..fca2244265b 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py @@ -56,7 +56,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): # Validate the request self.validate_request(model, messages) - verbose_logger.debug(f"Processing Azure AI Anthropic CountTokens request for model: {model}") + verbose_logger.debug("Processing Azure AI Anthropic CountTokens request for model: %s", model) # Transform request to Anthropic format request_body = self.transform_request_to_count_tokens( @@ -66,12 +66,12 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): system=system, ) - verbose_logger.debug(f"Transformed request: {request_body}") + verbose_logger.debug("Transformed request: %s", request_body) # Get endpoint URL endpoint_url = self.get_count_tokens_endpoint(api_base) - verbose_logger.debug(f"Making request to: {endpoint_url}") + verbose_logger.debug("Making request to: %s", endpoint_url) # Get required headers with Azure authentication headers = self.get_required_headers( @@ -92,11 +92,11 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): timeout=request_timeout, ) - verbose_logger.debug(f"Response status: {response.status_code}") + verbose_logger.debug("Response status: %s", response.status_code) if response.status_code != 200: error_text = response.text - verbose_logger.error(f"Azure AI Anthropic API error: {error_text}") + verbose_logger.error("Azure AI Anthropic API error: %s", error_text) raise AnthropicError( status_code=response.status_code, message=error_text, @@ -104,7 +104,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): azure_response = response.json() - verbose_logger.debug(f"Azure AI Anthropic response: {azure_response}") + verbose_logger.debug("Azure AI Anthropic response: %s", azure_response) # Return Anthropic-compatible response directly - no transformation needed return azure_response @@ -114,13 +114,13 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): raise except httpx.HTTPStatusError as e: # HTTP errors - preserve the actual status code - verbose_logger.error(f"HTTP error in CountTokens handler: {e}") + verbose_logger.error("HTTP error in CountTokens handler: %s", e) raise AnthropicError( status_code=e.response.status_code, message=e.response.text, ) except Exception as e: - verbose_logger.error(f"Error in CountTokens handler: {e}") + verbose_logger.error("Error in CountTokens handler: %s", e) raise AnthropicError( status_code=500, message=f"CountTokens processing error: {e}", diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py index 129d7bb7aa9..277c07584ed 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py @@ -95,7 +95,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): ) except AnthropicError as e: verbose_logger.warning( - f"Azure AI Anthropic CountTokens API error: status={e.status_code}, message={e.message}" + "Azure AI Anthropic CountTokens API error: status=%s, message=%s", e.status_code, e.message ) return TokenCountResponse( total_tokens=0, @@ -107,7 +107,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): status_code=e.status_code, ) except Exception as e: - verbose_logger.warning(f"Error calling Azure AI Anthropic CountTokens API: {e}") + verbose_logger.warning("Error calling Azure AI Anthropic CountTokens API: %s", e) return TokenCountResponse( total_tokens=0, request_model=request_model, diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 707ddc9e12b..083bed024f7 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -206,7 +206,7 @@ class AzureAIStudioConfig(OpenAIConfig): dynamic_api_key = api_key or get_secret_str("AZURE_AI_API_KEY") if self._is_azure_openai_model(model=model, api_base=api_base): - verbose_logger.debug(f"Model={model} is Azure OpenAI model. Setting custom_llm_provider='azure'.") + verbose_logger.debug("Model=%s is Azure OpenAI model. Setting custom_llm_provider='azure'.", model) custom_llm_provider = "azure" return api_base, dynamic_api_key, custom_llm_provider diff --git a/litellm/llms/azure_ai/cost_calculator.py b/litellm/llms/azure_ai/cost_calculator.py index 6cc0cb20e27..3642fb37e86 100644 --- a/litellm/llms/azure_ai/cost_calculator.py +++ b/litellm/llms/azure_ai/cost_calculator.py @@ -107,7 +107,7 @@ def cost_per_token( # Re-raise for non-router models - they should have pricing defined raise verbose_logger.debug( - f"Azure AI Model Router: model '{model}' not in cost map, calculating routing flat cost only. Error: {e}" + "Azure AI Model Router: model '%s' not in cost map, calculating routing flat cost only. Error: %s", model, e ) # Add flat cost for Azure Model Router diff --git a/litellm/llms/azure_ai/image_generation/__init__.py b/litellm/llms/azure_ai/image_generation/__init__.py index fd511654665..742922ad295 100644 --- a/litellm/llms/azure_ai/image_generation/__init__.py +++ b/litellm/llms/azure_ai/image_generation/__init__.py @@ -32,6 +32,6 @@ def get_azure_ai_image_generation_config(model: str) -> BaseImageGenerationConfi return AzureFoundryFluxImageGenerationConfig() else: verbose_logger.debug( - f"Using AzureGPTImageGenerationConfig for model: {model}. This follows the gpt-image-1 model format." + "Using AzureGPTImageGenerationConfig for model: %s. This follows the gpt-image-1 model format.", model ) return AzureFoundryGPTImageGenerationConfig() diff --git a/litellm/llms/azure_ai/ocr/common_utils.py b/litellm/llms/azure_ai/ocr/common_utils.py index 14b77338fd7..fbeb8b8f4bd 100644 --- a/litellm/llms/azure_ai/ocr/common_utils.py +++ b/litellm/llms/azure_ai/ocr/common_utils.py @@ -53,9 +53,9 @@ def get_azure_ai_ocr_config(model: str) -> Optional["BaseOCRConfig"]: # Check for Azure Document Intelligence models if is_azure_document_intelligence_model(model): - verbose_logger.debug(f"Routing {model} to Azure Document Intelligence OCR config") + verbose_logger.debug("Routing %s to Azure Document Intelligence OCR config", model) return AzureDocumentIntelligenceOCRConfig() # Default to Mistral-based OCR for other azure_ai models - verbose_logger.debug(f"Routing {model} to Azure AI (Mistral) OCR config") + verbose_logger.debug("Routing %s to Azure AI (Mistral) OCR config", model) return AzureAIOCRConfig() diff --git a/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py b/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py index 66ce84cea0f..503b58a44b7 100644 --- a/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py +++ b/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py @@ -353,7 +353,7 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): Returns: OCRRequestData with JSON data """ - verbose_logger.debug(f"Azure Document Intelligence transform_ocr_request - model: {model}") + verbose_logger.debug("Azure Document Intelligence transform_ocr_request - model: %s", model) if not isinstance(document, dict): raise ValueError(f"Expected document dict, got {type(document)}") @@ -455,7 +455,7 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): Retry-after duration in seconds (default: 2) """ retry_after = int(response.headers.get("retry-after", "2")) - verbose_logger.debug(f"Retry polling after: {retry_after} seconds") + verbose_logger.debug("Retry polling after: %s seconds", retry_after) return retry_after @staticmethod @@ -476,7 +476,7 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): result = response.json() status = result.get("status") - verbose_logger.debug(f"Azure DI operation status: {status}") + verbose_logger.debug("Azure DI operation status: %s", status) if status == "succeeded": return "succeeded" @@ -519,7 +519,7 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): client = _get_httpx_client() start_time = time.time() - verbose_logger.debug(f"Polling Azure DI operation: {operation_url}") + verbose_logger.debug("Polling Azure DI operation: %s", operation_url) while True: self._check_timeout(start_time=start_time, timeout_secs=timeout_secs) @@ -560,7 +560,7 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): client = get_async_httpx_client(llm_provider=litellm.LlmProviders.AZURE_AI) start_time = time.time() - verbose_logger.debug(f"Polling Azure DI operation (async): {operation_url}") + verbose_logger.debug("Polling Azure DI operation (async): %s", operation_url) while True: self._check_timeout(start_time=start_time, timeout_secs=timeout_secs) @@ -603,7 +603,7 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): """ operation = AzureDocumentIntelligenceOperation.model_validate(raw_response.json()) - verbose_logger.debug(f"Azure Document Intelligence response status: {operation.status}") + verbose_logger.debug("Azure Document Intelligence response status: %s", operation.status) if operation.status != "succeeded": raise ValueError(f"Azure Document Intelligence analysis failed with status: {operation.status}") diff --git a/litellm/llms/azure_ai/ocr/transformation.py b/litellm/llms/azure_ai/ocr/transformation.py index d757a7f1378..36f7159c0d4 100644 --- a/litellm/llms/azure_ai/ocr/transformation.py +++ b/litellm/llms/azure_ai/ocr/transformation.py @@ -117,13 +117,13 @@ class AzureAIOCRConfig(MistralOCRConfig): Returns: Base64 data URI string """ - verbose_logger.debug(f"Azure AI OCR: Converting URL to base64 data URI (sync): {url}") + verbose_logger.debug("Azure AI OCR: Converting URL to base64 data URI (sync): %s", url) # Fetch and convert to base64 data URI # convert_url_to_base64 already returns a full data URI like "data:image/jpeg;base64,..." data_uri = convert_url_to_base64(url=url) - verbose_logger.debug(f"Azure AI OCR: Converted URL to data URI (length: {len(data_uri)})") + verbose_logger.debug("Azure AI OCR: Converted URL to data URI (length: %s)", len(data_uri)) return data_uri @@ -140,13 +140,13 @@ class AzureAIOCRConfig(MistralOCRConfig): Returns: Base64 data URI string """ - verbose_logger.debug(f"Azure AI OCR: Converting URL to base64 data URI (async): {url}") + verbose_logger.debug("Azure AI OCR: Converting URL to base64 data URI (async): %s", url) # Fetch and convert to base64 data URI asynchronously # async_convert_url_to_base64 already returns a full data URI like "data:image/jpeg;base64,..." data_uri = await async_convert_url_to_base64(url=url) - verbose_logger.debug(f"Azure AI OCR: Converted URL to data URI (length: {len(data_uri)})") + verbose_logger.debug("Azure AI OCR: Converted URL to data URI (length: %s)", len(data_uri)) return data_uri @@ -174,7 +174,7 @@ class AzureAIOCRConfig(MistralOCRConfig): Returns: OCRRequestData with JSON data """ - verbose_logger.debug(f"Azure AI OCR transform_ocr_request (sync) - model: {model}") + verbose_logger.debug("Azure AI OCR transform_ocr_request (sync) - model: %s", model) if not isinstance(document, dict): raise ValueError(f"Expected document dict, got {type(document)}") @@ -231,7 +231,7 @@ class AzureAIOCRConfig(MistralOCRConfig): Returns: OCRRequestData with JSON data """ - verbose_logger.debug(f"Azure AI OCR async_transform_ocr_request - model: {model}") + verbose_logger.debug("Azure AI OCR async_transform_ocr_request - model: %s", model) if not isinstance(document, dict): raise ValueError(f"Expected document dict, got {type(document)}") diff --git a/litellm/llms/base_llm/files/azure_blob_storage_backend.py b/litellm/llms/base_llm/files/azure_blob_storage_backend.py index 33255657287..86229a16a63 100644 --- a/litellm/llms/base_llm/files/azure_blob_storage_backend.py +++ b/litellm/llms/base_llm/files/azure_blob_storage_backend.py @@ -129,11 +129,11 @@ class AzureBlobStorageBackend(BaseFileStorageBackend, AzureBlobStorageLogger): full_path=full_path, ) - verbose_logger.debug(f"Successfully uploaded file to Azure Blob Storage: {storage_url}") + verbose_logger.debug("Successfully uploaded file to Azure Blob Storage: %s", storage_url) return storage_url except Exception as e: - verbose_logger.exception(f"Error uploading file to Azure Blob Storage: {e}") + verbose_logger.exception("Error uploading file to Azure Blob Storage: %s", e) raise async def _upload_file_with_account_key(self, file_content: bytes, full_path: str) -> str: @@ -145,7 +145,7 @@ class AzureBlobStorageBackend(BaseFileStorageBackend, AzureBlobStorageLogger): # Create filesystem (container) if it doesn't exist if not await file_system_client.exists(): await file_system_client.create_file_system() - verbose_logger.debug(f"Created filesystem: {self.azure_storage_file_system}") + verbose_logger.debug("Created filesystem: %s", self.azure_storage_file_system) # Extract directory and filename (similar to logger's pattern) path_parts = full_path.split("/") @@ -157,7 +157,7 @@ class AzureBlobStorageBackend(BaseFileStorageBackend, AzureBlobStorageLogger): directory_client = file_system_client.get_directory_client(directory_path) if not await directory_client.exists(): await directory_client.create_directory() - verbose_logger.debug(f"Created directory: {directory_path}") + verbose_logger.debug("Created directory: %s", directory_path) # Get file client from directory (same pattern as logger) file_client = directory_client.get_file_client(file_name) @@ -247,7 +247,7 @@ class AzureBlobStorageBackend(BaseFileStorageBackend, AzureBlobStorageLogger): return await self._download_file_with_azure_ad(file_path) except Exception as e: - verbose_logger.exception(f"Error downloading file from Azure Blob Storage: {e}") + verbose_logger.exception("Error downloading file from Azure Blob Storage: %s", e) raise async def _download_file_with_account_key(self, file_path: str) -> bytes: diff --git a/litellm/llms/base_llm/files/storage_backend_factory.py b/litellm/llms/base_llm/files/storage_backend_factory.py index 8fd918af0dc..0cf8164bc4a 100644 --- a/litellm/llms/base_llm/files/storage_backend_factory.py +++ b/litellm/llms/base_llm/files/storage_backend_factory.py @@ -29,7 +29,7 @@ def get_storage_backend(backend_type: str) -> BaseFileStorageBackend: Raises: ValueError: If backend_type is not supported """ - verbose_logger.debug(f"Creating storage backend: type={backend_type}") + verbose_logger.debug("Creating storage backend: type=%s", backend_type) if backend_type == "azure_storage": return AzureBlobStorageBackend() diff --git a/litellm/llms/base_llm/managed_resources/base_managed_resource.py b/litellm/llms/base_llm/managed_resources/base_managed_resource.py index fd9eaf9801b..44ad555377b 100644 --- a/litellm/llms/base_llm/managed_resources/base_managed_resource.py +++ b/litellm/llms/base_llm/managed_resources/base_managed_resource.py @@ -156,7 +156,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]): user_api_key_dict: User API key authentication details additional_db_fields: Additional fields to store in database """ - verbose_logger.info(f"Storing LiteLLM Managed {self.resource_type} with id={unified_resource_id} in cache") + verbose_logger.info("Storing LiteLLM Managed %s with id=%s in cache", self.resource_type, unified_resource_id) # Prepare cache data cache_data = { @@ -215,7 +215,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]): result = await table.create(data=db_data) verbose_logger.debug( - f"LiteLLM Managed {self.resource_type} with id={unified_resource_id} stored in db: {result}" + "LiteLLM Managed %s with id=%s stored in db: %s", self.resource_type, unified_resource_id, result ) async def get_unified_resource_id( @@ -579,7 +579,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]): except Exception as e: verbose_logger.warning( - f"Failed to parse {self.resource_type} object {resource.unified_resource_id}: {e}" + "Failed to parse %s object %s: %s", self.resource_type, resource.unified_resource_id, e ) continue diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index d7b7806a4a8..bb5ee111f2e 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -816,7 +816,10 @@ class BaseAWSLLM: import boto3 verbose_logger.debug( - f"IN Web Identity Token: {aws_web_identity_token} | Role Name: {aws_role_name} | Session Name: {aws_session_name}" + "IN Web Identity Token: %s | Role Name: %s | Session Name: %s", + aws_web_identity_token, + aws_role_name, + aws_session_name, ) # get_secret() expands environment-variable references (an os.environ/ @@ -932,7 +935,8 @@ class BaseAWSLLM: if sts_response["PackedPolicySize"] > BEDROCK_MAX_POLICY_SIZE: verbose_logger.warning( - f"The policy size is greater than 75% of the allowed size, PackedPolicySize: {sts_response['PackedPolicySize']}" + "The policy size is greater than 75%% of the allowed size, PackedPolicySize: %s", + sts_response["PackedPolicySize"], ) with tracer.trace("boto3.Session(**iam_creds_dict)"): @@ -970,7 +974,7 @@ class BaseAWSLLM: sts_client = boto3.client("sts", **irsa_sts_kwargs) # Manually assume the IRSA role with the session name - verbose_logger.debug(f"Manually assuming IRSA role {irsa_role_arn} with session {aws_session_name}") + verbose_logger.debug("Manually assuming IRSA role %s with session %s", irsa_role_arn, aws_session_name) irsa_response = sts_client.assume_role_with_web_identity( RoleArn=irsa_role_arn, RoleSessionName=aws_session_name, @@ -994,13 +998,13 @@ class BaseAWSLLM: try: caller_identity = sts_client_with_creds.get_caller_identity() verbose_logger.debug( - f"Current identity after manual IRSA assumption: {caller_identity.get('Arn', 'unknown')}" + "Current identity after manual IRSA assumption: %s", caller_identity.get("Arn", "unknown") ) except Exception as e: - verbose_logger.debug(f"Failed to get caller identity: {e}") + verbose_logger.debug("Failed to get caller identity: %s", e) # Now assume the target role - verbose_logger.debug(f"Attempting to assume target role: {aws_role_name} with session: {aws_session_name}") + verbose_logger.debug("Attempting to assume target role: %s with session: %s", aws_role_name, aws_session_name) assume_role_params = { "RoleArn": aws_role_name, "RoleSessionName": aws_session_name, @@ -1035,12 +1039,12 @@ class BaseAWSLLM: # Get current caller identity for debugging try: caller_identity = sts_client.get_caller_identity() - verbose_logger.debug(f"Current IRSA identity: {caller_identity.get('Arn', 'unknown')}") + verbose_logger.debug("Current IRSA identity: %s", caller_identity.get("Arn", "unknown")) except Exception as e: - verbose_logger.debug(f"Failed to get caller identity: {e}") + verbose_logger.debug("Failed to get caller identity: %s", e) # Assume the role - verbose_logger.debug(f"Attempting to assume role: {aws_role_name} with session: {aws_session_name}") + verbose_logger.debug("Attempting to assume role: %s with session: %s", aws_role_name, aws_session_name) assume_role_params = { "RoleArn": aws_role_name, "RoleSessionName": aws_session_name, @@ -1142,7 +1146,7 @@ class BaseAWSLLM: if web_identity_token_file and irsa_role_arn and aws_access_key_id is None and aws_secret_access_key is None: # For cross-account role assumption with specific session names, # we need to manually assume the IRSA role first with the correct session name - verbose_logger.debug(f"IRSA detected: using web identity token from {web_identity_token_file}") + verbose_logger.debug("IRSA detected: using web identity token from %s", web_identity_token_file) try: # Check if we need to do cross-account role assumption @@ -1168,13 +1172,13 @@ class BaseAWSLLM: return self._extract_credentials_and_ttl(sts_response) except Exception as e: - verbose_logger.debug(f"Failed to assume role via IRSA: {e}") + verbose_logger.debug("Failed to assume role via IRSA: %s", e) if "AccessDenied" in str(e) and "is not authorized to perform: sts:AssumeRole" in str(e): # Provide a more helpful error message for trust policy issues verbose_logger.error( - f"Access denied when trying to assume role {aws_role_name}. " - f"Please ensure the trust policy of {aws_role_name} allows " - f"the current role to assume it. Current identity: check logs with verbose mode." + "Access denied when trying to assume role %s. Please ensure the trust policy of %s allows the current role to assume it. Current identity: check logs with verbose mode.", + aws_role_name, + aws_role_name, ) # Re-raise the exception instead of falling through raise diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index d6626562393..7efad7a906d 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -131,7 +131,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): jwt_token = api_key or optional_params.get("api_key") if jwt_token: verbose_logger.debug( - f"AgentCore: Using Bearer token authentication (Cognito/JWT) - token: {jwt_token[:50]}..." + "AgentCore: Using Bearer token authentication (Cognito/JWT) - token: %s...", jwt_token[:50] ) headers["Content-Type"] = "application/json" headers["Authorization"] = f"Bearer {jwt_token}" @@ -182,12 +182,12 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): """ session_id = optional_params.get("runtimeSessionId", None) if session_id: - verbose_logger.debug(f"Using provided runtimeSessionId: {session_id}") + verbose_logger.debug("Using provided runtimeSessionId: %s", session_id) return session_id # Generate a session ID with 33+ characters generated_id = f"litellm-session-{uuid.uuid4()}" - verbose_logger.debug(f"Generated new session ID: {generated_id}") + verbose_logger.debug("Generated new session ID: %s", generated_id) return generated_id def _get_runtime_user_id(self, optional_params: dict) -> str | None: @@ -196,7 +196,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): """ user_id = optional_params.get("runtimeUserId", None) if user_id: - verbose_logger.debug(f"Using provided runtimeUserId: {user_id}") + verbose_logger.debug("Using provided runtimeUserId: %s", user_id) return user_id def transform_request( @@ -231,7 +231,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): dict: Payload dict containing the prompt and (optionally) the OpenAI content list. """ - verbose_logger.debug(f"AgentCore transform_request - optional_params keys: {list(optional_params.keys())}") + verbose_logger.debug("AgentCore transform_request - optional_params keys: %s", list(optional_params.keys())) # Use the last message content as the prompt prompt = convert_content_list_to_str(messages[-1]) @@ -264,7 +264,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): # The request data is the payload dict (will be JSON encoded by the HTTP handler) # Qualifier will be handled as a query parameter in get_complete_url - verbose_logger.debug(f"PAYLOAD: {payload}") + verbose_logger.debug("PAYLOAD: %s", payload) return payload @staticmethod @@ -302,7 +302,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): # Skip non-dict data (some lines contain JSON strings) return data if isinstance(data, dict) else None except json.JSONDecodeError: - verbose_logger.debug(f"Skipping non-JSON line: {line[:100]}") + verbose_logger.debug("Skipping non-JSON line: %s", line[:100]) return None def _extract_usage_from_event(self, event_data: dict) -> AgentCoreUsage | None: @@ -361,7 +361,10 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): total_tokens = prompt_tokens + completion_tokens verbose_logger.debug( - f"Calculated usage - prompt: {prompt_tokens}, completion: {completion_tokens}, total: {total_tokens}" + "Calculated usage - prompt: %s, completion: %s, total: %s", + prompt_tokens, + completion_tokens, + total_tokens, ) return Usage( @@ -370,7 +373,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): total_tokens=total_tokens, ) except Exception as e: - verbose_logger.warning(f"Failed to calculate token usage: {e}") + verbose_logger.warning("Failed to calculate token usage: %s", e) return None def _parse_json_response(self, response_json: dict) -> AgentCoreParsedResponse: @@ -439,8 +442,8 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): # Strategy 4: fallback - return raw JSON as content verbose_logger.warning( - f"AgentCore: Could not extract content from JSON response keys " - f"{list(response_json.keys())}. Returning raw JSON as content." + "AgentCore: Could not extract content from JSON response keys %s. Returning raw JSON as content.", + list(response_json.keys()), ) return AgentCoreParsedResponse( content=json.dumps(response_json), @@ -459,20 +462,20 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): AgentCoreParsedResponse: Parsed response data """ content_type = raw_response.headers.get("content-type", "").lower() - verbose_logger.debug(f"AgentCore response Content-Type: {content_type}") + verbose_logger.debug("AgentCore response Content-Type: %s", content_type) # Parse response based on content type if "application/json" in content_type: # Direct JSON response verbose_logger.debug("Parsing JSON response") response_json = raw_response.json() - verbose_logger.debug(f"Response JSON: {response_json}") + verbose_logger.debug("Response JSON: %s", response_json) return self._parse_json_response(response_json) else: # SSE stream response (text/event-stream or default) verbose_logger.debug("Parsing SSE stream response") response_text = raw_response.text - verbose_logger.debug(f"AgentCore response (first 500 chars): {response_text[:500]}") + verbose_logger.debug("AgentCore response (first 500 chars): %s", response_text[:500]) return self._parse_sse_stream(response_text) def _parse_sse_stream(self, response_text: str) -> AgentCoreParsedResponse: @@ -496,7 +499,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): if not data: continue - verbose_logger.debug(f"SSE event keys: {list(data.keys())}") + verbose_logger.debug("SSE event keys: %s", list(data.keys())) # Check for final complete message if "message" in data and isinstance(data["message"], dict): @@ -506,12 +509,12 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): # Process event data if "event" in data and isinstance(data["event"], dict): event_payload = data["event"] - verbose_logger.debug(f"Event payload keys: {list(event_payload.keys())}") + verbose_logger.debug("Event payload keys: %s", list(event_payload.keys())) # Extract usage metadata if usage := self._extract_usage_from_event(data): usage_data = usage - verbose_logger.debug(f"Found usage data: {usage_data}") + verbose_logger.debug("Found usage data: %s", usage_data) # Collect content deltas if text := self._extract_content_delta(data): @@ -520,7 +523,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): # Build final content content = self._extract_content_from_message(final_message) if final_message else "".join(content_blocks) - verbose_logger.debug(f"Final usage_data: {usage_data}") + verbose_logger.debug("Final usage_data: %s", usage_data) return AgentCoreParsedResponse(content=content, usage=usage_data, final_message=final_message) @@ -624,7 +627,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): yield chunk except json.JSONDecodeError: - verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}") + verbose_logger.debug("Skipping non-JSON SSE line: %s", line[:100]) continue def get_sync_custom_stream_wrapper( @@ -651,7 +654,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): if client is None or not isinstance(client, HTTPHandler): client = _get_httpx_client(params={}) - verbose_logger.debug(f"Making sync streaming request to: {api_base}") + verbose_logger.debug("Making sync streaming request to: %s", api_base) # Make streaming request response = client.post( @@ -837,7 +840,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): yield chunk except json.JSONDecodeError: - verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}") + verbose_logger.debug("Skipping non-JSON SSE line: %s", line[:100]) continue async def get_async_custom_stream_wrapper( @@ -864,7 +867,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): if client is None or not isinstance(client, AsyncHTTPHandler): client = get_async_httpx_client(llm_provider=cast(Any, "bedrock"), params={}) - verbose_logger.debug(f"Making async streaming request to: {api_base}") + verbose_logger.debug("Making async streaming request to: %s", api_base) # Make async streaming request response = await client.post( @@ -990,8 +993,8 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): content = parsed_data["content"] usage_data = parsed_data["usage"] - verbose_logger.debug(f"Parsed content length: {len(content)}") - verbose_logger.debug(f"Usage data: {usage_data}") + verbose_logger.debug("Parsed content length: %s", len(content)) + verbose_logger.debug("Usage data: %s", usage_data) # Create the message message = Message(content=content, role="assistant") @@ -1023,7 +1026,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): return model_response except Exception as e: - verbose_logger.error(f"Error processing Bedrock AgentCore response: {e}") + verbose_logger.error("Error processing Bedrock AgentCore response: %s", e) raise BedrockError( message=f"Error processing response: {e}", status_code=raw_response.status_code, diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 2b34c9f2654..5096cc44b76 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -352,8 +352,8 @@ class AmazonConverseConfig(BaseConfig): # Model strings can be like: "amazon.nova-pro-v1:0", "us.amazon.nova-pro-v1:0", etc. if "nova" not in model.lower(): verbose_logger.debug( - f"web_search_options passed but model {model} is not a Nova model. " - "Nova grounding is only supported on Amazon Nova models." + "web_search_options passed but model %s is not a Nova model. Nova grounding is only supported on Amazon Nova models.", + model, ) return None @@ -950,8 +950,8 @@ class AmazonConverseConfig(BaseConfig): if isinstance(tool_choice_block, dict): if "any" in tool_choice_block or "tool" in tool_choice_block: verbose_logger.info( - f"{model} does not support forced tool use (tool_choice='required' or specific tool) " - f"when reasoning is enabled. Changing tool_choice to 'auto'." + "%s does not support forced tool use (tool_choice='required' or specific tool) when reasoning is enabled. Changing tool_choice to 'auto'.", + model, ) optional_params["tool_choice"] = ToolChoiceValuesBlock(auto={}) diff --git a/litellm/llms/bedrock/chat/invoke_agent/transformation.py b/litellm/llms/bedrock/chat/invoke_agent/transformation.py index da6224ec487..625ace20614 100644 --- a/litellm/llms/bedrock/chat/invoke_agent/transformation.py +++ b/litellm/llms/bedrock/chat/invoke_agent/transformation.py @@ -212,12 +212,12 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): } events.append(parsed_event) except json.JSONDecodeError as e: - verbose_logger.warning(f"Failed to parse trace event JSON: {e}") + verbose_logger.warning("Failed to parse trace event JSON: %s", e) else: - verbose_logger.debug(f"Unknown event type: {event_type}") + verbose_logger.debug("Unknown event type: %s", event_type) except Exception as e: - verbose_logger.error(f"Error processing event: {e}") + verbose_logger.error("Error processing event: %s", e) continue return events @@ -226,11 +226,11 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): """Extract message content from an AWS event, adapted from AWSEventStreamDecoder.""" try: response_dict = event.to_response_dict() - verbose_logger.debug(f"Response dict: {response_dict}") + verbose_logger.debug("Response dict: %s", response_dict) # Use the same response shape parsing as the existing decoder parsed_response = parser.parse(response_dict, self._get_response_stream_shape()) - verbose_logger.debug(f"Parsed response: {parsed_response}") + verbose_logger.debug("Parsed response: %s", parsed_response) if response_dict["status_code"] != 200: decoded_body = response_dict["body"].decode() @@ -259,7 +259,7 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): return chunk.decode() except Exception as e: - verbose_logger.debug(f"Error parsing message from event: {e}") + verbose_logger.debug("Error parsing message from event: %s", e) return None def _extract_headers_from_event(self, event) -> InvokeAgentEventHeaders: @@ -275,7 +275,7 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): message_type=headers.get(":message-type", ""), ) except Exception as e: - verbose_logger.debug(f"Error extracting headers: {e}") + verbose_logger.debug("Error extracting headers: %s", e) return InvokeAgentEventHeaders(event_type="", content_type="", message_type="") def _get_response_stream_shape(self): @@ -302,7 +302,7 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): decoded_content = base64.b64decode(encoded_bytes).decode("utf-8") response_parts.append(decoded_content) except Exception as e: - verbose_logger.warning(f"Failed to decode chunk content: {e}") + verbose_logger.warning("Failed to decode chunk content: %s", e) return "".join(response_parts) @@ -324,7 +324,7 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): if not trace_data: continue - verbose_logger.debug(f"Trace event: {trace_data}") + verbose_logger.debug("Trace event: %s", trace_data) # Extract usage from pre-processing trace self._extract_and_update_preprocessing_usage( @@ -443,11 +443,11 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): try: # Get the raw binary content raw_content = raw_response.content - verbose_logger.debug(f"Processing {len(raw_content)} bytes of AWS event stream data") + verbose_logger.debug("Processing %s bytes of AWS event stream data", len(raw_content)) # Parse the AWS event stream format events = self._parse_aws_event_stream(raw_content) - verbose_logger.debug(f"Parsed {len(events)} events from stream") + verbose_logger.debug("Parsed %s events from stream", len(events)) # Extract response content from chunk events content = self._extract_response_content(events) @@ -464,7 +464,7 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): ) except Exception as e: - verbose_logger.error(f"Error processing Bedrock Invoke Agent response: {e}") + verbose_logger.error("Error processing Bedrock Invoke Agent response: %s", e) raise BedrockError( message=f"Error processing response: {e}", status_code=raw_response.status_code, diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 4a429b639d2..a5ffe6abff4 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -530,7 +530,7 @@ class AWSEventStreamDecoder: # and use it as the consistent ID for all subsequent chunks. self._initialize_converse_response_id(chunk_data) - verbose_logger.debug(f"\n\nRaw Chunk: {chunk_data}\n\n") + verbose_logger.debug("\n\nRaw Chunk: %s\n\n", chunk_data) text = "" tool_use: ChatCompletionToolCallChunk | None = None finish_reason = "" diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py index 934d416d256..23998c62644 100644 --- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -88,7 +88,7 @@ class BedrockTokenCounter(BaseTokenCounter): original_response=result, ) except BedrockError as e: - verbose_logger.warning(f"Bedrock CountTokens API error: status={e.status_code}, message={e.message}") + verbose_logger.warning("Bedrock CountTokens API error: status=%s, message=%s", e.status_code, e.message) return TokenCountResponse( total_tokens=0, request_model=request_model, @@ -99,7 +99,7 @@ class BedrockTokenCounter(BaseTokenCounter): status_code=e.status_code, ) except Exception as e: - verbose_logger.warning(f"Error calling Bedrock CountTokens API: {e}") + verbose_logger.warning("Error calling Bedrock CountTokens API: %s", e) return TokenCountResponse( total_tokens=0, request_model=request_model, diff --git a/litellm/llms/bedrock/count_tokens/handler.py b/litellm/llms/bedrock/count_tokens/handler.py index 44cc535385d..100f0753b51 100644 --- a/litellm/llms/bedrock/count_tokens/handler.py +++ b/litellm/llms/bedrock/count_tokens/handler.py @@ -43,7 +43,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): # Validate the request self.validate_count_tokens_request(request_data) - verbose_logger.debug(f"Processing CountTokens request for resolved model: {resolved_model}") + verbose_logger.debug("Processing CountTokens request for resolved model: %s", resolved_model) # Get AWS region using existing LiteLLM function aws_region_name = self._get_aws_region_name( @@ -52,12 +52,12 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): model_id=None, ) - verbose_logger.debug(f"Retrieved AWS region: {aws_region_name}") + verbose_logger.debug("Retrieved AWS region: %s", aws_region_name) # Transform request to Bedrock format (supports both Converse and InvokeModel) bedrock_request = self.transform_anthropic_to_bedrock_count_tokens(request_data=request_data) - verbose_logger.debug(f"Transformed request: {bedrock_request}") + verbose_logger.debug("Transformed request: %s", bedrock_request) # Get endpoint URL using simplified function api_base = litellm_params.get("api_base", None) @@ -69,7 +69,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, ) - verbose_logger.debug(f"Making request to: {endpoint_url}") + verbose_logger.debug("Making request to: %s", endpoint_url) # Use existing _sign_request method from BaseAWSLLM # Extract api_key for bearer token auth if provided @@ -94,11 +94,11 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): timeout=30.0, ) - verbose_logger.debug(f"Response status: {response.status_code}") + verbose_logger.debug("Response status: %s", response.status_code) if response.status_code != 200: error_text = response.text - verbose_logger.error(f"AWS Bedrock error: {error_text}") + verbose_logger.error("AWS Bedrock error: %s", error_text) raise BedrockError( status_code=response.status_code, message=error_text, @@ -106,12 +106,12 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): bedrock_response = response.json() - verbose_logger.debug(f"Bedrock response: {bedrock_response}") + verbose_logger.debug("Bedrock response: %s", bedrock_response) # Transform response back to expected format final_response = self.transform_bedrock_response_to_anthropic(bedrock_response) - verbose_logger.debug(f"Final response: {final_response}") + verbose_logger.debug("Final response: %s", final_response) return final_response @@ -120,13 +120,13 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): raise except httpx.HTTPStatusError as e: # HTTP errors - preserve the actual status code - verbose_logger.error(f"HTTP error in CountTokens handler: {e}") + verbose_logger.error("HTTP error in CountTokens handler: %s", e) raise BedrockError( status_code=e.response.status_code, message=e.response.text, ) except Exception as e: - verbose_logger.error(f"Error in CountTokens handler: {e}") + verbose_logger.error("Error in CountTokens handler: %s", e) raise BedrockError( status_code=500, message=f"CountTokens processing error: {e}", diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index d3e61829681..9f61c50b25f 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -652,7 +652,8 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): ) except Exception as e: verbose_logger.exception( - f"litellm.llms.bedrock.files.transformation.py::_transform_openai_jsonl_content_to_bedrock_jsonl_content() - Error inferring custom_llm_provider - {e}" + "litellm.llms.bedrock.files.transformation.py::_transform_openai_jsonl_content_to_bedrock_jsonl_content() - Error inferring custom_llm_provider - %s", + e, ) # Determine provider from model name diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index a8969894dda..874052d2b29 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -83,7 +83,7 @@ class BedrockRealtime(BaseAWSLLM): else: endpoint_uri = f"https://bedrock-runtime.{aws_region_name}.amazonaws.com" - verbose_proxy_logger.debug(f"Bedrock Realtime: Connecting to {endpoint_uri} with model {model}") + verbose_proxy_logger.debug("Bedrock Realtime: Connecting to %s with model %s", endpoint_uri, model) credentials = self.get_credentials( aws_access_key_id=aws_access_key_id, @@ -173,7 +173,7 @@ class BedrockRealtime(BaseAWSLLM): ) except Exception as e: - verbose_proxy_logger.exception(f"Error in BedrockRealtime.async_realtime: {e}") + verbose_proxy_logger.exception("Error in BedrockRealtime.async_realtime: %s", e) try: await websocket.close(code=1011, reason=_redact_string(f"Internal error: {e}")) except Exception: @@ -200,13 +200,13 @@ class BedrockRealtime(BaseAWSLLM): value=BidirectionalInputPayloadPart(bytes_=bedrock_message.encode("utf-8")) ) await bedrock_stream.input_stream.send(event) - verbose_proxy_logger.debug(f"Bedrock Realtime: Sent to Bedrock: {bedrock_message[:200]}") + verbose_proxy_logger.debug("Bedrock Realtime: Sent to Bedrock: %s", bedrock_message[:200]) try: while True: # Receive message from client message = await client_ws.receive_text() - verbose_proxy_logger.debug(f"Bedrock Realtime: Received from client: {message[:200]}") + verbose_proxy_logger.debug("Bedrock Realtime: Received from client: %s", message[:200]) # Transform OpenAI format to Bedrock format transformed_messages = transformation_config.transform_realtime_request( @@ -237,7 +237,7 @@ class BedrockRealtime(BaseAWSLLM): ) except Exception as e: - verbose_proxy_logger.debug(f"Client to Bedrock forwarding ended: {e}", exc_info=True) + verbose_proxy_logger.debug("Client to Bedrock forwarding ended: %s", e, exc_info=True) for close_message in transformation_config.session_close_messages(): with contextlib.suppress(Exception): await send_to_bedrock(close_message) @@ -266,7 +266,7 @@ class BedrockRealtime(BaseAWSLLM): if result.value and result.value.bytes_: bedrock_response = result.value.bytes_.decode("utf-8") - verbose_proxy_logger.debug(f"Bedrock Realtime: Received from Bedrock: {bedrock_response[:200]}") + verbose_proxy_logger.debug("Bedrock Realtime: Received from Bedrock: %s", bedrock_response[:200]) # Transform Bedrock format to OpenAI format from litellm.types.realtime import RealtimeResponseTransformInput @@ -306,10 +306,10 @@ class BedrockRealtime(BaseAWSLLM): for openai_message in openai_messages: message_json = json.dumps(openai_message) await client_ws.send_text(message_json) - verbose_proxy_logger.debug(f"Bedrock Realtime: Sent to client: {message_json[:200]}") + verbose_proxy_logger.debug("Bedrock Realtime: Sent to client: %s", message_json[:200]) except Exception as e: - verbose_proxy_logger.debug(f"Bedrock to client forwarding ended: {e}", exc_info=True) + verbose_proxy_logger.debug("Bedrock to client forwarding ended: %s", e, exc_info=True) finally: # Close the client WebSocket try: diff --git a/litellm/llms/bedrock/realtime/transformation.py b/litellm/llms/bedrock/realtime/transformation.py index 39f5d25cf89..68782c8b412 100644 --- a/litellm/llms/bedrock/realtime/transformation.py +++ b/litellm/llms/bedrock/realtime/transformation.py @@ -601,7 +601,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): try: json_message = json.loads(message) except json.JSONDecodeError: - verbose_logger.warning(f"Invalid JSON message: {message[:200]}") + verbose_logger.warning("Invalid JSON message: %s", message[:200]) return [] message_type = json_message.get("type") @@ -620,7 +620,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): elif message_type == "response.cancel": return self.transform_response_cancel_event(json_message) else: - verbose_logger.warning(f"Unknown message type: {message_type}") + verbose_logger.warning("Unknown message type: %s", message_type) return [] def _session_object( @@ -866,7 +866,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): Tuple of (events, reset_delta_chunks) """ content_end = event["contentEnd"] - verbose_logger.debug(f"Handling contentEnd: {content_end}") + verbose_logger.debug("Handling contentEnd: %s", content_end) if not current_output_item_id or not current_response_id: return [], current_delta_chunks @@ -1149,7 +1149,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): message_preview = ( message[:200].decode("utf-8", errors="replace") if isinstance(message, bytes) else message[:200] ) - verbose_logger.warning(f"Invalid JSON message: {message_preview}") + verbose_logger.warning("Invalid JSON message: %s", message_preview) return { "response": [], "current_output_item_id": realtime_response_transform_input.get("current_output_item_id"), @@ -1230,7 +1230,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): ) returned_messages.extend(events) # Store tool call info for potential use - verbose_logger.debug(f"Tool use event: {tool_name} (ID: {tool_call_id})") + verbose_logger.debug("Tool use event: %s (ID: %s)", tool_name, tool_call_id) elif "promptEnd" in event or "completionEnd" in event: ( diff --git a/litellm/llms/bedrock_mantle/chat/transformation.py b/litellm/llms/bedrock_mantle/chat/transformation.py index d85edbd0c86..bd1664ff95a 100644 --- a/litellm/llms/bedrock_mantle/chat/transformation.py +++ b/litellm/llms/bedrock_mantle/chat/transformation.py @@ -102,7 +102,7 @@ class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig): if "reasoning_effort" not in base_params: base_params.append("reasoning_effort") except Exception as e: - verbose_logger.debug(f"BedrockMantleChatConfig: error checking reasoning support: {e}") + verbose_logger.debug("BedrockMantleChatConfig: error checking reasoning support: %s", e) return base_params def get_model_response_iterator( diff --git a/litellm/llms/black_forest_labs/image_edit/handler.py b/litellm/llms/black_forest_labs/image_edit/handler.py index cf0cc31283b..6a6c01e8b24 100644 --- a/litellm/llms/black_forest_labs/image_edit/handler.py +++ b/litellm/llms/black_forest_labs/image_edit/handler.py @@ -342,7 +342,7 @@ class BlackForestLabsImageEdit: polling_headers = {"x-key": headers.get("x-key", "")} start_time = time.time() - verbose_logger.debug(f"BFL starting sync polling at {polling_url}") + verbose_logger.debug("BFL starting sync polling at %s", polling_url) while time.time() - start_time < max_wait: response = sync_client.get( @@ -359,7 +359,7 @@ class BlackForestLabsImageEdit: data = response.json() status = data.get("status") - verbose_logger.debug(f"BFL poll status: {status}") + verbose_logger.debug("BFL poll status: %s", status) if status == "Ready": return response @@ -433,7 +433,7 @@ class BlackForestLabsImageEdit: polling_headers = {"x-key": headers.get("x-key", "")} start_time = time.time() - verbose_logger.debug(f"BFL starting async polling at {polling_url}") + verbose_logger.debug("BFL starting async polling at %s", polling_url) while time.time() - start_time < max_wait: response = await async_client.get( @@ -450,7 +450,7 @@ class BlackForestLabsImageEdit: data = response.json() status = data.get("status") - verbose_logger.debug(f"BFL poll status: {status}") + verbose_logger.debug("BFL poll status: %s", status) if status == "Ready": return response diff --git a/litellm/llms/black_forest_labs/image_generation/handler.py b/litellm/llms/black_forest_labs/image_generation/handler.py index 054d28003f1..4df3b49d7f1 100644 --- a/litellm/llms/black_forest_labs/image_generation/handler.py +++ b/litellm/llms/black_forest_labs/image_generation/handler.py @@ -336,7 +336,7 @@ class BlackForestLabsImageGeneration: polling_headers = {"x-key": headers.get("x-key", "")} start_time = time.time() - verbose_logger.debug(f"BFL starting sync polling at {polling_url}") + verbose_logger.debug("BFL starting sync polling at %s", polling_url) while time.time() - start_time < max_wait: response = sync_client.get( @@ -353,7 +353,7 @@ class BlackForestLabsImageGeneration: data = response.json() status = data.get("status") - verbose_logger.debug(f"BFL poll status: {status}") + verbose_logger.debug("BFL poll status: %s", status) if status == "Ready": return response @@ -427,7 +427,7 @@ class BlackForestLabsImageGeneration: polling_headers = {"x-key": headers.get("x-key", "")} start_time = time.time() - verbose_logger.debug(f"BFL starting async polling at {polling_url}") + verbose_logger.debug("BFL starting async polling at %s", polling_url) while time.time() - start_time < max_wait: response = await async_client.get( @@ -444,7 +444,7 @@ class BlackForestLabsImageGeneration: data = response.json() status = data.get("status") - verbose_logger.debug(f"BFL poll status: {status}") + verbose_logger.debug("BFL poll status: %s", status) if status == "Ready": return response diff --git a/litellm/llms/custom_httpx/aiohttp_transport.py b/litellm/llms/custom_httpx/aiohttp_transport.py index ac7a8908616..9ac67272b3e 100644 --- a/litellm/llms/custom_httpx/aiohttp_transport.py +++ b/litellm/llms/custom_httpx/aiohttp_transport.py @@ -289,7 +289,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport): if self._owns_session: self._close_recycled_session(old_session) except Exception as e: - verbose_logger.debug(f"Error closing old session: {e}") + verbose_logger.debug("Error closing old session: %s", e) # Create a new session in the current event loop self.client = self._rebuild_session() @@ -302,9 +302,9 @@ class LiteLLMAiohttpTransport(AiohttpTransport): try: self._close_recycled_session(old_session) except (RuntimeError, AttributeError, OSError) as close_error: - verbose_logger.debug(f"Error closing old session: {close_error}") + verbose_logger.debug("Error closing old session: %s", close_error) self.client = self._rebuild_session() - verbose_logger.debug(f"Error checking session loop, created new session: {e}") + verbose_logger.debug("Error checking session loop, created new session: %s", e) return self.client @@ -397,7 +397,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport): except RuntimeError as e: # Handle the case where session was closed between our check and actual use if "Session is closed" in str(e): - verbose_logger.debug(f"Session closed during request, retrying with new session: {e}") + verbose_logger.debug("Session closed during request, retrying with new session: %s", e) # Dispose of the session that actually faulted. Do NOT read # self.client here: a concurrent task may already have # replaced it with a healthy session that must stay open. @@ -436,7 +436,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport): try: proxy = self._proxy_from_env(request.url) except Exception as e: # pragma: no cover - best effort - verbose_logger.debug(f"Error reading proxy env: {e}") + verbose_logger.debug("Error reading proxy env: %s", e) return proxy diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 3e34b483002..861adf919c0 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -215,19 +215,20 @@ def _create_ssl_context( if ssl_ecdh_curve and isinstance(ssl_ecdh_curve, str): try: custom_ssl_context.set_ecdh_curve(ssl_ecdh_curve) - verbose_logger.debug(f"SSL ECDH curve set to: {ssl_ecdh_curve}") + verbose_logger.debug("SSL ECDH curve set to: %s", ssl_ecdh_curve) except AttributeError: verbose_logger.warning( - f"SSL ECDH curve configuration not supported. " - f"Python version: {sys.version.split()[0]}, OpenSSL version: {ssl.OPENSSL_VERSION}. " - f"Requested curve: {ssl_ecdh_curve}. Continuing with default curves." + "SSL ECDH curve configuration not supported. Python version: %s, OpenSSL version: %s. Requested curve: %s. Continuing with default curves.", + sys.version.split()[0], + ssl.OPENSSL_VERSION, + ssl_ecdh_curve, ) except ValueError as e: # Invalid curve name verbose_logger.warning( - f"Invalid SSL ECDH curve name: '{ssl_ecdh_curve}'. {e}. " - f"Common valid curves: X25519, prime256v1, secp384r1, secp521r1. " - f"Continuing with default curves (including PQC)." + "Invalid SSL ECDH curve name: '%s'. %s. Common valid curves: X25519, prime256v1, secp384r1, secp521r1. Continuing with default curves (including PQC).", + ssl_ecdh_curve, + e, ) return custom_ssl_context @@ -1033,7 +1034,7 @@ class AsyncHTTPHandler: # Use shared session if provided and valid if shared_session is not None and not shared_session.closed: - verbose_logger.debug(f"SHARED SESSION: Reusing existing ClientSession (ID: {id(shared_session)})") + verbose_logger.debug("SHARED SESSION: Reusing existing ClientSession (ID: %s)", id(shared_session)) return LiteLLMAiohttpTransport( client=shared_session, ssl_verify=ssl_for_transport, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index f7bf174f9ac..fcb039c9b4f 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -370,7 +370,7 @@ class BaseLLMHTTPHandler: ): if client is None: verbose_logger.debug( - f"Creating HTTP client with shared_session: {id(shared_session) if shared_session else None}" + "Creating HTTP client with shared_session: %s", id(shared_session) if shared_session else None ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), @@ -2676,7 +2676,8 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): verbose_logger.debug( - f"Creating HTTP client for responses API with shared_session: {id(shared_session) if shared_session else None}" + "Creating HTTP client for responses API with shared_session: %s", + id(shared_session) if shared_session else None, ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), @@ -2859,7 +2860,8 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): verbose_logger.debug( - f"Creating HTTP client for delete_response with shared_session: {id(shared_session) if shared_session else None}" + "Creating HTTP client for delete_response with shared_session: %s", + id(shared_session) if shared_session else None, ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), @@ -3112,7 +3114,8 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): verbose_logger.debug( - f"Creating HTTP client for get_responses with shared_session: {id(shared_session) if shared_session else None}" + "Creating HTTP client for get_responses with shared_session: %s", + id(shared_session) if shared_session else None, ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), @@ -3156,7 +3159,7 @@ class BaseLLMHTTPHandler: response = await async_httpx_client.get(url=url, headers=headers, params=data) response.raise_for_status() except Exception as e: - verbose_logger.debug(f"Error retrieving response: {e}") + verbose_logger.debug("Error retrieving response: %s", e) raise self._handle_error( e=e, provider_config=responses_api_provider_config, @@ -3275,7 +3278,8 @@ class BaseLLMHTTPHandler: ) -> dict: if client is None or not isinstance(client, AsyncHTTPHandler): verbose_logger.debug( - f"Creating HTTP client for list_input_items with shared_session: {id(shared_session) if shared_session else None}" + "Creating HTTP client for list_input_items with shared_session: %s", + id(shared_session) if shared_session else None, ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), @@ -3495,7 +3499,7 @@ class BaseLLMHTTPHandler: timeout=timeout, ) except Exception as e: - verbose_logger.exception(f"Error creating file: {e}") + verbose_logger.exception("Error creating file: %s", e) raise self._handle_error(e=e, provider_config=provider_config) elif isinstance(transformed_request, str) or isinstance(transformed_request, bytes): # Handle traditional file uploads @@ -3626,7 +3630,7 @@ class BaseLLMHTTPHandler: if initial_response_data: litellm_params["initial_file_response"] = initial_response_data except Exception as e: - verbose_logger.exception(f"Error creating file: {e}") + verbose_logger.exception("Error creating file: %s", e) raise self._handle_error( e=e, provider_config=provider_config, @@ -3657,7 +3661,7 @@ class BaseLLMHTTPHandler: timeout=timeout, ) except Exception as e: - verbose_logger.exception(f"Error creating file: {e}") + verbose_logger.exception("Error creating file: %s", e) raise self._handle_error(e=e, provider_config=provider_config) elif isinstance(transformed_request, str) or isinstance(transformed_request, bytes): # Handle traditional file uploads @@ -3868,7 +3872,7 @@ class BaseLLMHTTPHandler: timeout=timeout, ) except Exception as e: - verbose_logger.exception(f"Error creating batch: {e}") + verbose_logger.exception("Error creating batch: %s", e) raise self._handle_error( e=e, provider_config=provider_config, @@ -3960,7 +3964,7 @@ class BaseLLMHTTPHandler: headers=headers, ) except Exception as e: - verbose_logger.exception(f"Error retrieving batch: {e}") + verbose_logger.exception("Error retrieving batch: %s", e) raise self._handle_error( e=e, provider_config=provider_config, @@ -4033,7 +4037,7 @@ class BaseLLMHTTPHandler: timeout=timeout, ) except Exception as e: - verbose_logger.exception(f"Error creating batch: {e}") + verbose_logger.exception("Error creating batch: %s", e) raise self._handle_error( e=e, provider_config=provider_config, @@ -4117,7 +4121,7 @@ class BaseLLMHTTPHandler: headers=headers, ) except Exception as e: - verbose_logger.exception(f"Error retrieving batch: {e}") + verbose_logger.exception("Error retrieving batch: %s", e) raise self._handle_error( e=e, provider_config=provider_config, @@ -4230,7 +4234,8 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): verbose_logger.debug( - f"Creating HTTP client for cancel_response with shared_session: {id(shared_session) if shared_session else None}" + "Creating HTTP client for cancel_response with shared_session: %s", + id(shared_session) if shared_session else None, ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), @@ -4404,7 +4409,8 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): verbose_logger.debug( - f"Creating HTTP client for compact_response with shared_session: {id(shared_session) if shared_session else None}" + "Creating HTTP client for compact_response with shared_session: %s", + id(shared_session) if shared_session else None, ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), @@ -5659,7 +5665,7 @@ class BaseLLMHTTPHandler: fingerprint=fingerprint, ) except Exception as e: - verbose_logger.exception(f"LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: {e}") + verbose_logger.exception("LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: %s", e) # Check if we need to convert response to fake stream for chat completions # This happens when: @@ -5901,10 +5907,10 @@ class BaseLLMHTTPHandler: await realtime_streaming.bidirectional_forward() except websockets.exceptions.InvalidStatusCode as e: # type: ignore - verbose_logger.exception(f"Error connecting to backend: {e}") + verbose_logger.exception("Error connecting to backend: %s", e) await websocket.close(code=e.status_code, reason=_redact_string(str(e))) except Exception as e: - verbose_logger.exception(f"Error connecting to backend: {e}") + verbose_logger.exception("Error connecting to backend: %s", e) try: await websocket.close(code=1011, reason=_redact_string(f"Internal server error: {e}")) except RuntimeError as close_error: @@ -6298,10 +6304,10 @@ class BaseLLMHTTPHandler: await streaming.bidirectional_forward() except websockets.exceptions.InvalidStatusCode as e: # type: ignore - verbose_logger.exception(f"Error connecting to responses WS backend: {e}") + verbose_logger.exception("Error connecting to responses WS backend: %s", e) await websocket.close(code=e.status_code, reason=_redact_string(str(e))) except Exception as e: - verbose_logger.exception(f"Error in responses WS: {e}") + verbose_logger.exception("Error in responses WS: %s", e) try: await websocket.close(code=1011, reason=_redact_string(f"Internal server error: {e}")) except RuntimeError as close_error: diff --git a/litellm/llms/databricks/common_utils.py b/litellm/llms/databricks/common_utils.py index 62e2245db99..d69b97b32a1 100644 --- a/litellm/llms/databricks/common_utils.py +++ b/litellm/llms/databricks/common_utils.py @@ -386,7 +386,7 @@ class DatabricksBase: headers["User-Agent"] = self._build_user_agent(custom_user_agent) # Debug logging with redaction (never log actual tokens) - verbose_logger.debug(f"Databricks request headers: {self.redact_headers_for_logging(headers)}") + verbose_logger.debug("Databricks request headers: %s", self.redact_headers_for_logging(headers)) if endpoint_type == "chat_completions" and custom_endpoint is not True: api_base = f"{api_base}/chat/completions" diff --git a/litellm/llms/databricks/streaming_utils.py b/litellm/llms/databricks/streaming_utils.py index 74216888111..635aa04b0be 100644 --- a/litellm/llms/databricks/streaming_utils.py +++ b/litellm/llms/databricks/streaming_utils.py @@ -126,7 +126,9 @@ class ModelResponseIterator: except StopIteration: raise StopIteration except ValueError as e: - verbose_logger.debug(f"Error parsing chunk: {e},\nReceived chunk: {chunk}. Defaulting to empty chunk here.") + verbose_logger.debug( + "Error parsing chunk: %s,\nReceived chunk: %s. Defaulting to empty chunk here.", e, chunk + ) return GenericStreamingChunk( text="", is_finished=False, @@ -171,7 +173,9 @@ class ModelResponseIterator: except StopAsyncIteration: raise StopAsyncIteration except ValueError as e: - verbose_logger.debug(f"Error parsing chunk: {e},\nReceived chunk: {chunk}. Defaulting to empty chunk here.") + verbose_logger.debug( + "Error parsing chunk: %s,\nReceived chunk: %s. Defaulting to empty chunk here.", e, chunk + ) return GenericStreamingChunk( text="", is_finished=False, diff --git a/litellm/llms/gemini/files/transformation.py b/litellm/llms/gemini/files/transformation.py index 89ac56979bb..274e5474101 100644 --- a/litellm/llms/gemini/files/transformation.py +++ b/litellm/llms/gemini/files/transformation.py @@ -190,7 +190,7 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): status_details=None, ) except Exception as e: - verbose_logger.exception(f"Error parsing file upload response: {e}") + verbose_logger.exception("Error parsing file upload response: %s", e) raise ValueError(f"Error parsing file upload response: {e}") def transform_retrieve_file_request( @@ -263,9 +263,9 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): Transform Gemini's file retrieval response into OpenAI-style FileObject """ try: - verbose_logger.debug(f"Retrieve file response: {raw_response.text}") + verbose_logger.debug("Retrieve file response: %s", raw_response.text) response_json = raw_response.json() - verbose_logger.debug(f"Response JSON: {response_json}") + verbose_logger.debug("Response JSON: %s", response_json) # Map Gemini state to OpenAI status gemini_state = response_json.get("state", "STATE_UNSPECIFIED") # Explicitly type status as the Literal union @@ -294,7 +294,7 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): status_details=(str(response_json.get("error", "")) if gemini_state == "FAILED" else None), ) except Exception as e: - verbose_logger.exception(f"Error parsing file retrieve response: {e}") + verbose_logger.exception("Error parsing file retrieve response: %s", e) raise ValueError(f"Error parsing file retrieve response: {e}") def transform_delete_file_request( @@ -362,7 +362,7 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): else: raise ValueError(f"Failed to delete file: {raw_response.text}") except Exception as e: - verbose_logger.exception(f"Error parsing file delete response: {e}") + verbose_logger.exception("Error parsing file delete response: %s", e) raise ValueError(f"Error parsing file delete response: {e}") def transform_list_files_request( diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 6631c9d9ec7..0d56e3f92bc 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -154,7 +154,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): if "parts" in model_turn: parts = model_turn["parts"] if len(parts) != 1: - verbose_logger.warning(f"Realtime: Expected 1 part, got {len(parts)} for Gemini model turn event.") + verbose_logger.warning("Realtime: Expected 1 part, got %s for Gemini model turn event.", len(parts)) part = parts[0] if "text" in part: return OpenAIRealtimeEventTypes.RESPONSE_TEXT_DELTA @@ -472,7 +472,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): call_id = item.get("call_id", "") output = item.get("output", "{}") - verbose_logger.debug(f"Gemini Realtime: Transforming function_call_output for call_id={call_id}") + verbose_logger.debug("Gemini Realtime: Transforming function_call_output for call_id=%s", call_id) # Gemini functionResponses[].response must be a dict; wrap non-dicts. try: @@ -487,8 +487,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): self._tool_call_id_to_name.move_to_end(call_id) else: verbose_logger.warning( - f"Gemini Realtime: Function name not found for call_id={call_id}. " - "This may cause Gemini to reject the response." + "Gemini Realtime: Function name not found for call_id=%s. This may cause Gemini to reject the response.", + call_id, ) function_response: dict[str, Any] = {"response": output_dict} @@ -868,7 +868,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): resolved_response_id = response_id or f"resp_{uuid.uuid4()}" resolved_output_item_id = output_item_id or f"item_{uuid.uuid4()}" - verbose_logger.debug(f"Gemini Realtime: Transforming {len(function_calls)} tool call(s) to OpenAI format") + verbose_logger.debug("Gemini Realtime: Transforming %s tool call(s) to OpenAI format", len(function_calls)) events: list[OpenAIRealtimeFunctionCallArgumentsDone] = [] for idx, fc in enumerate(function_calls): diff --git a/litellm/llms/gigachat/authenticator.py b/litellm/llms/gigachat/authenticator.py index 356d438c6b2..9fc60ee058d 100644 --- a/litellm/llms/gigachat/authenticator.py +++ b/litellm/llms/gigachat/authenticator.py @@ -162,7 +162,7 @@ def _request_token_sync( } data = {"scope": scope} - verbose_logger.debug(f"Requesting GigaChat access token from {auth_url}") + verbose_logger.debug("Requesting GigaChat access token from %s", auth_url) try: client = _get_http_client() @@ -194,7 +194,7 @@ async def _request_token_async( } data = {"scope": scope} - verbose_logger.debug(f"Requesting GigaChat access token from {auth_url}") + verbose_logger.debug("Requesting GigaChat access token from %s", auth_url) try: client = get_async_httpx_client( diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index 4007588cfc5..d907ae2dcdf 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -268,7 +268,7 @@ class GigaChatConfig(BaseConfig): api_base=self._current_api_base, ) except Exception as e: - verbose_logger.error(f"Failed to upload image: {e}") + verbose_logger.error("Failed to upload image: %s", e) return None def transform_request( diff --git a/litellm/llms/gigachat/file_handler.py b/litellm/llms/gigachat/file_handler.py index ee16a6c5870..6c2e6d17915 100644 --- a/litellm/llms/gigachat/file_handler.py +++ b/litellm/llms/gigachat/file_handler.py @@ -97,7 +97,7 @@ def upload_file_sync( # Check cache if url_hash in _file_cache: - verbose_logger.debug(f"Image found in cache: {url_hash[:16]}...") + verbose_logger.debug("Image found in cache: %s...", url_hash[:16]) return _file_cache[url_hash] try: @@ -107,7 +107,7 @@ def upload_file_sync( content_bytes, content_type, ext = parsed verbose_logger.debug("Decoded base64 image") else: - verbose_logger.debug(f"Downloading image from URL: {image_url[:80]}...") + verbose_logger.debug("Downloading image from URL: %s...", image_url[:80]) content_bytes, content_type, ext = _download_image_sync(image_url) filename = f"{uuid.uuid4()}.{ext}" @@ -133,12 +133,12 @@ def upload_file_sync( file_id = result.get("id") if file_id: _file_cache[url_hash] = file_id - verbose_logger.debug(f"File uploaded successfully, file_id: {file_id}") + verbose_logger.debug("File uploaded successfully, file_id: %s", file_id) return file_id except Exception as e: - verbose_logger.error(f"Error uploading file to GigaChat: {e}") + verbose_logger.error("Error uploading file to GigaChat: %s", e) return None @@ -162,7 +162,7 @@ async def upload_file_async( # Check cache if url_hash in _file_cache: - verbose_logger.debug(f"Image found in cache: {url_hash[:16]}...") + verbose_logger.debug("Image found in cache: %s...", url_hash[:16]) return _file_cache[url_hash] try: @@ -172,7 +172,7 @@ async def upload_file_async( content_bytes, content_type, ext = parsed verbose_logger.debug("Decoded base64 image") else: - verbose_logger.debug(f"Downloading image from URL: {image_url[:80]}...") + verbose_logger.debug("Downloading image from URL: %s...", image_url[:80]) content_bytes, content_type, ext = await _download_image_async(image_url) filename = f"{uuid.uuid4()}.{ext}" @@ -201,10 +201,10 @@ async def upload_file_async( file_id = result.get("id") if file_id: _file_cache[url_hash] = file_id - verbose_logger.debug(f"File uploaded successfully, file_id: {file_id}") + verbose_logger.debug("File uploaded successfully, file_id: %s", file_id) return file_id except Exception as e: - verbose_logger.error(f"Error uploading file to GigaChat: {e}") + verbose_logger.error("Error uploading file to GigaChat: %s", e) return None diff --git a/litellm/llms/github_copilot/authenticator.py b/litellm/llms/github_copilot/authenticator.py index 180c2215212..b4bd9c4de3d 100644 --- a/litellm/llms/github_copilot/authenticator.py +++ b/litellm/llms/github_copilot/authenticator.py @@ -58,7 +58,7 @@ class Authenticator: verbose_logger.warning("No existing access token found or error reading file") for attempt in range(3): - verbose_logger.debug(f"Access token acquisition attempt {attempt + 1}/3") + verbose_logger.debug("Access token acquisition attempt %s/3", attempt + 1) try: access_token = self._login() try: @@ -68,7 +68,7 @@ class Authenticator: verbose_logger.error("Error saving access token to file") return access_token except (GetDeviceCodeError, GetAccessTokenError, RefreshAPIKeyError) as e: - verbose_logger.warning(f"Failed attempt {attempt + 1}: {e}") + verbose_logger.warning("Failed attempt %s: %s", attempt + 1, e) continue raise GetAccessTokenError( @@ -100,7 +100,7 @@ class Authenticator: except OSError: verbose_logger.warning("No API key file found or error opening file") except (json.JSONDecodeError, KeyError) as e: - verbose_logger.warning(f"Error reading API key from file: {e}") + verbose_logger.warning("Error reading API key from file: %s", e) except APIKeyExpiredError: pass # Already logged in the try block @@ -117,7 +117,7 @@ class Authenticator: status_code=401, ) except OSError as e: - verbose_logger.error(f"Error saving API key to file: {e}") + verbose_logger.error("Error saving API key to file: %s", e) raise GetAPIKeyError( message=f"Failed to save API key: {e}", status_code=500, @@ -142,7 +142,7 @@ class Authenticator: api_endpoint = endpoints.get("api") return api_endpoint except (OSError, json.JSONDecodeError, KeyError) as e: - verbose_logger.warning(f"Error reading API endpoint from file: {e}") + verbose_logger.warning("Error reading API endpoint from file: %s", e) return None def _refresh_api_key(self) -> dict[str, Any]: @@ -171,11 +171,11 @@ class Authenticator: if "token" in response_json: return response_json else: - verbose_logger.warning(f"API key response missing token: {response_json}") + verbose_logger.warning("API key response missing token: %s", response_json) except httpx.HTTPStatusError as e: - verbose_logger.error(f"HTTP error refreshing API key (attempt {attempt + 1}/{max_retries}): {e}") + verbose_logger.error("HTTP error refreshing API key (attempt %s/%s): %s", attempt + 1, max_retries, e) except Exception as e: - verbose_logger.error(f"Unexpected error refreshing API key: {e}") + verbose_logger.error("Unexpected error refreshing API key: %s", e) raise RefreshAPIKeyError( message="Failed to refresh API key after maximum retries", @@ -237,7 +237,7 @@ class Authenticator: required_fields = ["device_code", "user_code", "verification_uri"] if not all(field in resp_json for field in required_fields): - verbose_logger.error(f"Response missing required fields: {resp_json}") + verbose_logger.error("Response missing required fields: %s", resp_json) raise GetDeviceCodeError( message="Response missing required fields", status_code=400, @@ -245,19 +245,19 @@ class Authenticator: return resp_json except httpx.HTTPStatusError as e: - verbose_logger.error(f"HTTP error getting device code: {e}") + verbose_logger.error("HTTP error getting device code: %s", e) raise GetDeviceCodeError( message=f"Failed to get device code: {e}", status_code=400, ) except json.JSONDecodeError as e: - verbose_logger.error(f"Error decoding JSON response: {e}") + verbose_logger.error("Error decoding JSON response: %s", e) raise GetDeviceCodeError( message=f"Failed to decode device code response: {e}", status_code=400, ) except Exception as e: - verbose_logger.error(f"Unexpected error getting device code: {e}") + verbose_logger.error("Unexpected error getting device code: %s", e) raise GetDeviceCodeError( message=f"Failed to get device code: {e}", status_code=400, @@ -300,23 +300,23 @@ class Authenticator: verbose_logger.info("Authentication successful!") return resp_json["access_token"] elif "error" in resp_json and resp_json.get("error") == "authorization_pending": - verbose_logger.debug(f"Authorization pending (attempt {attempt + 1}/{max_attempts})") + verbose_logger.debug("Authorization pending (attempt %s/%s)", attempt + 1, max_attempts) else: - verbose_logger.warning(f"Unexpected response: {resp_json}") + verbose_logger.warning("Unexpected response: %s", resp_json) except httpx.HTTPStatusError as e: - verbose_logger.error(f"HTTP error polling for access token: {e}") + verbose_logger.error("HTTP error polling for access token: %s", e) raise GetAccessTokenError( message=f"Failed to get access token: {e}", status_code=400, ) except json.JSONDecodeError as e: - verbose_logger.error(f"Error decoding JSON response: {e}") + verbose_logger.error("Error decoding JSON response: %s", e) raise GetAccessTokenError( message=f"Failed to decode access token response: {e}", status_code=400, ) except Exception as e: - verbose_logger.error(f"Unexpected error polling for access token: {e}") + verbose_logger.error("Unexpected error polling for access token: %s", e) raise GetAccessTokenError( message=f"Failed to get access token: {e}", status_code=400, diff --git a/litellm/llms/github_copilot/embedding/transformation.py b/litellm/llms/github_copilot/embedding/transformation.py index 75c2d0e8c40..89e195e2d76 100644 --- a/litellm/llms/github_copilot/embedding/transformation.py +++ b/litellm/llms/github_copilot/embedding/transformation.py @@ -75,7 +75,7 @@ class GithubCopilotEmbeddingConfig(BaseEmbeddingConfig): # Merge with existing headers (user's extra_headers take priority) merged_headers = {**default_headers, **headers} - verbose_logger.debug(f"GitHub Copilot Embedding API: Successfully configured headers for model {model}") + verbose_logger.debug("GitHub Copilot Embedding API: Successfully configured headers for model %s", model) return merged_headers diff --git a/litellm/llms/github_copilot/responses/transformation.py b/litellm/llms/github_copilot/responses/transformation.py index 170ad938efb..079ad760aea 100644 --- a/litellm/llms/github_copilot/responses/transformation.py +++ b/litellm/llms/github_copilot/responses/transformation.py @@ -222,14 +222,14 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): if input_param is not None: initiator = self._get_initiator(input_param) merged_headers["X-Initiator"] = initiator - verbose_logger.debug(f"GitHub Copilot Responses API: Set X-Initiator={initiator}") + verbose_logger.debug("GitHub Copilot Responses API: Set X-Initiator=%s", initiator) # Add vision header if input contains images if self._has_vision_input(input_param): merged_headers["copilot-vision-request"] = "true" verbose_logger.debug("GitHub Copilot Responses API: Enabled vision request") - verbose_logger.debug(f"GitHub Copilot Responses API: Successfully configured headers for model {model}") + verbose_logger.debug("GitHub Copilot Responses API: Successfully configured headers for model %s", model) return merged_headers @@ -295,7 +295,8 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): filtered_item[k] = v verbose_logger.debug( - f"GitHub Copilot reasoning item processed, encrypted_content preserved: {encrypted_content is not None}" + "GitHub Copilot reasoning item processed, encrypted_content preserved: %s", + encrypted_content is not None, ) return filtered_item return item @@ -379,7 +380,7 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): """ if depth > max_depth: verbose_logger.warning( - f"[GitHub Copilot] Max recursion depth {max_depth} reached while checking for vision content" + "[GitHub Copilot] Max recursion depth %s reached while checking for vision content", max_depth ) return False diff --git a/litellm/llms/groq/chat/transformation.py b/litellm/llms/groq/chat/transformation.py index 64537e33d0e..35a4a14057f 100644 --- a/litellm/llms/groq/chat/transformation.py +++ b/litellm/llms/groq/chat/transformation.py @@ -99,7 +99,7 @@ class GroqChatConfig(OpenAILikeChatConfig): if litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider): base_params.append("reasoning_effort") except Exception as e: - verbose_logger.debug(f"Error checking if model supports reasoning: {e}") + verbose_logger.debug("Error checking if model supports reasoning: %s", e) return base_params diff --git a/litellm/llms/huggingface/chat/transformation.py b/litellm/llms/huggingface/chat/transformation.py index da1ebd7c23a..b02e5c174a8 100644 --- a/litellm/llms/huggingface/chat/transformation.py +++ b/litellm/llms/huggingface/chat/transformation.py @@ -148,7 +148,7 @@ class HuggingFaceChatConfig(OpenAIGPTConfig): provider_mapping = provider_mapping[provider] if provider_mapping["status"] == "staging": logger.warning( - f"Model {model_id} is in staging mode for provider {provider}. Meant for test purposes only." + "Model %s is in staging mode for provider %s. Meant for test purposes only.", model_id, provider ) mapped_model = provider_mapping["providerId"] diff --git a/litellm/llms/langflow/chat/transformation.py b/litellm/llms/langflow/chat/transformation.py index 69af32fc840..e84b7677ad2 100644 --- a/litellm/llms/langflow/chat/transformation.py +++ b/litellm/llms/langflow/chat/transformation.py @@ -168,7 +168,7 @@ class LangFlowConfig(BaseConfig): if session_id: payload["session_id"] = session_id - verbose_logger.debug(f"LangFlow request payload: {payload}") + verbose_logger.debug("LangFlow request payload: %s", payload) return payload def _extract_content_from_response(self, response_json: dict) -> str | None: @@ -235,7 +235,7 @@ class LangFlowConfig(BaseConfig): status_code=raw_response.status_code, ) - verbose_logger.debug(f"LangFlow response: {response_json}") + verbose_logger.debug("LangFlow response: %s", response_json) content = self._extract_content_from_response(response_json) if content is None: @@ -265,7 +265,7 @@ class LangFlowConfig(BaseConfig): ) setattr(model_response, "usage", usage) except Exception as e: - verbose_logger.warning(f"Failed to calculate token usage: {e}") + verbose_logger.warning("Failed to calculate token usage: %s", e) return model_response diff --git a/litellm/llms/langgraph/chat/sse_iterator.py b/litellm/llms/langgraph/chat/sse_iterator.py index bdaa34871cf..8815d5c93da 100644 --- a/litellm/llms/langgraph/chat/sse_iterator.py +++ b/litellm/llms/langgraph/chat/sse_iterator.py @@ -62,7 +62,7 @@ class LangGraphSSEStreamIterator: data = json.loads(json_str) return self._process_data(data) except json.JSONDecodeError: - verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}") + verbose_logger.debug("Skipping non-JSON SSE line: %s", line[:100]) return None return None @@ -196,7 +196,7 @@ class LangGraphSSEStreamIterator: except httpx.StreamClosed: raise StopIteration except Exception as e: - verbose_logger.error(f"Error in LangGraph SSE stream: {e}") + verbose_logger.error("Error in LangGraph SSE stream: %s", e) raise StopIteration async def __anext__(self) -> ModelResponseStream: @@ -224,5 +224,5 @@ class LangGraphSSEStreamIterator: except httpx.StreamClosed: raise StopAsyncIteration except Exception as e: - verbose_logger.error(f"Error in LangGraph SSE stream: {e}") + verbose_logger.error("Error in LangGraph SSE stream: %s", e) raise StopAsyncIteration diff --git a/litellm/llms/langgraph/chat/transformation.py b/litellm/llms/langgraph/chat/transformation.py index 2aa96ddb978..3f6b0c327d2 100644 --- a/litellm/llms/langgraph/chat/transformation.py +++ b/litellm/llms/langgraph/chat/transformation.py @@ -223,7 +223,7 @@ class LangGraphConfig(BaseConfig): if "thread_id" in optional_params: payload["thread_id"] = optional_params["thread_id"] - verbose_logger.debug(f"LangGraph request payload: {payload}") + verbose_logger.debug("LangGraph request payload: %s", payload) return payload def _extract_content_from_response(self, response_json: dict) -> str: @@ -297,7 +297,7 @@ class LangGraphConfig(BaseConfig): if client is None or not isinstance(client, HTTPHandler): client = _get_httpx_client(params={}) - verbose_logger.debug(f"Making sync streaming request to: {api_base}") + verbose_logger.debug("Making sync streaming request to: %s", api_base) # Make streaming request response = client.post( @@ -356,7 +356,7 @@ class LangGraphConfig(BaseConfig): if client is None or not isinstance(client, AsyncHTTPHandler): client = get_async_httpx_client(llm_provider=cast(Any, "langgraph"), params={}) - verbose_logger.debug(f"Making async streaming request to: {api_base}") + verbose_logger.debug("Making async streaming request to: %s", api_base) # Make async streaming request response = await client.post( @@ -422,7 +422,7 @@ class LangGraphConfig(BaseConfig): """ try: response_json = raw_response.json() - verbose_logger.debug(f"LangGraph response: {response_json}") + verbose_logger.debug("LangGraph response: %s", response_json) content = self._extract_content_from_response(response_json) @@ -451,12 +451,12 @@ class LangGraphConfig(BaseConfig): ) setattr(model_response, "usage", usage) except Exception as e: - verbose_logger.warning(f"Failed to calculate token usage: {e}") + verbose_logger.warning("Failed to calculate token usage: %s", e) return model_response except Exception as e: - verbose_logger.error(f"Error processing LangGraph response: {e}") + verbose_logger.error("Error processing LangGraph response: %s", e) raise LangGraphError( message=f"Error processing response: {e}", status_code=raw_response.status_code, diff --git a/litellm/llms/litellm_proxy/skills/code_execution.py b/litellm/llms/litellm_proxy/skills/code_execution.py index f1142a8e355..11fd80db377 100644 --- a/litellm/llms/litellm_proxy/skills/code_execution.py +++ b/litellm/llms/litellm_proxy/skills/code_execution.py @@ -141,7 +141,7 @@ class CodeExecutionHandler: response: Any = None # Initialize to avoid possibly unbound error for iteration in range(self.max_iterations): - verbose_logger.debug(f"CodeExecutionHandler: Iteration {iteration + 1}/{self.max_iterations}") + verbose_logger.debug("CodeExecutionHandler: Iteration %s/%s", iteration + 1, self.max_iterations) # Make LLM call response = await litellm.acompletion( @@ -175,7 +175,7 @@ class CodeExecutionHandler: # Check if we're done (no tool calls or not tool_calls finish reason) if stop_reason != "tool_calls" or not assistant_message.tool_calls: - verbose_logger.debug(f"CodeExecutionHandler: Completed after {iteration + 1} iterations") + verbose_logger.debug("CodeExecutionHandler: Completed after %s iterations", iteration + 1) return { "response": response, "files": generated_files, # Files returned directly with base64 content @@ -193,14 +193,14 @@ class CodeExecutionHandler: args = json.loads(tool_call.function.arguments) code = args.get("code", "") - verbose_logger.debug(f"CodeExecutionHandler: Executing code ({len(code)} chars)") + verbose_logger.debug("CodeExecutionHandler: Executing code (%s chars)", len(code)) exec_result = executor.execute( code=code, skill_files=skill_files, ) - verbose_logger.debug(f"CodeExecutionHandler: Execution result: {exec_result}") + verbose_logger.debug("CodeExecutionHandler: Execution result: %s", exec_result) execution_results.append( { @@ -232,7 +232,7 @@ class CodeExecutionHandler: tool_result += f"\n- {f['name']} ({len(file_content)} bytes)" verbose_logger.debug( - f"CodeExecutionHandler: Generated file {f['name']} ({len(file_content)} bytes)" + "CodeExecutionHandler: Generated file %s (%s bytes)", f["name"], len(file_content) ) if exec_result["error"]: @@ -268,7 +268,7 @@ class CodeExecutionHandler: ) # Max iterations reached - verbose_logger.warning(f"CodeExecutionHandler: Max iterations ({self.max_iterations}) reached") + verbose_logger.warning("CodeExecutionHandler: Max iterations (%s) reached", self.max_iterations) return { "response": response, "files": generated_files, diff --git a/litellm/llms/litellm_proxy/skills/handler.py b/litellm/llms/litellm_proxy/skills/handler.py index cc307917af4..efe5c27b7ba 100644 --- a/litellm/llms/litellm_proxy/skills/handler.py +++ b/litellm/llms/litellm_proxy/skills/handler.py @@ -100,7 +100,7 @@ class LiteLLMSkillsHandler: if data.file_type is not None: skill_data["file_type"] = data.file_type - verbose_logger.debug(f"LiteLLMSkillsHandler: Creating skill {skill_id} with title={data.display_title}") + verbose_logger.debug("LiteLLMSkillsHandler: Creating skill %s with title=%s", skill_id, data.display_title) new_skill = await SkillsRepository(prisma_client).table.create(data=skill_data) return _prisma_skill_to_litellm(new_skill) @@ -113,7 +113,7 @@ class LiteLLMSkillsHandler: ) -> list[LiteLLM_SkillsTable]: prisma_client = await LiteLLMSkillsHandler._get_prisma_client() - verbose_logger.debug(f"LiteLLMSkillsHandler: Listing skills with limit={limit}, offset={offset}") + verbose_logger.debug("LiteLLMSkillsHandler: Listing skills with limit=%s, offset=%s", limit, offset) find_many_kwargs: dict[str, Any] = { "take": limit, @@ -150,7 +150,7 @@ class LiteLLMSkillsHandler: skill_id: str, user_api_key_dict: UserAPIKeyAuth | None = None, ) -> LiteLLM_SkillsTable: - verbose_logger.debug(f"LiteLLMSkillsHandler: Getting skill {skill_id}") + verbose_logger.debug("LiteLLMSkillsHandler: Getting skill %s", skill_id) skill = await LiteLLMSkillsHandler._load_skill(skill_id) # Same "not found" message for both "missing" and "cross-tenant" @@ -166,7 +166,7 @@ class LiteLLMSkillsHandler: user_api_key_dict: UserAPIKeyAuth | None = None, ) -> dict[str, str]: prisma_client = await LiteLLMSkillsHandler._get_prisma_client() - verbose_logger.debug(f"LiteLLMSkillsHandler: Deleting skill {skill_id}") + verbose_logger.debug("LiteLLMSkillsHandler: Deleting skill %s", skill_id) skill = await LiteLLMSkillsHandler._load_skill(skill_id) if skill is None or not user_can_access_resource_owner(getattr(skill, "created_by", None), user_api_key_dict): @@ -189,5 +189,5 @@ class LiteLLMSkillsHandler: except ValueError: return None except Exception as e: - verbose_logger.warning(f"LiteLLMSkillsHandler: Error fetching skill {skill_id}: {e}") + verbose_logger.warning("LiteLLMSkillsHandler: Error fetching skill %s: %s", skill_id, e) return None diff --git a/litellm/llms/litellm_proxy/skills/prompt_injection.py b/litellm/llms/litellm_proxy/skills/prompt_injection.py index 244d4196404..f411da8f04c 100644 --- a/litellm/llms/litellm_proxy/skills/prompt_injection.py +++ b/litellm/llms/litellm_proxy/skills/prompt_injection.py @@ -66,7 +66,7 @@ class SkillPromptInjectionHandler: return f"## Skill: {skill.display_title or skill.skill_id}\n\n{content}" except Exception as e: verbose_logger.warning( - f"SkillPromptInjectionHandler: Error extracting content from skill {skill.skill_id}: {e}" + "SkillPromptInjectionHandler: Error extracting content from skill %s: %s", skill.skill_id, e ) return skill.instructions @@ -111,14 +111,16 @@ class SkillPromptInjectionHandler: normalized = posixpath.normpath(clean_path) if normalized.startswith("..") or posixpath.isabs(normalized): verbose_logger.warning( - f"SkillPromptInjectionHandler: Skipping entry with invalid path in skill {skill.skill_id}: {name}" + "SkillPromptInjectionHandler: Skipping entry with invalid path in skill %s: %s", + skill.skill_id, + name, ) continue files[normalized] = zf.read(name) except Exception as e: verbose_logger.warning( - f"SkillPromptInjectionHandler: Error extracting files from skill {skill.skill_id}: {e}" + "SkillPromptInjectionHandler: Error extracting files from skill %s: %s", skill.skill_id, e ) return files diff --git a/litellm/llms/litellm_proxy/skills/sandbox_executor.py b/litellm/llms/litellm_proxy/skills/sandbox_executor.py index e79b0c948c4..78939f3e03b 100644 --- a/litellm/llms/litellm_proxy/skills/sandbox_executor.py +++ b/litellm/llms/litellm_proxy/skills/sandbox_executor.py @@ -96,7 +96,7 @@ class SkillsSandboxExecutor: # Create the file in temp directory local_path = os.path.abspath(os.path.join(tmpdir, path)) if not local_path.startswith(tmpdir_abs + os.sep): - verbose_logger.warning(f"SkillsSandboxExecutor: Skipping file with invalid path: {path}") + verbose_logger.warning("SkillsSandboxExecutor: Skipping file with invalid path: %s", path) continue os.makedirs(os.path.dirname(local_path), exist_ok=True) with open(local_path, "wb") as f: @@ -106,7 +106,7 @@ class SkillsSandboxExecutor: sandbox_path = f"/sandbox/{path}" session.copy_to_runtime(local_path, sandbox_path) - verbose_logger.debug(f"SkillsSandboxExecutor: Copied {len(skill_files)} files to sandbox") + verbose_logger.debug("SkillsSandboxExecutor: Copied %s files to sandbox", len(skill_files)) # 2. Install requirements if present. Let pip parse the # requirements file inside the sandbox so standard syntax like @@ -171,10 +171,10 @@ sys.path.insert(0, '/sandbox') verbose_logger.debug("SkillsSandboxExecutor: Code execution succeeded") else: verbose_logger.debug( - f"SkillsSandboxExecutor: Code execution failed with exit code {result.exit_code}" + "SkillsSandboxExecutor: Code execution failed with exit code %s", result.exit_code ) - verbose_logger.debug(f"SkillsSandboxExecutor: stderr: {error[:500] if error else 'No stderr'}") - verbose_logger.debug(f"SkillsSandboxExecutor: stdout: {output[:500] if output else 'No stdout'}") + verbose_logger.debug("SkillsSandboxExecutor: stderr: %s", error[:500] if error else "No stderr") + verbose_logger.debug("SkillsSandboxExecutor: stdout: %s", output[:500] if output else "No stdout") # 4. Collect generated files generated_files = self._collect_generated_files(session, skill_files) @@ -187,7 +187,7 @@ sys.path.insert(0, '/sandbox') } except Exception as e: - verbose_logger.error(f"SkillsSandboxExecutor: Execution failed: {e}") + verbose_logger.error("SkillsSandboxExecutor: Execution failed: %s", e) return { "success": False, "output": "", @@ -270,15 +270,15 @@ print(json.dumps(files)) } ) - verbose_logger.debug(f"SkillsSandboxExecutor: Collected generated file: {rel_path}") + verbose_logger.debug("SkillsSandboxExecutor: Collected generated file: %s", rel_path) except Exception as e: - verbose_logger.warning(f"SkillsSandboxExecutor: Error copying file {filepath}: {e}") + verbose_logger.warning("SkillsSandboxExecutor: Error copying file %s: %s", filepath, e) finally: if os.path.exists(tmp_path): os.unlink(tmp_path) except Exception as e: - verbose_logger.warning(f"SkillsSandboxExecutor: Error collecting generated files: {e}") + verbose_logger.warning("SkillsSandboxExecutor: Error collecting generated files: %s", e) return generated_files diff --git a/litellm/llms/manus/files/transformation.py b/litellm/llms/manus/files/transformation.py index 325f6f36814..667910fab28 100644 --- a/litellm/llms/manus/files/transformation.py +++ b/litellm/llms/manus/files/transformation.py @@ -245,10 +245,10 @@ class ManusFilesConfig(BaseFilesConfig): response_json = initial_response_data else: # Log raw response for debugging - verbose_logger.debug(f"Manus raw response text: {raw_response.text}") + verbose_logger.debug("Manus raw response text: %s", raw_response.text) response_json = raw_response.json() - verbose_logger.debug(f"Manus file response: {response_json}") + verbose_logger.debug("Manus file response: %s", response_json) # Parse created_at timestamp created_at_str = response_json.get("created_at", "") @@ -279,7 +279,7 @@ class ManusFilesConfig(BaseFilesConfig): status_details=response_json.get("status_details"), ) except Exception as e: - verbose_logger.exception(f"Error parsing Manus file response: {e}") + verbose_logger.exception("Error parsing Manus file response: %s", e) raise ValueError(f"Error parsing Manus file response: {e}") def transform_retrieve_file_request( diff --git a/litellm/llms/manus/responses/transformation.py b/litellm/llms/manus/responses/transformation.py index 25d4d0b8db6..80ffb87c76f 100644 --- a/litellm/llms/manus/responses/transformation.py +++ b/litellm/llms/manus/responses/transformation.py @@ -157,7 +157,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): if extra_body: base_request.update(extra_body) - verbose_logger.debug(f"Manus: Using agent_profile={agent_profile}, task_mode=agent") + verbose_logger.debug("Manus: Using agent_profile=%s, task_mode=agent", agent_profile) return base_request @@ -219,7 +219,9 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): try: response = ResponsesAPIResponse.model_validate(raw_response_json) except Exception: - verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct") + verbose_logger.debug( + "Error constructing ResponsesAPIResponse: %s, using model_construct", raw_response_json + ) response = ResponsesAPIResponse.model_construct(**raw_response_json) # Store processed headers in additional_headers so they get returned to the client @@ -307,7 +309,9 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): try: response = ResponsesAPIResponse.model_validate(raw_response_json) except Exception: - verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct") + verbose_logger.debug( + "Error constructing ResponsesAPIResponse: %s, using model_construct", raw_response_json + ) response = ResponsesAPIResponse.model_construct(**raw_response_json) # Store processed headers in additional_headers so they get returned to the client diff --git a/litellm/llms/mistral/ocr/transformation.py b/litellm/llms/mistral/ocr/transformation.py index e9d8280cc85..dbcb3307980 100644 --- a/litellm/llms/mistral/ocr/transformation.py +++ b/litellm/llms/mistral/ocr/transformation.py @@ -175,7 +175,7 @@ class MistralOCRConfig(BaseOCRConfig): Returns: OCRRequestData with JSON data """ - verbose_logger.debug(f"Mistral OCR transform_ocr_request - model: {model}") + verbose_logger.debug("Mistral OCR transform_ocr_request - model: %s", model) # Document parameter is the Mistral-format dict from the user # Just pass it through as-is to the Mistral API @@ -226,7 +226,7 @@ class MistralOCRConfig(BaseOCRConfig): try: response_json = raw_response.json() - verbose_logger.debug(f"Mistral OCR response keys: {response_json.keys()}") + verbose_logger.debug("Mistral OCR response keys: %s", response_json.keys()) # Return native Mistral format - no transformation return OCRResponse( @@ -237,5 +237,5 @@ class MistralOCRConfig(BaseOCRConfig): object="ocr", ) except Exception as e: - verbose_logger.error(f"Error parsing Mistral OCR response: {e}") + verbose_logger.error("Error parsing Mistral OCR response: %s", e) raise e diff --git a/litellm/llms/ollama/common_utils.py b/litellm/llms/ollama/common_utils.py index 83c697d7cb5..ecc56e6f110 100644 --- a/litellm/llms/ollama/common_utils.py +++ b/litellm/llms/ollama/common_utils.py @@ -119,7 +119,7 @@ class OllamaModelInfo(BaseLLMModelInfo): if isinstance(nm, str): names.add(nm if nm.startswith("ollama/") else f"ollama/{nm}") except Exception as e: - verbose_logger.warning(f"Error retrieving ollama tag endpoint: {e}") + verbose_logger.warning("Error retrieving ollama tag endpoint: %s", e) # If tags endpoint fails, fall back to static list try: from litellm import models_by_provider @@ -127,7 +127,7 @@ class OllamaModelInfo(BaseLLMModelInfo): static = models_by_provider.get("ollama", []) or [] return [f"ollama/{m}" for m in static] except Exception as e1: - verbose_logger.warning(f"Error retrieving static ollama models as fallback: {e1}") + verbose_logger.warning("Error retrieving static ollama models as fallback: %s", e1) return [] # assemble full model names result = sorted(names) diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 5823c2dad75..ccc9fe666f2 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -538,5 +538,5 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator): ) # raise Exception(f"Unable to parse ollama chunk - {chunk}") except Exception as e: - verbose_proxy_logger.error(f"Unable to parse ollama chunk - {chunk}") + verbose_proxy_logger.error("Unable to parse ollama chunk - %s", chunk) raise e diff --git a/litellm/llms/openai/chat/o_series_transformation.py b/litellm/llms/openai/chat/o_series_transformation.py index 0aaf0315f2a..e5c4de0b29f 100644 --- a/litellm/llms/openai/chat/o_series_transformation.py +++ b/litellm/llms/openai/chat/o_series_transformation.py @@ -66,7 +66,7 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig): model, custom_llm_provider, api_base, api_key = get_llm_provider(model=model) except Exception: verbose_logger.debug( - f"Unable to infer model provider for model={model}, defaulting to openai for o1 supported param check" + "Unable to infer model provider for model=%s, defaulting to openai for o1 supported param check", model ) custom_llm_provider = "openai" diff --git a/litellm/llms/openai/cost_calculation.py b/litellm/llms/openai/cost_calculation.py index 87568a4d399..40cfd1dd621 100644 --- a/litellm/llms/openai/cost_calculation.py +++ b/litellm/llms/openai/cost_calculation.py @@ -111,13 +111,19 @@ def cost_per_second(model: str, custom_llm_provider: str | None, duration: float ## Speech / Audio cost calculation if "output_cost_per_second" in model_info and model_info["output_cost_per_second"] is not None: verbose_logger.debug( - f"For model={model} - output_cost_per_second: {model_info.get('output_cost_per_second')}; duration: {duration}" + "For model=%s - output_cost_per_second: %s; duration: %s", + model, + model_info.get("output_cost_per_second"), + duration, ) ## COST PER SECOND ## completion_cost = model_info["output_cost_per_second"] * duration elif "input_cost_per_second" in model_info and model_info["input_cost_per_second"] is not None: verbose_logger.debug( - f"For model={model} - input_cost_per_second: {model_info.get('input_cost_per_second')}; duration: {duration}" + "For model=%s - input_cost_per_second: %s; duration: %s", + model, + model_info.get("input_cost_per_second"), + duration, ) ## COST PER SECOND ## prompt_cost = model_info["input_cost_per_second"] * duration @@ -199,19 +205,23 @@ def video_generation_cost( video_cost_per_second = model_info.get("output_cost_per_video_per_second") if video_cost_per_second is not None: verbose_logger.debug( - f"For model={model} - output_cost_per_video_per_second: {video_cost_per_second}; duration: {duration_seconds}" + "For model=%s - output_cost_per_video_per_second: %s; duration: %s", + model, + video_cost_per_second, + duration_seconds, ) return video_cost_per_second * duration_seconds output_cost_per_second = _video_output_cost_per_second(model_info, video_resolution) if output_cost_per_second is not None: verbose_logger.debug( - f"For model={model} - output_cost_per_second: {output_cost_per_second}; duration: {duration_seconds}" + "For model=%s - output_cost_per_second: %s; duration: %s", model, output_cost_per_second, duration_seconds ) return output_cost_per_second * duration_seconds # If no cost information found, return 0 verbose_logger.warning( - f"No cost information found for video model {model}. Please add pricing to model_prices_and_context_window.json" + "No cost information found for video model %s. Please add pricing to model_prices_and_context_window.json", + model, ) return 0.0 diff --git a/litellm/llms/openai/image_generation/cost_calculator.py b/litellm/llms/openai/image_generation/cost_calculator.py index b134ecc9a24..33893fd6fa7 100644 --- a/litellm/llms/openai/image_generation/cost_calculator.py +++ b/litellm/llms/openai/image_generation/cost_calculator.py @@ -20,7 +20,7 @@ def cost_calculator( """Calculate cost for OpenAI gpt-image models (token-based pricing).""" usage = getattr(image_response, "usage", None) if usage is None: - verbose_logger.debug(f"No usage data available for {model}, cannot calculate token-based cost") + verbose_logger.debug("No usage data available for %s, cannot calculate token-based cost", model) return 0.0 provider = custom_llm_provider or "openai" diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index f01730a06a5..151241dfb50 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -557,7 +557,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): except Exception as e: verbose_logger.exception( - f"LiteLLM.AgenticHookError: Exception in agentic completion hooks for OpenAI: {e}" + "LiteLLM.AgenticHookError: Exception in agentic completion hooks for OpenAI: %s", e ) return None diff --git a/litellm/llms/openai/responses/count_tokens/handler.py b/litellm/llms/openai/responses/count_tokens/handler.py index e59a28c2d09..e1deb47f457 100644 --- a/litellm/llms/openai/responses/count_tokens/handler.py +++ b/litellm/llms/openai/responses/count_tokens/handler.py @@ -45,7 +45,7 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig): try: self.validate_request(model, input) - verbose_logger.debug(f"Processing OpenAI CountTokens request for model: {model}") + verbose_logger.debug("Processing OpenAI CountTokens request for model: %s", model) request_body = self.transform_request_to_count_tokens( model=model, @@ -56,7 +56,7 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig): endpoint_url = self.get_openai_count_tokens_endpoint(api_base) - verbose_logger.debug(f"Making request to: {endpoint_url}") + verbose_logger.debug("Making request to: %s", endpoint_url) headers = self.get_required_headers(api_key) @@ -71,30 +71,30 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig): timeout=request_timeout, ) - verbose_logger.debug(f"Response status: {response.status_code}") + verbose_logger.debug("Response status: %s", response.status_code) if response.status_code != 200: error_text = response.text - verbose_logger.error(f"OpenAI API error: {error_text}") + verbose_logger.error("OpenAI API error: %s", error_text) raise OpenAIError( status_code=response.status_code, message=error_text, ) openai_response = response.json() - verbose_logger.debug(f"OpenAI response: {openai_response}") + verbose_logger.debug("OpenAI response: %s", openai_response) return openai_response except OpenAIError: raise except httpx.HTTPStatusError as e: - verbose_logger.error(f"HTTP error in CountTokens handler: {e}") + verbose_logger.error("HTTP error in CountTokens handler: %s", e) raise OpenAIError( status_code=e.response.status_code, message=e.response.text, ) except (httpx.RequestError, json.JSONDecodeError, ValueError) as e: - verbose_logger.error(f"Error in CountTokens handler: {e}") + verbose_logger.error("Error in CountTokens handler: %s", e) raise OpenAIError( status_code=500, message=f"CountTokens processing error: {e}", diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index d4494759f6c..af65cdbe91d 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -89,7 +89,7 @@ class OpenAITokenCounter(BaseTokenCounter): original_response=result, ) except OpenAIError as e: - verbose_logger.warning(f"OpenAI CountTokens API error: status={e.status_code}, message={e.message}") + verbose_logger.warning("OpenAI CountTokens API error: status=%s, message=%s", e.status_code, e.message) return TokenCountResponse( total_tokens=0, request_model=request_model, @@ -100,7 +100,7 @@ class OpenAITokenCounter(BaseTokenCounter): status_code=e.status_code, ) except Exception as e: - verbose_logger.warning(f"Error calling OpenAI CountTokens API: {e}") + verbose_logger.warning("Error calling OpenAI CountTokens API: %s", e) return TokenCountResponse( total_tokens=0, request_model=request_model, diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 2d0ce47e595..d84a9d2cda7 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -209,7 +209,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): elif isinstance(item, dict): # Handle reasoning items specifically to filter out status=None if item.get("type") == "reasoning": - verbose_logger.debug(f"Handling reasoning item: {item}") + verbose_logger.debug("Handling reasoning item: %s", item) # Type assertion since we know it's a dict at this point dict_item = cast(dict[str, Any], item) filtered_item = self._handle_reasoning_item(dict_item) @@ -251,7 +251,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return dict_reasoning_item except Exception as e: - verbose_logger.debug(f"Failed to create ResponseReasoningItem, falling back to manual filtering: {e}") + verbose_logger.debug("Failed to create ResponseReasoningItem, falling back to manual filtering: %s", e) # Fallback: manually filter out known None fields filtered_item = { k: v @@ -282,7 +282,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): try: response = ResponsesAPIResponse.model_validate(raw_response_json) except Exception: - verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct") + verbose_logger.debug( + "Error constructing ResponsesAPIResponse: %s, using model_construct", raw_response_json + ) response = ResponsesAPIResponse.model_construct(**raw_response_json) # Store processed headers in additional_headers so they get returned to the client @@ -429,7 +431,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ): return True except Exception as e: - verbose_logger.debug(f"Error getting model info in OpenAIResponsesAPIConfig: {e}") + verbose_logger.debug("Error getting model info in OpenAIResponsesAPIConfig: %s", e) return False def supports_native_websocket(self) -> bool: @@ -649,7 +651,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): try: response = ResponsesAPIResponse.model_validate(raw_response_json) except Exception: - verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct") + verbose_logger.debug( + "Error constructing ResponsesAPIResponse: %s, using model_construct", raw_response_json + ) response = ResponsesAPIResponse.model_construct(**raw_response_json) response._hidden_params["additional_headers"] = processed_headers diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index 40c3e2a07a7..57af268fe6a 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -110,8 +110,9 @@ def create_config_class(provider: SimpleProviderConfig): if param in supported_params: supported_params.remove(param) verbose_logger.debug( - f"Model {model} on provider {provider.slug} does not support " - f"function calling — removed tool-related params from supported params." + "Model %s on provider %s does not support function calling — removed tool-related params from supported params.", + model, + provider.slug, ) _supports_reasoning = supports_reasoning(model=model, custom_llm_provider=provider.slug) diff --git a/litellm/llms/openai_like/json_loader.py b/litellm/llms/openai_like/json_loader.py index bc10b7bd62f..5c6e8f643ce 100644 --- a/litellm/llms/openai_like/json_loader.py +++ b/litellm/llms/openai_like/json_loader.py @@ -51,7 +51,7 @@ class JSONProviderRegistry: cls._loaded = True except Exception as e: - verbose_logger.warning(f"Warning: Failed to load JSON provider configs: {e}") + verbose_logger.warning("Warning: Failed to load JSON provider configs: %s", e) cls._loaded = True @classmethod diff --git a/litellm/llms/perplexity/chat/transformation.py b/litellm/llms/perplexity/chat/transformation.py index c6fb750ec1b..5a4e3440201 100644 --- a/litellm/llms/perplexity/chat/transformation.py +++ b/litellm/llms/perplexity/chat/transformation.py @@ -52,13 +52,13 @@ class PerplexityChatConfig(OpenAIGPTConfig): if litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider): base_openai_params.append("reasoning_effort") except Exception as e: - verbose_logger.debug(f"Error checking if model supports reasoning: {e}") + verbose_logger.debug("Error checking if model supports reasoning: %s", e) try: if litellm.supports_web_search(model=model, custom_llm_provider=self.custom_llm_provider): base_openai_params.append("web_search_options") except Exception as e: - verbose_logger.debug(f"Error checking if model supports web search: {e}") + verbose_logger.debug("Error checking if model supports web search: %s", e) return base_openai_params @@ -97,7 +97,7 @@ class PerplexityChatConfig(OpenAIGPTConfig): self._enhance_usage_with_perplexity_fields(model_response, raw_response_json) self._add_citations_as_annotations(model_response, raw_response_json) except Exception as e: - verbose_logger.debug(f"Error extracting Perplexity-specific usage fields: {e}") + verbose_logger.debug("Error extracting Perplexity-specific usage fields: %s", e) return model_response diff --git a/litellm/llms/runwayml/image_generation/transformation.py b/litellm/llms/runwayml/image_generation/transformation.py index ba51a7ba093..5833892c866 100644 --- a/litellm/llms/runwayml/image_generation/transformation.py +++ b/litellm/llms/runwayml/image_generation/transformation.py @@ -170,7 +170,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): """ status = response_data.get("status", "").upper() - verbose_logger.debug(f"RunwayML task status: {status}") + verbose_logger.debug("RunwayML task status: %s", status) if status == "SUCCEEDED": return "succeeded" @@ -216,7 +216,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): api_base = api_base.rstrip("/") task_url = f"{api_base}/v1/tasks/{task_id}" - verbose_logger.debug(f"Polling RunwayML task: {task_url}") + verbose_logger.debug("Polling RunwayML task: %s", task_url) while True: self._check_timeout(start_time=start_time, timeout_secs=timeout_secs) @@ -265,7 +265,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): api_base = api_base.rstrip("/") task_url = f"{api_base}/v1/tasks/{task_id}" - verbose_logger.debug(f"Polling RunwayML task (async): {task_url}") + verbose_logger.debug("Polling RunwayML task (async): %s", task_url) while True: self._check_timeout(start_time=start_time, timeout_secs=timeout_secs) diff --git a/litellm/llms/runwayml/text_to_speech/transformation.py b/litellm/llms/runwayml/text_to_speech/transformation.py index 46a5f606853..21ac716df01 100644 --- a/litellm/llms/runwayml/text_to_speech/transformation.py +++ b/litellm/llms/runwayml/text_to_speech/transformation.py @@ -259,7 +259,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): """ status = response_data.get("status", "").upper() - verbose_logger.debug(f"RunwayML TTS task status: {status}") + verbose_logger.debug("RunwayML TTS task status: %s", status) if status == "SUCCEEDED": return "succeeded" @@ -305,7 +305,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): api_base = api_base.rstrip("/") task_url = f"{api_base}/v1/tasks/{task_id}" - verbose_logger.debug(f"Polling RunwayML TTS task: {task_url}") + verbose_logger.debug("Polling RunwayML TTS task: %s", task_url) while True: self._check_timeout(start_time=start_time, timeout_secs=timeout_secs) @@ -353,7 +353,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): api_base = api_base.rstrip("/") task_url = f"{api_base}/v1/tasks/{task_id}" - verbose_logger.debug(f"Polling RunwayML TTS task (async): {task_url}") + verbose_logger.debug("Polling RunwayML TTS task (async): %s", task_url) while True: self._check_timeout(start_time=start_time, timeout_secs=timeout_secs) diff --git a/litellm/llms/sagemaker/common_utils.py b/litellm/llms/sagemaker/common_utils.py index 5f5c0250273..5916747bf28 100644 --- a/litellm/llms/sagemaker/common_utils.py +++ b/litellm/llms/sagemaker/common_utils.py @@ -130,7 +130,7 @@ class AWSEventStreamDecoder: yield self._chunk_parser(chunk_data=_data) except json.JSONDecodeError: # Handle or log any unparseable data at the end - verbose_logger.error(f"Warning: Unparseable JSON data remained: {accumulated_json}") + verbose_logger.error("Warning: Unparseable JSON data remained: %s", accumulated_json) yield None async def aiter_bytes( @@ -168,10 +168,10 @@ class AWSEventStreamDecoder: # If it's not valid JSON yet, continue to the next event continue except UnicodeDecodeError as e: - verbose_logger.warning(f"UnicodeDecodeError: {e}. Attempting to combine with next event.") + verbose_logger.warning("UnicodeDecodeError: %s. Attempting to combine with next event.", e) continue except Exception as e: - verbose_logger.error(f"Error parsing message: {e}. Attempting to combine with next event.") + verbose_logger.error("Error parsing message: %s. Attempting to combine with next event.", e) continue # Handle any remaining data after the iterator is exhausted @@ -184,10 +184,10 @@ class AWSEventStreamDecoder: yield self._chunk_parser(chunk_data=_data) except json.JSONDecodeError: # Handle or log any unparseable data at the end - verbose_logger.error(f"Warning: Unparseable JSON data remained: {accumulated_json}") + verbose_logger.error("Warning: Unparseable JSON data remained: %s", accumulated_json) yield None except Exception as e: - verbose_logger.error(f"Final error parsing accumulated JSON: {e}") + verbose_logger.error("Final error parsing accumulated JSON: %s", e) def _parse_message_from_event(self, event) -> str | None: response_stream_shape = get_sagemaker_response_stream_shape() diff --git a/litellm/llms/sap/credentials.py b/litellm/llms/sap/credentials.py index 5b2e02875b8..fe5df75fc72 100644 --- a/litellm/llms/sap/credentials.py +++ b/litellm/llms/sap/credentials.py @@ -45,7 +45,7 @@ def _get_nested(d: dict[str, Any] | str, path: Sequence[str]) -> Any: for k in path: if not isinstance(cur, dict): verbose_logger.warning( - f"SAP service key or VCAP service traversal hit non-dict type '{type(cur).__name__}' at key '{k}'." + "SAP service key or VCAP service traversal hit non-dict type '%s' at key '%s'.", type(cur).__name__, k ) return None if k not in cur: @@ -173,7 +173,7 @@ def resolve_credentials(sources: list[Source]) -> dict[str, str]: for source in sources: credentials = extract_credentials(source) if credentials: - verbose_logger.debug(f"Resolved SAP credentials from source {source.name}") + verbose_logger.debug("Resolved SAP credentials from source %s", source.name) return credentials raise ValueError("No credentials found in any source") @@ -184,7 +184,7 @@ def resolve_resource_group(sources: list[Source]) -> str | None: for source in sources: value = source.get(rg_cred) if value is not None: - verbose_logger.debug(f"Resolved GEN AI Hub resource_group from source {source.name}") + verbose_logger.debug("Resolved GEN AI Hub resource_group from source %s", source.name) return value return rg_cred.default @@ -208,7 +208,7 @@ def _parse_service_key_once( verbose_logger.warning("SAP service key is a string but not valid JSON. Skipping this source.") return None verbose_logger.warning( - f"SAP service key has unexpected type '{type(service_key).__name__}'. Expected str or dict. Ignoring." + "SAP service key has unexpected type '%s'. Expected str or dict. Ignoring.", type(service_key).__name__ ) return None diff --git a/litellm/llms/together_ai/chat.py b/litellm/llms/together_ai/chat.py index 5920f02a44e..0eefd1ff1d6 100644 --- a/litellm/llms/together_ai/chat.py +++ b/litellm/llms/together_ai/chat.py @@ -29,7 +29,7 @@ class TogetherAIConfig(OpenAIGPTConfig): try: supports_fc = supports_function_calling(model, custom_llm_provider="together_ai") except Exception as e: - verbose_logger.debug(f"Error getting supported openai params: {e}") + verbose_logger.debug("Error getting supported openai params: %s", e) optional_params = super().get_supported_openai_params(model) if supports_fc is not True: diff --git a/litellm/llms/vertex_ai/agent_engine/transformation.py b/litellm/llms/vertex_ai/agent_engine/transformation.py index 9f01a9ee506..b40e38916ed 100644 --- a/litellm/llms/vertex_ai/agent_engine/transformation.py +++ b/litellm/llms/vertex_ai/agent_engine/transformation.py @@ -137,7 +137,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): # (create_session, get_session, list_sessions, delete_session, etc.) endpoint = f"{base_url}/v1beta1/{resource_path}:streamQuery" - verbose_logger.debug(f"Vertex Agent Engine URL: {endpoint}") + verbose_logger.debug("Vertex Agent Engine URL: %s", endpoint) return endpoint def _get_auth_headers( @@ -155,7 +155,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): project_id=vertex_project, ) - verbose_logger.debug(f"Vertex Agent Engine: Authenticated for project {project_id}") + verbose_logger.debug("Vertex Agent Engine: Authenticated for project %s", project_id) return { "Authorization": f"Bearer {access_token}", @@ -219,7 +219,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): "input": input_data, } - verbose_logger.debug(f"Vertex Agent Engine payload: {payload}") + verbose_logger.debug("Vertex Agent Engine payload: %s", payload) return payload def validate_environment( @@ -270,7 +270,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): total_tokens=total_tokens, ) except Exception as e: - verbose_logger.warning(f"Failed to calculate token usage: {e}") + verbose_logger.warning("Failed to calculate token usage: %s", e) return None def transform_response( @@ -295,11 +295,11 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): """ try: content_type = raw_response.headers.get("content-type", "").lower() - verbose_logger.debug(f"Vertex Agent Engine response Content-Type: {content_type}") + verbose_logger.debug("Vertex Agent Engine response Content-Type: %s", content_type) # Parse the SSE response response_text = raw_response.text - verbose_logger.debug(f"Response (first 500 chars): {response_text[:500]}") + verbose_logger.debug("Response (first 500 chars): %s", response_text[:500]) # Extract content from SSE stream content = "" @@ -335,7 +335,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): return model_response except Exception as e: - verbose_logger.error(f"Error processing Vertex Agent Engine response: {e}") + verbose_logger.error("Error processing Vertex Agent Engine response: %s", e) raise VertexAgentEngineError( message=f"Error processing response: {e}", status_code=raw_response.status_code, diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 81d084e7e03..8d63df1ea3d 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -221,7 +221,8 @@ def get_supports_system_message( supports_system_message = True except Exception as e: verbose_logger.warning( - f"Unable to identify if system message supported. Defaulting to 'False'. Received error message - {e}\nAdd it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json" + "Unable to identify if system message supported. Defaulting to 'False'. Received error message - %s\nAdd it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json", + e, ) supports_system_message = False diff --git a/litellm/llms/vertex_ai/cost_calculator.py b/litellm/llms/vertex_ai/cost_calculator.py index a53e54e5fc2..572a4ed14db 100644 --- a/litellm/llms/vertex_ai/cost_calculator.py +++ b/litellm/llms/vertex_ai/cost_calculator.py @@ -114,7 +114,8 @@ def cost_per_character( prompt_cost = prompt_characters * model_info["input_cost_per_character"] except Exception as e: verbose_logger.debug( - f"litellm.litellm_core_utils.llm_cost_calc.google.py::cost_per_character(): Exception occured - {e}\nDefaulting to None" + "litellm.litellm_core_utils.llm_cost_calc.google.py::cost_per_character(): Exception occured - %s\nDefaulting to None", + e, ) prompt_cost, _ = cost_per_token( model=model, @@ -152,7 +153,8 @@ def cost_per_character( completion_cost = completion_characters * model_info["output_cost_per_character"] except Exception as e: verbose_logger.debug( - f"litellm.litellm_core_utils.llm_cost_calc.google.py::cost_per_character(): Exception occured - {e}\nDefaulting to None" + "litellm.litellm_core_utils.llm_cost_calc.google.py::cost_per_character(): Exception occured - %s\nDefaulting to None", + e, ) _, completion_cost = cost_per_token( model=model, diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index cadc8760601..c3977b19c87 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -447,8 +447,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): transformed_config["environment"] = env_value else: verbose_logger.info( - f"Invalid environment value for computer_use: {env_value}. " - f"Supported: 'browser', 'unspecified', 'ENVIRONMENT_BROWSER', 'ENVIRONMENT_UNSPECIFIED'" + "Invalid environment value for computer_use: %s. Supported: 'browser', 'unspecified', 'ENVIRONMENT_BROWSER', 'ENVIRONMENT_UNSPECIFIED'", + env_value, ) # Transform excluded_predefined_functions to camelCase @@ -626,7 +626,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "web_search", "web_search_preview", ): - verbose_logger.info(f"Gemini: Transforming OpenAI-style '{tool['type']}' tool to googleSearch") + verbose_logger.info("Gemini: Transforming OpenAI-style '%s' tool to googleSearch", tool["type"]) tool = {VertexToolName.GOOGLE_SEARCH.value: {}} # Handle tools with 'type' field (OpenAI spec compliance) Ignore this field -> https://github.com/BerriAI/litellm/issues/14644#issuecomment-3342061838 elif "type" in tool: @@ -1087,36 +1087,29 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if VertexGeminiConfig._is_gemini_3_or_newer(model): if value is not None and value < 1.0: verbose_logger.info( - f"Warning: Setting temperature < 1.0 for Gemini 3 models ({model}) " - "can cause infinite loops, degraded reasoning performance, and failure on complex tasks. " - "Strongly recommended to use temperature = 1.0 (default)." + "Warning: Setting temperature < 1.0 for Gemini 3 models (%s) can cause infinite loops, degraded reasoning performance, and failure on complex tasks. Strongly recommended to use temperature = 1.0 (default).", + model, ) if not gemini_sampling_params_warned: verbose_logger.warning( - "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to " - f"function for Gemini 3+ ({model}) but are planned for removal in a " - "future release. Move sampling guidance into the `system` " - "instructions instead." + "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.", + model, ) gemini_sampling_params_warned = True optional_params["temperature"] = value elif param == "top_p": if VertexGeminiConfig._is_gemini_3_or_newer(model) and not gemini_sampling_params_warned: verbose_logger.warning( - "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to " - f"function for Gemini 3+ ({model}) but are planned for removal in a " - "future release. Move sampling guidance into the `system` " - "instructions instead." + "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.", + model, ) gemini_sampling_params_warned = True optional_params["top_p"] = value elif param == "top_k": if VertexGeminiConfig._is_gemini_3_or_newer(model) and not gemini_sampling_params_warned: verbose_logger.warning( - "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to " - f"function for Gemini 3+ ({model}) but are planned for removal in a " - "future release. Move sampling guidance into the `system` " - "instructions instead." + "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.", + model, ) gemini_sampling_params_warned = True optional_params["top_k"] = value @@ -1977,7 +1970,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): prompt_feedback = processed_chunk.get("promptFeedback") if prompt_feedback and "blockReason" in prompt_feedback: verbose_logger.debug( - f"Prompt blocked due to: {prompt_feedback.get('blockReason')} - {prompt_feedback.get('blockReasonMessage')}" + "Prompt blocked due to: %s - %s", + prompt_feedback.get("blockReason"), + prompt_feedback.get("blockReasonMessage"), ) # Create a content_filter response (consistent with non-streaming _handle_blocked_response) @@ -3248,7 +3243,7 @@ class ModelResponseIterator: def chunk_parser(self, chunk: dict) -> Optional["ModelResponseStream"]: try: - verbose_logger.debug(f"RAW GEMINI CHUNK: {chunk}") + verbose_logger.debug("RAW GEMINI CHUNK: %s", chunk) # Detect mid-stream error chunks (e.g. 429 RESOURCE_EXHAUSTED). # Vertex AI can return errors as HTTP 200 but with an "error" field in the SSE body. diff --git a/litellm/llms/vertex_ai/ocr/deepseek_transformation.py b/litellm/llms/vertex_ai/ocr/deepseek_transformation.py index b66dead91b1..f148f8e8be8 100644 --- a/litellm/llms/vertex_ai/ocr/deepseek_transformation.py +++ b/litellm/llms/vertex_ai/ocr/deepseek_transformation.py @@ -259,7 +259,7 @@ class VertexAIDeepSeekOCRConfig(BaseOCRConfig): OCRResponse in standard format """ verbose_logger.debug("Vertex AI DeepSeek OCR transform_ocr_response called") - verbose_logger.debug(f"Raw response: {raw_response.text}") + verbose_logger.debug("Raw response: %s", raw_response.text) try: response_json = raw_response.json() @@ -345,7 +345,7 @@ class VertexAIDeepSeekOCRConfig(BaseOCRConfig): ) except Exception as e: - verbose_logger.error(f"Error parsing Vertex AI DeepSeek OCR response: {e}") + verbose_logger.error("Error parsing Vertex AI DeepSeek OCR response: %s", e) raise e async def async_transform_ocr_response( diff --git a/litellm/llms/vertex_ai/ocr/transformation.py b/litellm/llms/vertex_ai/ocr/transformation.py index 0fb9523f3eb..e1254c5f833 100644 --- a/litellm/llms/vertex_ai/ocr/transformation.py +++ b/litellm/llms/vertex_ai/ocr/transformation.py @@ -138,13 +138,13 @@ class VertexAIOCRConfig(MistralOCRConfig): Returns: Base64 data URI string """ - verbose_logger.debug(f"Vertex AI OCR: Converting URL to base64 data URI (sync): {url}") + verbose_logger.debug("Vertex AI OCR: Converting URL to base64 data URI (sync): %s", url) # Fetch and convert to base64 data URI # convert_url_to_base64 already returns a full data URI like "data:image/jpeg;base64,..." data_uri = convert_url_to_base64(url=url) - verbose_logger.debug(f"Vertex AI OCR: Converted URL to data URI (length: {len(data_uri)})") + verbose_logger.debug("Vertex AI OCR: Converted URL to data URI (length: %s)", len(data_uri)) return data_uri @@ -161,13 +161,13 @@ class VertexAIOCRConfig(MistralOCRConfig): Returns: Base64 data URI string """ - verbose_logger.debug(f"Vertex AI OCR: Converting URL to base64 data URI (async): {url}") + verbose_logger.debug("Vertex AI OCR: Converting URL to base64 data URI (async): %s", url) # Fetch and convert to base64 data URI asynchronously # async_convert_url_to_base64 already returns a full data URI like "data:image/jpeg;base64,..." data_uri = await async_convert_url_to_base64(url=url) - verbose_logger.debug(f"Vertex AI OCR: Converted URL to data URI (length: {len(data_uri)})") + verbose_logger.debug("Vertex AI OCR: Converted URL to data URI (length: %s)", len(data_uri)) return data_uri @@ -252,7 +252,7 @@ class VertexAIOCRConfig(MistralOCRConfig): Returns: OCRRequestData with JSON data """ - verbose_logger.debug(f"Vertex AI OCR async_transform_ocr_request - model: {model}") + verbose_logger.debug("Vertex AI OCR async_transform_ocr_request - model: %s", model) if not isinstance(document, dict): raise ValueError(f"Expected document dict, got {type(document)}") diff --git a/litellm/llms/vertex_ai/rag_engine/ingestion.py b/litellm/llms/vertex_ai/rag_engine/ingestion.py index edafa2f8f7a..cb30dc0ce0a 100644 --- a/litellm/llms/vertex_ai/rag_engine/ingestion.py +++ b/litellm/llms/vertex_ai/rag_engine/ingestion.py @@ -133,7 +133,7 @@ class VertexAIRAGIngestion(BaseRAGIngestion): file_tuple = (filename, file_content, content_type) verbose_logger.debug( - f"Uploading file to GCS via litellm.files.acreate_file: {filename} (bucket: {self.gcs_bucket})" + "Uploading file to GCS via litellm.files.acreate_file: %s (bucket: %s)", filename, self.gcs_bucket ) # Upload to GCS using LiteLLM's file upload @@ -148,7 +148,7 @@ class VertexAIRAGIngestion(BaseRAGIngestion): # The response.id should be the GCS URI gcs_uri = response.id - verbose_logger.info(f"Uploaded file to GCS: {gcs_uri}") + verbose_logger.info("Uploaded file to GCS: %s", gcs_uri) return gcs_uri finally: @@ -185,7 +185,7 @@ class VertexAIRAGIngestion(BaseRAGIngestion): transformation_config = self._build_transformation_config() corpus_name = self._get_corpus_name() - verbose_logger.debug(f"Importing {gcs_uri} into corpus {self.corpus_id}") + verbose_logger.debug("Importing %s into corpus %s", gcs_uri, self.corpus_id) if self.wait_for_import: # Synchronous import - wait for completion @@ -195,7 +195,7 @@ class VertexAIRAGIngestion(BaseRAGIngestion): transformation_config=transformation_config, timeout=self.import_timeout, ) - verbose_logger.info(f"Import complete: {response.imported_rag_files_count} files imported") + verbose_logger.info("Import complete: %s files imported", response.imported_rag_files_count) else: # Async import - don't wait _ = rag.import_files_async( @@ -293,7 +293,7 @@ class VertexAIRAGIngestion(BaseRAGIngestion): try: await self._import_file_to_corpus_via_sdk(gcs_uri=gcs_uri) except Exception as e: - verbose_logger.error(f"Failed to import file into RAG corpus: {e}") + verbose_logger.error("Failed to import file into RAG corpus: %s", e) raise RuntimeError(f"Failed to import file into RAG corpus: {e}") from e return str(self.corpus_id), gcs_uri diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 2def1acb708..c0a6bd01167 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -766,7 +766,7 @@ class VertexBase: The original error if reauthentication fails """ verbose_logger.debug( - f"Handling reauthentication for project_id: {project_id}. Clearing cache and retrying once." + "Handling reauthentication for project_id: %s. Clearing cache and retrying once.", project_id ) # Clear the cached credentials @@ -782,8 +782,10 @@ class VertexBase: ) except Exception as retry_error: verbose_logger.error( - f"Reauthentication retry failed for project_id: {project_id}. " - f"Original error: {error}. Retry error: {retry_error}" + "Reauthentication retry failed for project_id: %s. Original error: %s. Retry error: %s", + project_id, + error, + retry_error, ) # Re-raise the original error for better context raise error @@ -799,7 +801,7 @@ class VertexBase: Async reauthentication retry that stays within the per-key async lock. """ verbose_logger.debug( - f"Handling async reauthentication for project_id: {project_id}. Clearing cache and retrying once." + "Handling async reauthentication for project_id: %s. Clearing cache and retrying once.", project_id ) self._credentials_project_mapping.pop(credential_cache_key, None) @@ -836,8 +838,10 @@ class VertexBase: return _credentials.token, project_id except Exception as retry_error: verbose_logger.error( - f"Async reauthentication retry failed for project_id: {project_id}. " - f"Original error: {error}. Retry error: {retry_error}" + "Async reauthentication retry failed for project_id: %s. Original error: %s. Retry error: %s", + project_id, + error, + retry_error, ) raise error @@ -870,10 +874,10 @@ class VertexBase: credential_cache_key = (cache_credentials, project_id) _credentials: GoogleCredentialsObject | None = None - verbose_logger.debug(f"Checking cached credentials for project_id: {project_id}") + verbose_logger.debug("Checking cached credentials for project_id: %s", project_id) if credential_cache_key in self._credentials_project_mapping: - verbose_logger.debug(f"Cached credentials found for project_id: {project_id}.") + verbose_logger.debug("Cached credentials found for project_id: %s.", project_id) # Retrieve both credentials and cached project_id cached_entry = self._credentials_project_mapping[credential_cache_key] verbose_logger.debug("cached_entry: %s", cached_entry) @@ -890,14 +894,15 @@ class VertexBase: else: verbose_logger.debug( - f"Credential cache key not found for project_id: {project_id}, loading new credentials" + "Credential cache key not found for project_id: %s, loading new credentials", project_id ) try: _credentials, credential_project_id = self.load_auth(credentials=credentials, project_id=project_id) except Exception as e: verbose_logger.exception( - f"Failed to load vertex credentials. Check to see if credentials containing partial/invalid information. Error: {e}" + "Failed to load vertex credentials. Check to see if credentials containing partial/invalid information. Error: %s", + e, ) raise e diff --git a/litellm/llms/watsonx/chat/transformation.py b/litellm/llms/watsonx/chat/transformation.py index adc7035d2f7..0990edc5c42 100644 --- a/litellm/llms/watsonx/chat/transformation.py +++ b/litellm/llms/watsonx/chat/transformation.py @@ -190,7 +190,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig): # Log the exception for debugging but don't raise it # The caller will fall back to default prompt factory try: - verbose_logger.debug(f"Failed to apply HuggingFace template for model {hf_model}: {e}") + verbose_logger.debug("Failed to apply HuggingFace template for model %s: %s", hf_model, e) except Exception: # If logging fails, silently continue - don't break the flow pass diff --git a/litellm/llms/xai/chat/transformation.py b/litellm/llms/xai/chat/transformation.py index 98d8fe5fadd..fa2f957260e 100644 --- a/litellm/llms/xai/chat/transformation.py +++ b/litellm/llms/xai/chat/transformation.py @@ -144,7 +144,7 @@ class XAIChatConfig(OpenAIGPTConfig): if litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider): base_openai_params.append("reasoning_effort") except Exception as e: - verbose_logger.debug(f"Error checking if model supports reasoning: {e}") + verbose_logger.debug("Error checking if model supports reasoning: %s", e) return base_openai_params @@ -277,7 +277,7 @@ class XAIChatConfig(OpenAIGPTConfig): raw_response_json = raw_response.json() self._enhance_usage_with_xai_web_search_fields(response, raw_response_json) except Exception as e: - verbose_logger.debug(f"Error extracting X.AI web search usage: {e}") + verbose_logger.debug("Error extracting X.AI web search usage: %s", e) self._fold_reasoning_tokens_into_completion(response) self._normalize_openai_compatible_usage_totals(getattr(response, "usage", None)) @@ -369,7 +369,7 @@ class XAIChatConfig(OpenAIGPTConfig): usage.prompt_tokens_details.web_search_requests = int(num_sources_used) setattr(usage, "num_sources_used", int(num_sources_used)) - verbose_logger.debug(f"X.AI web search sources used: {num_sources_used}") + verbose_logger.debug("X.AI web search sources used: %s", num_sources_used) @staticmethod def _normalize_openai_compatible_usage_totals( diff --git a/litellm/main.py b/litellm/main.py index 731a545a267..8b2c3c72f76 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -550,7 +550,7 @@ async def acompletion( # Log shared session usage if shared_session is not None: - verbose_logger.debug(f"🔄 SHARED SESSION: acompletion called with shared_session (ID: {id(shared_session)})") + verbose_logger.debug("🔄 SHARED SESSION: acompletion called with shared_session (ID: %s)", id(shared_session)) else: verbose_logger.debug("🔄 NO SHARED SESSION: acompletion called without shared_session") @@ -1002,7 +1002,7 @@ def responses_api_bridge_check( model = model.replace("responses/", "") except Exception as e: - verbose_logger.debug(f"Error getting model info: {e}") + verbose_logger.debug("Error getting model info: %s", e) if model.startswith("responses/"): # handle azure models - `azure/responses/` model = model.replace("responses/", "") @@ -2817,7 +2817,7 @@ def _complete_cohere_chat(ctx: _CompletionDispatchContext) -> _CompletionDispatc ) cohere_route = CohereModelInfo.get_cohere_route(model) - verbose_logger.debug(f"Cohere route: {cohere_route}") + verbose_logger.debug("Cohere route: %s", cohere_route) # Set API base based on route if cohere_route == "v2": api_base = api_base or litellm.api_base or get_secret_str("COHERE_API_BASE") or "https://api.cohere.com/v2/chat" @@ -2834,8 +2834,8 @@ def _complete_cohere_chat(ctx: _CompletionDispatchContext) -> _CompletionDispatc if extra_headers is not None: headers.update(extra_headers) - verbose_logger.debug(f"Model: {model}, API Base: {api_base}") - verbose_logger.debug(f"Provider Config: {provider_config}") + verbose_logger.debug("Model: %s, API Base: %s", model, api_base) + verbose_logger.debug("Provider Config: %s", provider_config) return base_llm_http_handler.completion( model=model, stream=stream, @@ -4998,7 +4998,7 @@ def completion( # type: ignore proxy_headers = litellm.proxy_auth.get_auth_headers() headers.update(proxy_headers) except Exception as e: - verbose_logger.warning(f"Failed to get proxy auth headers: {e}") + verbose_logger.warning("Failed to get proxy auth headers: %s", e) num_retries = kwargs.get( "num_retries", None ) ## alt. param for 'max_retries'. Use this to pass retries w/ instructor. @@ -5965,7 +5965,7 @@ def embedding( proxy_headers = litellm.proxy_auth.get_auth_headers() headers.update(proxy_headers) except Exception as e: - verbose_logger.warning(f"Failed to get proxy auth headers: {e}") + verbose_logger.warning("Failed to get proxy auth headers: %s", e) ### CUSTOM MODEL COST ### input_cost_per_token = kwargs.get("input_cost_per_token", None) output_cost_per_token = kwargs.get("output_cost_per_token", None) @@ -8669,7 +8669,7 @@ def stream_chunk_builder( processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj) return response except Exception as e: - verbose_logger.exception(f"litellm.main.py::stream_chunk_builder() - Exception occurred - {e}") + verbose_logger.exception("litellm.main.py::stream_chunk_builder() - Exception occurred - %s", e) raise litellm.APIError( status_code=500, message="Error building chunks for logging/streaming usage calculation", @@ -8759,7 +8759,7 @@ async def acount_tokens( if result is not None and not result.error: return result except Exception as e: - verbose_logger.debug(f"Provider token counting failed for model={model}, falling back to local: {e}") + verbose_logger.debug("Provider token counting failed for model=%s, falling back to local: %s", model, e) # Fallback to local tiktoken-based token counting fallback_messages = messages or [] diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index f53f32eecfa..171f3286a7d 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -119,7 +119,7 @@ def _prepare_ocr_request( if ocr_provider_config is None: raise ValueError(f"OCR is not supported for provider: {custom_llm_provider}") - verbose_logger.debug(f"OCR call - model: {model}, provider: {custom_llm_provider}") + verbose_logger.debug("OCR call - model: %s, provider: %s", model, custom_llm_provider) litellm_params = GenericLiteLLMParams.model_validate(kwargs) @@ -135,7 +135,7 @@ def _prepare_ocr_request( model=model, ) - verbose_logger.debug(f"OCR optional_params after mapping: {optional_params}") + verbose_logger.debug("OCR optional_params after mapping: %s", optional_params) effective_timeout = timeout or request_timeout @@ -553,14 +553,18 @@ def convert_file_document_to_url_document(document: dict[str, Any]) -> dict[str, if mime_type.startswith("image/"): verbose_logger.debug( - f"OCR file input: Converted file to image_url data URI " - f"(mime={mime_type}, size={len(file_bytes)} bytes, name={file_name})" + "OCR file input: Converted file to image_url data URI (mime=%s, size=%s bytes, name=%s)", + mime_type, + len(file_bytes), + file_name, ) return {"type": "image_url", "image_url": data_uri} verbose_logger.debug( - f"OCR file input: Converted file to document_url data URI " - f"(mime={mime_type}, size={len(file_bytes)} bytes, name={file_name})" + "OCR file input: Converted file to document_url data URI (mime=%s, size=%s bytes, name=%s)", + mime_type, + len(file_bytes), + file_name, ) return {"type": "document_url", "document_url": data_uri} diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index a256653f0f9..07a4eb164c3 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -384,14 +384,14 @@ class MCPRequestHandler: # Parse MCP servers from header mcp_servers_header = headers.get(MCPRequestHandler.LITELLM_MCP_SERVERS_HEADER_NAME) - verbose_logger.debug(f"Raw MCP servers header: {mcp_servers_header}") + verbose_logger.debug("Raw MCP servers header: %s", mcp_servers_header) mcp_servers = None if mcp_servers_header is not None: try: mcp_servers = [s.strip() for s in mcp_servers_header.split(",") if s.strip()] - verbose_logger.debug(f"Parsed MCP servers: {mcp_servers}") + verbose_logger.debug("Parsed MCP servers: %s", mcp_servers) except Exception as e: - verbose_logger.debug(f"Error parsing mcp_servers header: {e}") + verbose_logger.debug("Error parsing mcp_servers header: %s", e) mcp_servers = None if mcp_servers_header == "" or (mcp_servers is not None and len(mcp_servers) == 0): mcp_servers = [] @@ -1008,7 +1008,7 @@ class MCPRequestHandler: limits[source.team_id] = applicable return limits or None except Exception as e: # noqa: BLE001 # throttling metadata must never fail an allowed request - verbose_logger.warning(f"Failed to resolve per-team MCP rpm limits for admitted subject: {e}") + verbose_logger.warning("Failed to resolve per-team MCP rpm limits for admitted subject: %s", e) return None @staticmethod @@ -1091,7 +1091,7 @@ class MCPRequestHandler: user_id_upsert=False, ) except Exception as e: # noqa: BLE001 # mirror the builder's fail-open user lookup; DB errors are of any type - verbose_logger.debug(f"bridge admission: user lookup failed, skipping SCIM gate: {e}") + verbose_logger.debug("bridge admission: user lookup failed, skipping SCIM gate: %s", e) user_object = None if user_object is None or not isinstance(user_object.metadata, dict): return @@ -1194,8 +1194,8 @@ class MCPRequestHandler: auth_header = headers.get(mcp_client_side_auth_header_name) if auth_header: verbose_logger.warning( - f"The '{mcp_client_side_auth_header_name}' header is deprecated. " - f"Please use server-specific auth headers in the format 'x-mcp-{{server_alias}}-{{header_name}}' instead." + "The '%s' header is deprecated. Please use server-specific auth headers in the format 'x-mcp-{server_alias}-{header_name}' instead.", + mcp_client_side_auth_header_name, ) return auth_header @@ -1245,7 +1245,10 @@ class MCPRequestHandler: server_auth_headers[server_alias][auth_header_name] = header_value verbose_logger.debug( - f"Found server auth header: {server_alias} -> {auth_header_name}: {header_value[:10]}..." + "Found server auth header: %s -> %s: %s...", + server_alias, + auth_header_name, + header_value[:10], ) return server_auth_headers @@ -1331,7 +1334,7 @@ class MCPRequestHandler: headers_dict = {name.decode("latin-1"): value.decode("latin-1") for name, value in raw_headers} return Headers(headers_dict) except (UnicodeDecodeError, AttributeError, TypeError) as e: - verbose_logger.exception(f"Error getting headers from scope: {e}") + verbose_logger.exception("Error getting headers from scope: %s", e) # Return empty Headers object with empty dict return Headers({}) @@ -1455,7 +1458,9 @@ class MCPRequestHandler: if len(allowed_mcp_servers_for_end_user) > 0: has_lower_level_mcp_restrictions = True verbose_logger.debug( - f"End user {user_api_key_auth.end_user_id} has explicit MCP permissions: {allowed_mcp_servers_for_end_user}" + "End user %s has explicit MCP permissions: %s", + user_api_key_auth.end_user_id, + allowed_mcp_servers_for_end_user, ) # Always apply intersection: key/team AND end_user @@ -1466,12 +1471,13 @@ class MCPRequestHandler: filtered_servers.append(_mcp_server) allowed_mcp_servers = filtered_servers verbose_logger.debug( - f"Applied end_user intersection filter. Final allowed servers: {allowed_mcp_servers}" + "Applied end_user intersection filter. Final allowed servers: %s", allowed_mcp_servers ) # If flag is enabled but end_user has no permissions, block all access elif general_settings.get("require_end_user_mcp_access_defined", False): verbose_logger.debug( - f"require_end_user_mcp_access_defined=True and end_user {user_api_key_auth.end_user_id} has no MCP permissions - blocking MCP access" + "require_end_user_mcp_access_defined=True and end_user %s has no MCP permissions - blocking MCP access", + user_api_key_auth.end_user_id, ) return [] @@ -1487,7 +1493,7 @@ class MCPRequestHandler: # Intersect: agent can only use servers allowed by BOTH key/team AND agent config allowed_mcp_servers = [s for s in allowed_mcp_servers if s in allowed_mcp_servers_for_agent] verbose_logger.debug( - f"Applied agent intersection filter. Final allowed servers: {allowed_mcp_servers}" + "Applied agent intersection filter. Final allowed servers: %s", allowed_mcp_servers ) ######################################################### @@ -1514,9 +1520,9 @@ class MCPRequestHandler: if isinstance(e, UnloadableEntitlementError): # A ceiling we KNOW exists and cannot read. Denying is the only answer that does not # widen this caller past what an operator configured, for both caller shapes. - verbose_logger.warning(f"Denying MCP access, entitlement unreadable: {e}") + verbose_logger.warning("Denying MCP access, entitlement unreadable: %s", e) else: - verbose_logger.warning(f"Failed to get allowed MCP servers: {e}") + verbose_logger.warning("Failed to get allowed MCP servers: %s", e) return [] @staticmethod @@ -1543,8 +1549,9 @@ class MCPRequestHandler: allowed_mcp_servers_for_org = await MCPRequestHandler._get_allowed_mcp_servers_for_org(user_api_key_auth) if allowed_mcp_servers_for_org is None: verbose_logger.warning( - f"MCP org ceiling unresolved for org_id={user_api_key_auth.org_id!r}; " - f"{'denying (keyless admitted subject)' if keyless_source else 'leaving uncapped (key auth)'}" + "MCP org ceiling unresolved for org_id=%r; %s", + user_api_key_auth.org_id, + "denying (keyless admitted subject)" if keyless_source else "leaving uncapped (key auth)", ) return [] if keyless_source else allowed_mcp_servers if len(allowed_mcp_servers_for_org) == 0: @@ -1556,7 +1563,7 @@ class MCPRequestHandler: else: # No lower-level restrictions → org list becomes the ceiling. capped = allowed_mcp_servers_for_org - verbose_logger.debug(f"Applied org ceiling filter. Final allowed servers: {capped}") + verbose_logger.debug("Applied org ceiling filter. Final allowed servers: %s", capped) return capped @staticmethod @@ -1649,7 +1656,7 @@ class MCPRequestHandler: # Fault isolation is per SOURCE: an unresolvable team contributes nothing (fail closed for # it alone, access only narrows) while every other source stands. Raising would collapse the # whole union to deny-all over one momentarily-unreadable row. - verbose_logger.warning(f"MCP admitted-subject source team {team_id!r} unresolvable, skipping: {e}") + verbose_logger.warning("MCP admitted-subject source team %r unresolvable, skipping: %s", team_id, e) return None if team_obj is None: return None @@ -1682,10 +1689,10 @@ class MCPRequestHandler: proxy_logging_obj=proxy_logging_obj, ) except BudgetExceededError as e: - verbose_logger.info(f"MCP admitted-subject source team {team_id!r} over budget, not a grantor: {e}") + verbose_logger.info("MCP admitted-subject source team %r over budget, not a grantor: %s", team_id, e) return None except Exception as e: # noqa: BLE001 # per-source isolation: a budget-check fault narrows, never raises - verbose_logger.warning(f"MCP budget check failed for source team {team_id!r}, skipping source: {e}") + verbose_logger.warning("MCP budget check failed for source team %r, skipping source: %s", team_id, e) return None return team_obj @@ -1738,7 +1745,7 @@ class MCPRequestHandler: billed.org_id = source.org_id return billed except Exception as e: # noqa: BLE001 # attribution must never fail an authorized call - verbose_logger.warning(f"MCP billing attribution failed for {tool_name!r}, billing the user: {e}") + verbose_logger.warning("MCP billing attribution failed for %r, billing the user: %s", tool_name, e) return auth @staticmethod @@ -1828,7 +1835,7 @@ class MCPRequestHandler: ) verbose_logger.debug( - f"MCP team permission lookup: team_id={user_api_key_auth.team_id if user_api_key_auth else None}" + "MCP team permission lookup: team_id=%s", user_api_key_auth.team_id if user_api_key_auth else None ) if not user_api_key_auth or not user_api_key_auth.team_id or not prisma_client: return None @@ -1946,9 +1953,9 @@ class MCPRequestHandler: # than the None (allow-all) key auth gets for an indeterminate fault. unreadable_entitlement = isinstance(e, UnloadableEntitlementError) if unreadable_entitlement: - verbose_logger.warning(f"Denying MCP tools, entitlement unreadable: {e}") + verbose_logger.warning("Denying MCP tools, entitlement unreadable: %s", e) else: - verbose_logger.warning(f"Failed to get allowed tools for server: {e}") + verbose_logger.warning("Failed to get allowed tools for server: %s", e) # Fail CLOSED for a keyless admitted subject: ANY error must deny the server's tools ([]), # not collapse to allow-all (None); key/JWT auth keeps its prior allow-all-on-error. Both # keyless_source AND the marker are needed: each source resolves through an UNMARKED auth, so @@ -1998,8 +2005,9 @@ class MCPRequestHandler: if keyless_source or isinstance(e, UnloadableEntitlementError): raise verbose_logger.warning( - f"MCP org tool ceiling unresolvable for org_id={user_api_key_auth.org_id!r}; " - f"skipping org intersect, key/team/agent restrictions stand: {e}" + "MCP org tool ceiling unresolvable for org_id=%r; skipping org intersect, key/team/agent restrictions stand: %s", + user_api_key_auth.org_id, + e, ) return allowed_tools org_tools = ( @@ -2102,7 +2110,7 @@ class MCPRequestHandler: # Permission entries may be server_ids OR names/aliases — expand to ids. return global_mcp_server_manager.expand_permission_list(raw_server_ids) except Exception as e: - verbose_logger.warning(f"Failed to get key access group MCP server grants: {e}") + verbose_logger.warning("Failed to get key access group MCP server grants: %s", e) return [] @staticmethod @@ -2180,7 +2188,7 @@ class MCPRequestHandler: all_servers = direct_mcp_servers + access_group_servers + tool_perm_servers + toolset_servers return list(set(all_servers)) except Exception as e: - verbose_logger.warning(f"Failed to get allowed MCP servers for key: {e}") + verbose_logger.warning("Failed to get allowed MCP servers for key: %s", e) return [] @staticmethod @@ -2238,7 +2246,7 @@ class MCPRequestHandler: proxy_logging_obj=proxy_logging_obj, ) except Exception as e: # noqa: BLE001 # a team-resolution blip narrows access, never raises - verbose_logger.warning(f"Failed to resolve user teams for MCP grant: {e}") + verbose_logger.warning("Failed to resolve user teams for MCP grant: %s", e) return [] if user_object is None or not user_object.teams: return [] @@ -2323,7 +2331,7 @@ class MCPRequestHandler: servers = await MCPRequestHandler._team_granted_servers(team_obj, team_access_group_servers) return list(servers) except Exception as e: - verbose_logger.warning(f"Failed to get allowed MCP servers for team: {e}") + verbose_logger.warning("Failed to get allowed MCP servers for team: %s", e) return [] @staticmethod @@ -2403,7 +2411,7 @@ class MCPRequestHandler: # CONFIRMED absent: places no ceiling. Every OTHER exception propagates as an unresolvable # ceiling (denies for a keyless source, fail-open for a key); catching bare Exception here # would treat a DB outage as "no org" and silently drop a real ceiling for its duration. - verbose_logger.debug(f"MCP org ceiling: org {user_api_key_auth.org_id!r} does not exist: {e}") + verbose_logger.debug("MCP org ceiling: org %r does not exist: %s", user_api_key_auth.org_id, e) return None if org_obj is None or not org_obj.object_permission_id: @@ -2462,7 +2470,7 @@ class MCPRequestHandler: # A NAMED-but-unreadable ceiling is a stronger fact than "unresolved" and denies everywhere. if isinstance(e, UnloadableEntitlementError): raise - verbose_logger.warning(f"Failed to get allowed MCP servers for org: {e}") + verbose_logger.warning("Failed to get allowed MCP servers for org: %s", e) return None @staticmethod @@ -2490,7 +2498,7 @@ class MCPRequestHandler: route="/mcp", ) except Exception as e: # noqa: BLE001 # entitlement unknown, not known-absent: no ceiling, as before this level - verbose_logger.warning(f"Failed to resolve end_user for MCP permissions: {e}") + verbose_logger.warning("Failed to resolve end_user for MCP permissions: %s", e) return None if end_user_obj is None: @@ -2554,7 +2562,7 @@ class MCPRequestHandler: all_servers = direct_mcp_servers + access_group_servers + tool_perm_servers return list(set(all_servers)) except Exception as e: - verbose_logger.warning(f"Failed to get allowed MCP servers for end_user: {e}") + verbose_logger.warning("Failed to get allowed MCP servers for end_user: %s", e) return [] @staticmethod @@ -2637,7 +2645,7 @@ class MCPRequestHandler: ) return object_permission_id except Exception as e: # noqa: BLE001 # unknown whether entitled at all: no ceiling, as before - verbose_logger.warning(f"MCP user entitlement: link for {user_id!r} unresolved, no ceiling: {e}") + verbose_logger.warning("MCP user entitlement: link for %r unresolved, no ceiling: %s", user_id, e) return None @staticmethod @@ -2669,7 +2677,7 @@ class MCPRequestHandler: ) return list(set(direct_mcp_servers + access_group_servers + tool_perm_servers)) except Exception as e: # noqa: BLE001 # any resolution fault is an unresolved ceiling, never "no ceiling" - verbose_logger.warning(f"Failed to get allowed MCP servers for user: {e}") + verbose_logger.warning("Failed to get allowed MCP servers for user: %s", e) return None @staticmethod @@ -2700,7 +2708,7 @@ class MCPRequestHandler: if not entitled: return tuple(allowed_mcp_servers), False capped = tuple(server for server in allowed_mcp_servers if server in set(entitled)) - verbose_logger.debug(f"Applied user ceiling filter. Final allowed servers: {capped}") + verbose_logger.debug("Applied user ceiling filter. Final allowed servers: %s", capped) return capped, True @staticmethod @@ -2739,7 +2747,7 @@ class MCPRequestHandler: try: object_permissions = await MCPRequestHandler._get_user_object_permission(user_api_key_auth) except Exception as e: # noqa: BLE001 # an unresolved human entitlement must deny, not widen - verbose_logger.warning(f"MCP user tool ceiling unresolvable, denying tools on {server_id!r}: {e}") + verbose_logger.warning("MCP user tool ceiling unresolvable, denying tools on %r: %s", server_id, e) return [] if object_permissions is None or not object_permissions.mcp_tool_permissions: @@ -2785,7 +2793,7 @@ class MCPRequestHandler: ) return object_permission_id except Exception as e: # noqa: BLE001 # entitlement unknown, not known-absent: no ceiling, as before this level - verbose_logger.warning(f"Failed to resolve object_permission_id for agent {agent_id!r}: {e}") + verbose_logger.warning("Failed to resolve object_permission_id for agent %r: %s", agent_id, e) return None @staticmethod @@ -2869,7 +2877,7 @@ class MCPRequestHandler: all_servers = expanded_direct_servers + access_group_servers return list(set(all_servers)) except Exception as e: - verbose_logger.warning(f"Failed to get allowed MCP servers for agent: {e}") + verbose_logger.warning("Failed to get allowed MCP servers for agent: %s", e) return [] @staticmethod @@ -2911,7 +2919,7 @@ class MCPRequestHandler: tools = global_mcp_server_manager.expand_tool_permissions(mcp_tool_permissions).get(server_id) return list(tools) if tools else None except Exception as e: - verbose_logger.warning(f"Failed to get agent tool permissions for server: {e}") + verbose_logger.warning("Failed to get agent tool permissions for server: %s", e) return None @staticmethod @@ -2940,7 +2948,7 @@ class MCPRequestHandler: for server in mcp_servers: server_ids.add(server.server_id) except Exception as e: - verbose_logger.debug(f"Error getting MCP servers from access groups: {e}") + verbose_logger.debug("Error getting MCP servers from access groups: %s", e) return server_ids @staticmethod @@ -2969,7 +2977,7 @@ class MCPRequestHandler: return list(server_ids) except Exception as e: - verbose_logger.warning(f"Failed to get MCP servers from access groups: {e}") + verbose_logger.warning("Failed to get MCP servers from access groups: %s", e) return [] @staticmethod @@ -3029,7 +3037,7 @@ class MCPRequestHandler: return key_object_permission.mcp_access_groups or [] except Exception as e: - verbose_logger.warning(f"Failed to get MCP access groups for key: {e}") + verbose_logger.warning("Failed to get MCP access groups for key: %s", e) return [] @staticmethod @@ -3077,7 +3085,7 @@ class MCPRequestHandler: return object_permissions.mcp_access_groups or [] except Exception as e: - verbose_logger.warning(f"Failed to get MCP access groups for team: {e}") + verbose_logger.warning("Failed to get MCP access groups for team: %s", e) return [] @staticmethod diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 672396afd05..c5469785778 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -570,7 +570,7 @@ async def get_all_mcp_servers( decrypt_global_env_var_values(table.env_vars) return tables except Exception as e: - verbose_proxy_logger.debug(f"litellm.proxy._experimental.mcp_server.db.py::get_all_mcp_servers - {e}") + verbose_proxy_logger.debug("litellm.proxy._experimental.mcp_server.db.py::get_all_mcp_servers - %s", e) return [] diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index e16fb0d0e00..40844ec1937 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -2159,8 +2159,9 @@ async def _build_oauth_protected_resource_response( upstream_metadata = await fetch_upstream_oauth_protected_resource(mcp_server) except Exception as exc: verbose_logger.warning( - "Failed to fetch upstream oauth-protected-resource metadata " - f"for pass-through MCP server {mcp_server.name!r}: {exc}" + "Failed to fetch upstream oauth-protected-resource metadata for pass-through MCP server %r: %s", + mcp_server.name, + exc, ) raise HTTPException( status_code=502, @@ -2179,7 +2180,7 @@ async def _build_oauth_protected_resource_response( # so we must not fall through to the default gateway metadata — # that would point clients at the wrong IdP. verbose_logger.warning( - f"Upstream oauth-protected-resource metadata unavailable for pass-through MCP server {mcp_server.name!r}" + "Upstream oauth-protected-resource metadata unavailable for pass-through MCP server %r", mcp_server.name ) raise HTTPException( status_code=502, diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 0a6a0374d13..66a55f30b74 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -1566,7 +1566,7 @@ class MCPServerManager: if target_server_name == server_name and alias_name not in used_aliases: alias = alias_name used_aliases.add(alias_name) - verbose_logger.debug(f"Mapped alias '{alias_name}' to server '{server_name}'") + verbose_logger.debug("Mapped alias '%s' to server '%s'", alias_name, server_name) break # Create a temporary server object to use with get_server_prefix utility @@ -1785,7 +1785,7 @@ class MCPServerManager: # Check if this is an OpenAPI-based server spec_path = server_config.get("spec_path", None) if spec_path: - verbose_logger.info(f"Loading OpenAPI spec from {spec_path} for server {server_name}") + verbose_logger.info("Loading OpenAPI spec from %s for server %s", spec_path, server_name) await self._register_openapi_tools( spec_path=spec_path, server=new_server, @@ -1793,7 +1793,7 @@ class MCPServerManager: ) verbose_logger.debug( - f"Loaded MCP Servers: {json.dumps(_redacted_registry_dump(self.config_mcp_servers), indent=4)}" + "Loaded MCP Servers: %s", json.dumps(_redacted_registry_dump(self.config_mcp_servers), indent=4) ) await self._hydrate_config_servers_dcr_clients() @@ -1856,7 +1856,7 @@ class MCPServerManager: # Use base_url from config if provided, otherwise extract from spec if not base_url: base_url = get_openapi_base_url(spec, spec_path) - verbose_logger.info(f"Registering OpenAPI tools for server {server.name} with base URL: {base_url}") + verbose_logger.info("Registering OpenAPI tools for server %s with base URL: %s", server.name, base_url) # Get server prefix for tool naming server_prefix = get_server_prefix(server) @@ -1892,7 +1892,7 @@ class MCPServerManager: ) verbose_logger.debug( - f"Using headers for OpenAPI tools (excluding sensitive values): {list(headers.keys())}" + "Using headers for OpenAPI tools (excluding sensitive values): %s", list(headers.keys()) ) # Extract and register tools from OpenAPI paths @@ -1900,7 +1900,7 @@ class MCPServerManager: components = spec.get("components", {}) registered_count = 0 - verbose_logger.debug(f"Processing {len(paths)} paths from OpenAPI spec") + verbose_logger.debug("Processing %s paths from OpenAPI spec", len(paths)) for path, path_item in paths.items(): for method in ["get", "post", "put", "delete", "patch"]: @@ -1946,12 +1946,12 @@ class MCPServerManager: self.tool_name_to_mcp_server_name_mapping[prefixed_tool_name] = server_prefix registered_count += 1 - verbose_logger.debug(f"Registered OpenAPI tool: {prefixed_tool_name} for server {server.name}") + verbose_logger.debug("Registered OpenAPI tool: %s for server %s", prefixed_tool_name, server.name) - verbose_logger.info(f"Successfully registered {registered_count} OpenAPI tools for server {server.name}") + verbose_logger.info("Successfully registered %s OpenAPI tools for server %s", registered_count, server.name) except Exception as e: - verbose_logger.error(f"Failed to register OpenAPI tools for server {server.name}: {e}") + verbose_logger.error("Failed to register OpenAPI tools for server %s: %s", server.name, e) raise e def _cleanup_server_tool_routing_artifacts(self, server: MCPServer) -> None: @@ -2000,7 +2000,7 @@ class MCPServerManager: verbose_logger.debug("Removed MCP Server: %s", mcp_server.server_id or mcp_server.server_name) self._cleanup_server_tool_routing_artifacts(evicted) else: - verbose_logger.warning(f"Server ID {mcp_server.server_id} not found in registry") + verbose_logger.warning("Server ID %s not found in registry", mcp_server.server_id) def _resolve_env_vars_list( self, @@ -2295,7 +2295,7 @@ class MCPServerManager: async def _maybe_register_openapi_tools(self, server: MCPServer, *, initialize_mapping: bool = True): """Register OpenAPI tools if the server has a spec_path configured.""" if server.spec_path: - verbose_logger.info(f"Loading OpenAPI spec from {server.spec_path} for server {server.name}") + verbose_logger.info("Loading OpenAPI spec from %s for server %s", server.spec_path, server.name) await self._register_openapi_tools( spec_path=server.spec_path, server=server, @@ -2323,10 +2323,10 @@ class MCPServerManager: self._assign_unique_short_prefix(new_server) self.registry[mcp_server.server_id] = new_server await self._maybe_register_openapi_tools(new_server) - verbose_logger.debug(f"Added MCP Server: {new_server.name}") + verbose_logger.debug("Added MCP Server: %s", new_server.name) except Exception as e: - verbose_logger.debug(f"Failed to add MCP server: {e}") + verbose_logger.debug("Failed to add MCP server: %s", e) raise e async def update_server(self, mcp_server: LiteLLM_MCPServerTable): @@ -2357,10 +2357,10 @@ class MCPServerManager: self._assign_unique_short_prefix(new_server) self.registry[mcp_server.server_id] = new_server await self._maybe_register_openapi_tools(new_server) - verbose_logger.debug(f"Updated MCP Server: {new_server.name}") + verbose_logger.debug("Updated MCP Server: %s", new_server.name) except Exception as e: - verbose_logger.debug(f"Failed to udpate MCP server: {e}") + verbose_logger.debug("Failed to udpate MCP server: %s", e) raise e def get_all_mcp_server_ids(self) -> set[str]: @@ -2386,7 +2386,7 @@ class MCPServerManager: await user_api_key_cache.async_delete_cache(key=self.get_byom_submitted_servers_cache_key(user_id)) except Exception as e: # noqa: BLE001 - verbose_logger.warning(f"Failed to invalidate BYOM submitted MCP server cache: {e}") + verbose_logger.warning("Failed to invalidate BYOM submitted MCP server cache: %s", e) async def _get_active_submitted_mcp_server_ids_for_user( self, user_api_key_auth: UserAPIKeyAuth | None @@ -2401,7 +2401,7 @@ class MCPServerManager: ) from litellm.proxy.proxy_server import prisma_client, user_api_key_cache except Exception as e: # noqa: BLE001 - verbose_logger.warning(f"Failed to load BYOM submitted MCP server cache dependencies: {e}") + verbose_logger.warning("Failed to load BYOM submitted MCP server cache dependencies: %s", e) return [] byom_cache_key = self.get_byom_submitted_servers_cache_key(submitter_user_id) @@ -2411,7 +2411,7 @@ class MCPServerManager: if cached_submitted_server_ids is not None: submitted_server_ids = cast(list[str], cached_submitted_server_ids) except Exception as e: # noqa: BLE001 - verbose_logger.warning(f"Failed to read BYOM submitted MCP server cache: {e}") + verbose_logger.warning("Failed to read BYOM submitted MCP server cache: %s", e) if submitted_server_ids is None: if prisma_client is None: @@ -2422,7 +2422,7 @@ class MCPServerManager: prisma_client, submitter_user_id ) except Exception as e: # noqa: BLE001 - verbose_logger.warning(f"Failed to read BYOM submitted MCP servers from database: {e}") + verbose_logger.warning("Failed to read BYOM submitted MCP servers from database: %s", e) submitted_server_ids = [] try: await user_api_key_cache.async_set_cache( @@ -2431,7 +2431,7 @@ class MCPServerManager: ttl=60, ) except Exception as e: # noqa: BLE001 - verbose_logger.warning(f"Failed to write BYOM submitted MCP server cache: {e}") + verbose_logger.warning("Failed to write BYOM submitted MCP server cache: %s", e) return [server_id for server_id in submitted_server_ids if self.get_mcp_server_by_id(server_id) is not None] @@ -2522,7 +2522,7 @@ class MCPServerManager: key_object_permission.mcp_servers is not None ) if has_explicit_object_permission: - verbose_logger.debug(f"Object permission mcp_servers explicitly set: {key_object_permission.mcp_servers}") + verbose_logger.debug("Object permission mcp_servers explicitly set: %s", key_object_permission.mcp_servers) # BYOM creator visibility never widens a key that was explicitly scoped: # only keys without their own mcp_servers list get submitted servers unioned in. @@ -2551,7 +2551,7 @@ class MCPServerManager: # Get allowed servers from object permissions (respects object_permission even for admins) allowed_mcp_servers = await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth) - verbose_logger.debug(f"Allowed MCP Servers for user api key auth: {allowed_mcp_servers}") + verbose_logger.debug("Allowed MCP Servers for user api key auth: %s", allowed_mcp_servers) combined_servers = set(allowed_mcp_servers) combined_servers.update( await self.operator_open_server_ids( @@ -2647,7 +2647,7 @@ class MCPServerManager: ) return tool_permissions except Exception as e: - verbose_logger.warning(f"Failed to resolve toolset permissions: {e}") + verbose_logger.warning("Failed to resolve toolset permissions: %s", e) return {} def invalidate_toolset_cache(self, toolset_id: str | None = None) -> None: @@ -2682,7 +2682,7 @@ class MCPServerManager: for k in keys_to_remove: cache_dict.pop(k, None) except Exception as e: - verbose_logger.warning(f"invalidate_toolset_cache: failed to evict in-memory entries: {e}") + verbose_logger.warning("invalidate_toolset_cache: failed to evict in-memory entries: %s", e) async def get_toolset_by_name_cached( self, @@ -2760,11 +2760,11 @@ class MCPServerManager: try: server = self.get_mcp_server_by_id(server_id) if server is None: - verbose_logger.warning(f"MCP Server {server_id} not found") + verbose_logger.warning("MCP Server %s not found", server_id) return [] return await self._get_tools_from_server(server) except Exception as e: - verbose_logger.warning(f"Failed to get tools from server {server_id}: {e}") + verbose_logger.warning("Failed to get tools from server %s: %s", server_id, e) return [] async def list_tools( @@ -2793,7 +2793,7 @@ class MCPServerManager: """Fetch tools from a single server with error handling.""" server = self.get_mcp_server_by_id(server_id) if server is None: - verbose_logger.warning(f"MCP Server {server_id} not found") + verbose_logger.warning("MCP Server %s not found", server_id) return [] # Get server-specific auth header if available @@ -2822,7 +2822,7 @@ class MCPServerManager: return tools except Exception as e: verbose_logger.warning( - f"Failed to list tools from server {server.name}: {e}. Continuing with other servers." + "Failed to list tools from server %s: %s. Continuing with other servers.", server.name, e ) return [] @@ -2833,7 +2833,7 @@ class MCPServerManager: # Flatten results into single list list_tools_result: list[MCPTool] = [tool for tools in results for tool in tools] - verbose_logger.info(f"Successfully fetched {len(list_tools_result)} tools total from all servers") + verbose_logger.info("Successfully fetched %s tools total from all servers", len(list_tools_result)) return list_tools_result ######################################################### @@ -3345,8 +3345,8 @@ class MCPServerManager: global_mcp_tool_registry, ) - verbose_logger.debug(f"Connecting to url: {server.url}") - verbose_logger.info(f"_get_tools_from_server for {server.name}...") + verbose_logger.debug("Connecting to url: %s", server.url) + verbose_logger.info("_get_tools_from_server for %s...", server.name) client = None @@ -3476,12 +3476,12 @@ class MCPServerManager: www_authenticate=None if server.is_dcr_bridge else challenge_header, server_name=server.name, ) from e - verbose_logger.warning(f"Failed to get tools from server {server.name}: {e}") + verbose_logger.warning("Failed to get tools from server %s: %s", server.name, e) raise MCPServerListError(ServerListFault(tag="internal", status_code=e.status_code), server.name) from e except MCPServerListError: raise except Exception as e: - verbose_logger.warning(f"Failed to get tools from server {server.name}: {e}") + verbose_logger.warning("Failed to get tools from server %s: %s", server.name, e) raise_classified_list_failure(e, server.name, suppress_challenge=server.is_dcr_bridge) async def get_prompts_from_server( @@ -3503,8 +3503,8 @@ class MCPServerManager: List[Prompt]: List of prompts available on the server with prefixed names """ - verbose_logger.debug(f"Connecting to url: {server.url}") - verbose_logger.info(f"get_prompts_from_server for {server.name}...") + verbose_logger.debug("Connecting to url: %s", server.url) + verbose_logger.info("get_prompts_from_server for %s...", server.name) client = None @@ -3532,7 +3532,7 @@ class MCPServerManager: return prefixed_or_original_prompts except Exception as e: - verbose_logger.warning(f"Failed to get prompts from server {server.name}: {e}") + verbose_logger.warning("Failed to get prompts from server %s: %s", server.name, e) return [] async def get_resources_from_server( @@ -3545,8 +3545,8 @@ class MCPServerManager: ) -> list[Resource]: """Fetch available resources from a single MCP server.""" - verbose_logger.debug(f"Connecting to url: {server.url}") - verbose_logger.info(f"get_resources_from_server for {server.name}...") + verbose_logger.debug("Connecting to url: %s", server.url) + verbose_logger.info("get_resources_from_server for %s...", server.name) client = None @@ -3574,7 +3574,7 @@ class MCPServerManager: return prefixed_resources except Exception as e: - verbose_logger.warning(f"Failed to get resources from server {server.name}: {e}") + verbose_logger.warning("Failed to get resources from server %s: %s", server.name, e) return [] async def get_resource_templates_from_server( @@ -3587,8 +3587,8 @@ class MCPServerManager: ) -> list[ResourceTemplate]: """Fetch available resource templates from a single MCP server.""" - verbose_logger.debug(f"Connecting to url: {server.url}") - verbose_logger.info(f"get_resource_templates_from_server for {server.name}...") + verbose_logger.debug("Connecting to url: %s", server.url) + verbose_logger.info("get_resource_templates_from_server for %s...", server.name) client = None @@ -3618,7 +3618,7 @@ class MCPServerManager: return prefixed_templates except Exception as e: - verbose_logger.warning(f"Failed to get resource templates from server {server.name}: {e}") + verbose_logger.warning("Failed to get resource templates from server %s: %s", server.name, e) return [] async def read_resource_from_server( @@ -3631,8 +3631,8 @@ class MCPServerManager: ) -> ReadResourceResult: """Read resource contents from a specific MCP server.""" - verbose_logger.debug(f"Connecting to url: {server.url}") - verbose_logger.info(f"read_resource_from_server for {server.name}...") + verbose_logger.debug("Connecting to url: %s", server.url) + verbose_logger.info("read_resource_from_server for %s...", server.name) if server.static_headers: if extra_headers is None: @@ -3663,8 +3663,8 @@ class MCPServerManager: ) -> GetPromptResult: """Fetch a specific prompt definition from a single MCP server.""" - verbose_logger.debug(f"Connecting to url: {server.url}") - verbose_logger.info(f"get_prompt_from_server for {server.name}...") + verbose_logger.debug("Connecting to url: %s", server.url) + verbose_logger.info("get_prompt_from_server for %s...", server.name) if server.static_headers: if extra_headers is None: @@ -4206,19 +4206,19 @@ class MCPServerManager: try: with anyio.fail_after(MCP_TOOL_LISTING_TIMEOUT): tools = await client.list_tools(raise_on_error=True) - verbose_logger.debug(f"Tools from {server_name}: {tools}") + verbose_logger.debug("Tools from %s: %s", server_name, tools) return tools except TimeoutError as e: - verbose_logger.warning(f"Timeout while listing tools from {server_name}") + verbose_logger.warning("Timeout while listing tools from %s", server_name) raise MCPServerListError(ServerListFault(tag="timeout"), server_name) from e except asyncio.CancelledError as e: - verbose_logger.warning(f"Task cancelled while listing tools from {server_name}") + verbose_logger.warning("Task cancelled while listing tools from %s", server_name) raise MCPServerListError(ServerListFault(tag="internal"), server_name) from e except ConnectionError as e: - verbose_logger.warning(f"Connection error while listing tools from {server_name}: {e}") + verbose_logger.warning("Connection error while listing tools from %s: %s", server_name, e) raise MCPServerListError(ServerListFault(tag="unreachable"), server_name) from e except Exception as e: - verbose_logger.warning(f"Error listing tools from {server_name}: {e}") + verbose_logger.warning("Error listing tools from %s: %s", server_name, e) raise_classified_list_failure(e, server_name) _SHORT_PREFIX_MAX_REHASH_ATTEMPTS = 1024 @@ -4315,7 +4315,7 @@ class MCPServerManager: for spelling in iter_known_tool_name_spellings(original_name, server): self.tool_name_to_mcp_server_name_mapping[spelling] = prefix - verbose_logger.info(f"Successfully fetched {len(prefixed_tools)} tools from server {server.name}") + verbose_logger.info("Successfully fetched %s tools from server %s", len(prefixed_tools), server.name) return prefixed_tools def _create_prefixed_prompts( @@ -4342,7 +4342,7 @@ class MCPServerManager: prompt.name = name_to_use prefixed_prompts.append(prompt) - verbose_logger.info(f"Successfully fetched {len(prefixed_prompts)} prompts from server {server.name}") + verbose_logger.info("Successfully fetched %s prompts from server %s", len(prefixed_prompts), server.name) return prefixed_prompts def _create_prefixed_resources( @@ -4358,7 +4358,7 @@ class MCPServerManager: resource.name = name_to_use prefixed_resources.append(resource) - verbose_logger.info(f"Successfully fetched {len(prefixed_resources)} resources from server {server.name}") + verbose_logger.info("Successfully fetched %s resources from server %s", len(prefixed_resources), server.name) return prefixed_resources def _create_prefixed_resource_templates( @@ -4380,7 +4380,7 @@ class MCPServerManager: prefixed_templates.append(resource_template) verbose_logger.info( - f"Successfully fetched {len(prefixed_templates)} resource templates from server {server.name}" + "Successfully fetched %s resource templates from server %s", len(prefixed_templates), server.name ) return prefixed_templates @@ -4639,7 +4639,7 @@ class MCPServerManager: HTTPException, ) as e: # Re-raise guardrail exceptions to properly fail the MCP call - verbose_logger.error(f"Guardrail blocked MCP tool call pre call: {e}") + verbose_logger.error("Guardrail blocked MCP tool call pre call: %s", e) raise e return hook_result @@ -4995,7 +4995,7 @@ class MCPServerManager: GuardrailRaisedException, HTTPException, ) as e: - verbose_logger.error(f"Guardrail blocked MCP tool call during result check: {e}") + verbose_logger.error("Guardrail blocked MCP tool call during result check: %s", e) raise e # If proxy_logging_obj is None, the tool call result is at index 0 @@ -5194,7 +5194,7 @@ class MCPServerManager: GuardrailRaisedException, HTTPException, ) as e: - verbose_logger.error(f"Guardrail blocked MCP tool call during result check: {e}") + verbose_logger.error("Guardrail blocked MCP tool call during result check: %s", e) raise e async def call_tool( @@ -5345,7 +5345,7 @@ class MCPServerManager: asyncio.create_task(self._initialize_tool_name_to_mcp_server_name_mapping()) except RuntimeError as e: # no running event loop verbose_logger.exception( - f"No running event loop - skipping tool name to MCP server name mapping initialization: {e}" + "No running event loop - skipping tool name to MCP server name mapping initialization: %s", e ) async def _initialize_tool_name_to_mcp_server_name_mapping(self): @@ -5364,12 +5364,12 @@ class MCPServerManager: # at startup we have none, so an upstream 401 is normal. # Swallow it so we keep mapping the remaining servers. verbose_logger.debug( - f"Skipping tool name mapping for server {server.name} due to upstream auth error: {e}" + "Skipping tool name mapping for server %s due to upstream auth error: %s", server.name, e ) continue except Exception as e: verbose_logger.warning( - f"Failed to get tools from server {server.name} during tool name mapping initialization: {e}" + "Failed to get tools from server %s during tool name mapping initialization: %s", server.name, e ) continue for tool in tools: @@ -5449,7 +5449,7 @@ class MCPServerManager: } ) db_mcp_servers = [LiteLLM_MCPServerTable.model_validate(r.model_dump()) for r in raw_rows] - verbose_logger.info(f"Found {len(db_mcp_servers)} MCP servers in database") + verbose_logger.info("Found %s MCP servers in database", len(db_mcp_servers)) previous_registry = self.registry new_registry: dict[str, MCPServer] = {} @@ -5481,7 +5481,7 @@ class MCPServerManager: alias=getattr(server, "alias", None), server_name=getattr(server, "server_name", None), ) - verbose_logger.debug(f"Building server from DB: {server.server_id} ({server.server_name})") + verbose_logger.debug("Building server from DB: %s (%s)", server.server_id, server.server_name) # raw_rows come straight from the DB, so their global env var # values (like credentials) are still encrypted here, unlike the # already-decrypted records add_server/update_server are handed. @@ -5776,7 +5776,7 @@ class MCPServerManager: server = self.get_mcp_server_by_id(server_id) if not server: - verbose_logger.warning(f"MCP Server {server_id} not found") + verbose_logger.warning("MCP Server %s not found", server_id) return LiteLLM_MCPServerTable( server_id=server_id, server_name=None, @@ -5929,7 +5929,7 @@ class MCPServerManager: for server_id in allowed_server_ids: server = self.get_mcp_server_by_id(server_id) if not server: - verbose_logger.warning(f"MCP Server {server_id} not found in registry") + verbose_logger.warning("MCP Server %s not found in registry", server_id) continue mcp_server_table = self._build_mcp_server_table(server) diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py index db2851c60aa..5b30b58c0e3 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py @@ -138,8 +138,9 @@ def get_base_url(spec: dict[str, Any], spec_path: str | None = None) -> str: base_domain = f"{parsed.scheme}://{parsed.netloc}" full_base_url = base_domain + server_url verbose_logger.info( - f"OpenAPI spec has relative server URL '{server_url}'. " - f"Deriving base from spec_path: {full_base_url}" + "OpenAPI spec has relative server URL '%s'. Deriving base from spec_path: %s", + server_url, + full_base_url, ) return full_base_url @@ -160,12 +161,12 @@ def get_base_url(spec: dict[str, Any], spec_path: str | None = None) -> str: ]: if spec_path.endswith(suffix): base_url = spec_path[: -len(suffix)] - verbose_logger.info(f"No server info in OpenAPI spec. Using derived base URL: {base_url}") + verbose_logger.info("No server info in OpenAPI spec. Using derived base URL: %s", base_url) return base_url if spec_path.split("/")[-1].endswith((".json", ".yaml", ".yml")): base_url = "/".join(spec_path.split("/")[:-1]) - verbose_logger.info(f"No server info in OpenAPI spec. Using derived base URL: {base_url}") + verbose_logger.info("No server info in OpenAPI spec. Using derived base URL: %s", base_url) return base_url return "" @@ -497,4 +498,4 @@ def register_tools_from_openapi(spec: dict[str, Any], base_url: str): input_schema=input_schema, handler=tool_func, ) - verbose_logger.debug(f"Registered tool: {tool_name}") + verbose_logger.debug("Registered tool: %s", tool_name) diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index d0458db51c6..57dfe6823b9 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -50,7 +50,7 @@ MCP_AVAILABLE: bool = True try: importlib.import_module("mcp") except ImportError as e: - verbose_logger.debug(f"MCP module not found: {e}") + verbose_logger.debug("MCP module not found: %s", e) MCP_AVAILABLE = False @@ -328,8 +328,10 @@ if MCP_AVAILABLE: return {"Authorization": f"Bearer {cred['access_token']}"} except Exception as e: verbose_logger.warning( - f"_get_user_oauth_extra_headers: failed to retrieve credential for " - f"user={user_id} server={server_id}: {e}" + "_get_user_oauth_extra_headers: failed to retrieve credential for user=%s server=%s: %s", + user_id, + server_id, + e, ) return None @@ -356,7 +358,7 @@ if MCP_AVAILABLE: creds = await list_user_oauth_credentials(prisma_client, user_id) return {c["server_id"]: c for c in creds if "server_id" in c} except Exception as e: - verbose_logger.warning(f"_prefetch_user_oauth_creds: failed to prefetch for user={user_id}: {e}") + verbose_logger.warning("_prefetch_user_oauth_creds: failed to prefetch for user=%s: %s", user_id, e) return {} def _create_tool_response_objects(tools, server: MCPServer): @@ -641,7 +643,7 @@ if MCP_AVAILABLE: raise except MCPServerListError as e: fault = classify_list_exception(e) - verbose_logger.info(f"Listing tools from {server.name} failed with a {fault.tag} fault") + verbose_logger.info("Listing tools from %s failed with a %s fault", server.name, fault.tag) raise HTTPException( status_code=list_fault_http_status(fault), detail={ @@ -650,7 +652,7 @@ if MCP_AVAILABLE: }, ) from e except Exception as e: - verbose_logger.exception(f"Error getting tools from {server.name}: {e}") + verbose_logger.exception("Error getting tools from %s: %s", server.name, e) return { "tools": [], "error": "server_error", @@ -862,7 +864,7 @@ if MCP_AVAILABLE: ) list_tools_result.extend(tools_result) except Exception as e: - verbose_logger.exception(f"Error getting tools from {server.name}: {e}") + verbose_logger.exception("Error getting tools from %s: %s", server.name, e) errors.append( f"{get_server_prefix(server)}: {classify_list_exception(e).tag}" if isinstance(e, (MCPServerListError, MCPUpstreamAuthError)) @@ -1052,7 +1054,7 @@ if MCP_AVAILABLE: }, ) except BlockedPiiEntityError as e: - verbose_logger.error(f"BlockedPiiEntityError in MCP tool call: {e}") + verbose_logger.error("BlockedPiiEntityError in MCP tool call: %s", e) raise HTTPException( status_code=400, detail={ @@ -1063,7 +1065,7 @@ if MCP_AVAILABLE: }, ) except GuardrailRaisedException as e: - verbose_logger.error(f"GuardrailRaisedException in MCP tool call: {e}") + verbose_logger.error("GuardrailRaisedException in MCP tool call: %s", e) raise HTTPException( status_code=400, detail={ @@ -1076,16 +1078,16 @@ if MCP_AVAILABLE: # A client-forwarded pass-through upstream 401 from either the direct or the virtual call # branch. Relay it as a 401 + WWW-Authenticate so the MCP client can re-run upstream OAuth, # and log at info: an expected caller-must-reauth signal, not an operator-actionable error. - verbose_logger.info(f"MCP tool call relaying upstream HTTP {e.status_code}") + verbose_logger.info("MCP tool call relaying upstream HTTP %s", e.status_code) raise _relay_upstream_auth_http_exception(e, request) except HTTPException as e: # Locally generated denials (tool/server permission, IP filtering, BYOK) stay at error level # so restriction probing keeps full monitoring visibility; the relayed upstream 401 above is # the only status demoted to info. - verbose_logger.error(f"HTTPException in MCP tool call: {e}") + verbose_logger.error("HTTPException in MCP tool call: %s", e) raise e except Exception as e: - verbose_logger.exception(f"Unexpected error in MCP tool call: {e}") + verbose_logger.exception("Unexpected error in MCP tool call: %s", e) raise HTTPException( status_code=500, detail={ diff --git a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py index a7cc9fe3ed0..e2c60a488a4 100644 --- a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py +++ b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py @@ -98,7 +98,7 @@ class SemanticMCPToolFilter: tools = await global_mcp_server_manager.get_tools_for_server(server_id) all_tools.extend(tools) except Exception as e: - verbose_logger.warning(f"Failed to fetch tools from server {server_id}: {e}") + verbose_logger.warning("Failed to fetch tools from server %s: %s", server_id, e) continue if not all_tools: @@ -106,11 +106,11 @@ class SemanticMCPToolFilter: self.tool_router = None return - verbose_logger.info(f"Fetched {len(all_tools)} tools from {len(registry)} MCP servers") + verbose_logger.info("Fetched %s tools from %s MCP servers", len(all_tools), len(registry)) self._build_router(all_tools) except Exception as e: - verbose_logger.error(f"Failed to build router from MCP registry: {e}") + verbose_logger.error("Failed to build router from MCP registry: %s", e) self.tool_router = None raise @@ -172,10 +172,10 @@ class SemanticMCPToolFilter: auto_sync="local", ) - verbose_logger.info(f"Built semantic router with {len(routes)} tools") + verbose_logger.info("Built semantic router with %s tools", len(routes)) except Exception as e: - verbose_logger.error(f"Failed to build semantic router: {e}") + verbose_logger.error("Failed to build semantic router: %s", e) self.tool_router = None if _is_context_window_error(e): self.context_window_error = str(e) @@ -254,7 +254,7 @@ class SemanticMCPToolFilter: self._tool_map.update(missing) verbose_logger.info( - f"Semantic tool filter indexed {len(routes)} request-time tools missing from the startup index" + "Semantic tool filter indexed %s request-time tools missing from the startup index", len(routes) ) async def filter_tools( @@ -321,7 +321,8 @@ class SemanticMCPToolFilter: except Exception as e: if _is_context_window_error(e): verbose_logger.error( - f"Semantic tool filter embedding exceeded its context window: {e}", + "Semantic tool filter embedding exceeded its context window: %s", + e, exc_info=True, ) raise SemanticToolFilterContextWindowError( @@ -329,7 +330,7 @@ class SemanticMCPToolFilter: stage="the user query or the MCP tool descriptions being indexed", original_error=str(e), ) from e - verbose_logger.error(f"Semantic tool filter failed: {e}", exc_info=True) + verbose_logger.error("Semantic tool filter failed: %s", e, exc_info=True) return available_tools def _extract_tool_names_from_matches(self, matches) -> list[str]: diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index a894413019e..6e7af04e747 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -153,7 +153,7 @@ try: "active_mcp_session", default=None ) except ImportError as e: - verbose_logger.debug(f"MCP module not found: {e}") + verbose_logger.debug("MCP module not found: %s", e) MCP_AVAILABLE = False # When MCP is not available, we set these to None at module level # All code using these types is inside `if MCP_AVAILABLE:` blocks @@ -657,7 +657,7 @@ if MCP_AVAILABLE: try: await _purge_expired_stateful_session_auth_contexts() except Exception as e: - verbose_logger.exception(f"Error cleaning up expired MCP stateful sessions: {e}") + verbose_logger.exception("Error cleaning up expired MCP stateful sessions: %s", e) async def initialize_session_managers(): """Initialize the session managers. Can be called from main app lifespan.""" @@ -713,7 +713,7 @@ if MCP_AVAILABLE: if _sse_session_manager_cm: await _sse_session_manager_cm.__aexit__(None, None, None) except Exception as e: - verbose_logger.exception(f"Error during session manager shutdown: {e}") + verbose_logger.exception("Error during session manager shutdown: %s", e) _session_manager_cm = None _session_manager_stateful_cm = None @@ -765,10 +765,11 @@ if MCP_AVAILABLE: raw_headers, _client_ip, ) = await get_or_extract_auth_context() - verbose_logger.debug(f"MCP list_tools - User API Key Auth from context: {user_api_key_auth}") - verbose_logger.debug(f"MCP list_tools - MCP servers from context: {mcp_servers}") + verbose_logger.debug("MCP list_tools - User API Key Auth from context: %s", user_api_key_auth) + verbose_logger.debug("MCP list_tools - MCP servers from context: %s", mcp_servers) verbose_logger.debug( - f"MCP list_tools - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" + "MCP list_tools - MCP server auth headers: %s", + list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None, ) if getattr( getattr(user_api_key_auth, "object_permission", None), @@ -795,7 +796,7 @@ if MCP_AVAILABLE: log_list_tools_to_spendlogs=True, list_tools_log_source="mcp_protocol", ) - verbose_logger.info(f"MCP list_tools - Successfully returned {len(listing.tools)} tools") + verbose_logger.info("MCP list_tools - Successfully returned %s tools", len(listing.tools)) if not listing.outcomes: return listing.tools outcome_meta = { @@ -805,7 +806,7 @@ if MCP_AVAILABLE: } return ListToolsResult.model_validate({"tools": listing.tools, "_meta": outcome_meta}) except Exception as e: - verbose_logger.exception(f"Error in list_tools endpoint: {e}") + verbose_logger.exception("Error in list_tools endpoint: %s", e) # Return empty list instead of failing completely # This prevents the HTTP stream from failing and allows the client to get a response return [] @@ -823,7 +824,7 @@ if MCP_AVAILABLE: try: host_ctx = host_server.request_context except Exception as e: - verbose_logger.warning(f"Could not capture host progress context: {e}") + verbose_logger.warning("Could not capture host progress context: %s", e) return None if not (host_ctx and hasattr(host_ctx, "meta") and host_ctx.meta): @@ -841,11 +842,11 @@ if MCP_AVAILABLE: progress=progress, total=total, ) - verbose_logger.debug(f"Forwarded progress {progress}/{total} to Host") + verbose_logger.debug("Forwarded progress %s/%s to Host", progress, total) except Exception as e: - verbose_logger.error(f"Failed to forward progress to Host: {e}") + verbose_logger.error("Failed to forward progress to Host: %s", e) - verbose_logger.debug(f"Host progressToken captured: {str(host_token)[:8]}...") + verbose_logger.debug("Host progressToken captured: %s...", str(host_token)[:8]) return forward_progress async def _build_virtual_call_logging_obj( @@ -1000,10 +1001,12 @@ if MCP_AVAILABLE: _client_ip, ) = await get_or_extract_auth_context() verbose_logger.debug( - f"MCP mcp_server_tool_call - user_api_key_auth={user_api_key_auth}, user_role={getattr(user_api_key_auth, 'user_role', 'N/A')}" + "MCP mcp_server_tool_call - user_api_key_auth=%s, user_role=%s", + user_api_key_auth, + getattr(user_api_key_auth, "user_role", "N/A"), ) - verbose_logger.debug(f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}") + verbose_logger.debug("MCP mcp_server_tool_call - User API Key Auth from context: %s", user_api_key_auth) try: # Inside this try so virtual-tool errors convert to isError @@ -1080,7 +1083,7 @@ if MCP_AVAILABLE: isError=True, ) except BlockedPiiEntityError as e: - verbose_logger.error(f"BlockedPiiEntityError in MCP tool call: {e}") + verbose_logger.error("BlockedPiiEntityError in MCP tool call: %s", e) return CallToolResult( content=[ TextContent( @@ -1091,13 +1094,13 @@ if MCP_AVAILABLE: isError=True, ) except GuardrailRaisedException as e: - verbose_logger.error(f"GuardrailRaisedException in MCP tool call: {e}") + verbose_logger.error("GuardrailRaisedException in MCP tool call: %s", e) return CallToolResult( content=[TextContent(text=f"Error: Guardrail violation - {e}", type="text")], isError=True, ) except HTTPException as e: - verbose_logger.error(f"HTTPException in MCP tool call: {e}") + verbose_logger.error("HTTPException in MCP tool call: %s", e) return CallToolResult( content=[TextContent(text=f"Error: {e.detail}", type="text")], isError=True, @@ -1108,7 +1111,7 @@ if MCP_AVAILABLE: # call path and the connect-time preemptive check do. Return an explicit isError # naming the upstream status (at info level, not a traceback) so the client still # learns it must re-authenticate upstream and expected pass-through 401s don't spam. - verbose_logger.info(f"Upstream auth failure calling MCP tool: HTTP {e.status_code}") + verbose_logger.info("Upstream auth failure calling MCP tool: HTTP %s", e.status_code) return CallToolResult( content=[ TextContent( @@ -1119,7 +1122,7 @@ if MCP_AVAILABLE: isError=True, ) except Exception as e: - verbose_logger.exception(f"MCP mcp_server_tool_call - error: {e}") + verbose_logger.exception("MCP mcp_server_tool_call - error: %s", e) return CallToolResult( content=[TextContent(text=f"Error: {e}", type="text")], isError=True, @@ -1155,10 +1158,11 @@ if MCP_AVAILABLE: raw_headers, _client_ip, ) = await get_or_extract_auth_context() - verbose_logger.debug(f"MCP list_prompts - User API Key Auth from context: {user_api_key_auth}") - verbose_logger.debug(f"MCP list_prompts - MCP servers from context: {mcp_servers}") + verbose_logger.debug("MCP list_prompts - User API Key Auth from context: %s", user_api_key_auth) + verbose_logger.debug("MCP list_prompts - MCP servers from context: %s", mcp_servers) verbose_logger.debug( - f"MCP list_prompts - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" + "MCP list_prompts - MCP server auth headers: %s", + list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None, ) # Get mcp_servers from context variable verbose_logger.debug("MCP list_prompts - Calling _list_prompts") @@ -1170,10 +1174,10 @@ if MCP_AVAILABLE: oauth2_headers=oauth2_headers, raw_headers=raw_headers, ) - verbose_logger.info(f"MCP list_prompts - Successfully returned {len(prompts)} prompts") + verbose_logger.info("MCP list_prompts - Successfully returned %s prompts", len(prompts)) return prompts except Exception as e: - verbose_logger.exception(f"Error in list_prompts endpoint: {e}") + verbose_logger.exception("Error in list_prompts endpoint: %s", e) # Return empty list instead of failing completely # This prevents the HTTP stream from failing and allows the client to get a response return [] @@ -1213,7 +1217,7 @@ if MCP_AVAILABLE: _client_ip, ) = await get_or_extract_auth_context() - verbose_logger.debug(f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}") + verbose_logger.debug("MCP mcp_server_tool_call - User API Key Auth from context: %s", user_api_key_auth) return await mcp_get_prompt( name=name, arguments=arguments, @@ -1248,10 +1252,11 @@ if MCP_AVAILABLE: raw_headers, _client_ip, ) = await get_or_extract_auth_context() - verbose_logger.debug(f"MCP list_resources - User API Key Auth from context: {user_api_key_auth}") - verbose_logger.debug(f"MCP list_resources - MCP servers from context: {mcp_servers}") + verbose_logger.debug("MCP list_resources - User API Key Auth from context: %s", user_api_key_auth) + verbose_logger.debug("MCP list_resources - MCP servers from context: %s", mcp_servers) verbose_logger.debug( - f"MCP list_resources - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" + "MCP list_resources - MCP server auth headers: %s", + list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None, ) resources = await _list_mcp_resources( @@ -1262,10 +1267,10 @@ if MCP_AVAILABLE: oauth2_headers=oauth2_headers, raw_headers=raw_headers, ) - verbose_logger.info(f"MCP list_resources - Successfully returned {len(resources)} resources") + verbose_logger.info("MCP list_resources - Successfully returned %s resources", len(resources)) return resources except Exception as e: - verbose_logger.exception(f"Error in list_resources endpoint: {e}") + verbose_logger.exception("Error in list_resources endpoint: %s", e) return [] finally: if _session_reset_token is not None: @@ -1291,10 +1296,11 @@ if MCP_AVAILABLE: raw_headers, _client_ip, ) = await get_or_extract_auth_context() - verbose_logger.debug(f"MCP list_resource_templates - User API Key Auth from context: {user_api_key_auth}") - verbose_logger.debug(f"MCP list_resource_templates - MCP servers from context: {mcp_servers}") + verbose_logger.debug("MCP list_resource_templates - User API Key Auth from context: %s", user_api_key_auth) + verbose_logger.debug("MCP list_resource_templates - MCP servers from context: %s", mcp_servers) verbose_logger.debug( - f"MCP list_resource_templates - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" + "MCP list_resource_templates - MCP server auth headers: %s", + list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None, ) resource_templates = await _list_mcp_resource_templates( @@ -1306,11 +1312,11 @@ if MCP_AVAILABLE: raw_headers=raw_headers, ) verbose_logger.info( - f"MCP list_resource_templates - Successfully returned {len(resource_templates)} resource templates" + "MCP list_resource_templates - Successfully returned %s resource templates", len(resource_templates) ) return resource_templates except Exception as e: - verbose_logger.exception(f"Error in list_resource_templates endpoint: {e}") + verbose_logger.exception("Error in list_resource_templates endpoint: %s", e) return [] finally: if _session_reset_token is not None: @@ -1400,7 +1406,7 @@ if MCP_AVAILABLE: if server_id == server.server_id: filtered_server[server.server_id] = server except Exception as e: - verbose_logger.debug(f"Could not resolve '{server_or_group}' as access group: {e}") + verbose_logger.debug("Could not resolve '%s' as access group: %s", server_or_group, e) if filtered_server: return list(filtered_server.values()) @@ -1659,7 +1665,7 @@ if MCP_AVAILABLE: creds = await list_user_oauth_credentials(prisma_client, user_id) return {c["server_id"]: c for c in creds if "server_id" in c} except Exception as e: - verbose_logger.warning(f"_prefetch_oauth_creds_for_user: failed to prefetch for user={user_id}: {e}") + verbose_logger.warning("_prefetch_oauth_creds_for_user: failed to prefetch for user=%s: %s", user_id, e) return {} def _prepare_mcp_server_headers( @@ -2023,7 +2029,10 @@ if MCP_AVAILABLE: filtered_tools = apply_tool_overrides(filtered_tools, server) verbose_logger.debug( - f"Successfully fetched {len(tools)} tools from server {server.name}, {len(filtered_tools)} after filtering" + "Successfully fetched %s tools from server %s, %s after filtering", + len(tools), + server.name, + len(filtered_tools), ) return filtered_tools, ServerListOk(tool_count=len(filtered_tools)) except MCPUpstreamAuthError as e: @@ -2033,10 +2042,10 @@ if MCP_AVAILABLE: # 401 + WWW-Authenticate (the MCP session manager serializes it as a JSON-RPC # error). Single-server routes surface it via the request-scope preemptive # check in _raise_preemptive_401_for_unauthenticated_servers instead. - verbose_logger.debug(f"MCP list_tools: omitting {server.name}; it needs upstream auth") + verbose_logger.debug("MCP list_tools: omitting %s; it needs upstream auth", server.name) return [], classify_list_exception(e) except Exception as e: - verbose_logger.exception(f"Error getting tools from server {server.name}: {e}") + verbose_logger.exception("Error getting tools from server %s: %s", server.name, e) return [], classify_list_exception(e) # Fetch tools from all servers in parallel @@ -2089,7 +2098,7 @@ if MCP_AVAILABLE: log_exc, ) - verbose_logger.info(f"Successfully fetched {len(all_tools)} tools total from all MCP servers") + verbose_logger.info("Successfully fetched %s tools total from all MCP servers", len(all_tools)) return AggregateToolListing(tools=all_tools, outcomes=server_outcomes) except Exception as e: @@ -2167,12 +2176,12 @@ if MCP_AVAILABLE: all_prompts.extend(prompts) - verbose_logger.debug(f"Successfully fetched {len(prompts)} prompts from server {server.name}") + verbose_logger.debug("Successfully fetched %s prompts from server %s", len(prompts), server.name) except Exception as e: - verbose_logger.exception(f"Error getting prompts from server {server.name}: {e}") + verbose_logger.exception("Error getting prompts from server %s: %s", server.name, e) # Continue with other servers instead of failing completely - verbose_logger.info(f"Successfully fetched {len(all_prompts)} prompts total from all MCP servers") + verbose_logger.info("Successfully fetched %s prompts total from all MCP servers", len(all_prompts)) return all_prompts @@ -2219,11 +2228,11 @@ if MCP_AVAILABLE: ) all_resources.extend(resources) - verbose_logger.debug(f"Successfully fetched {len(resources)} resources from server {server.name}") + verbose_logger.debug("Successfully fetched %s resources from server %s", len(resources), server.name) except Exception as e: - verbose_logger.exception(f"Error getting resources from server {server.name}: {e}") + verbose_logger.exception("Error getting resources from server %s: %s", server.name, e) - verbose_logger.info(f"Successfully fetched {len(all_resources)} resources total from all MCP servers") + verbose_logger.info("Successfully fetched %s resources total from all MCP servers", len(all_resources)) return all_resources @@ -2356,10 +2365,10 @@ if MCP_AVAILABLE: list_tools_log_source=list_tools_log_source, client_ip=client_ip, ) - verbose_logger.debug(f"Successfully fetched {len(listing.tools)} tools from managed MCP servers") + verbose_logger.debug("Successfully fetched %s tools from managed MCP servers", len(listing.tools)) return listing except Exception as e: - verbose_logger.exception(f"Error getting tools from managed MCP servers: {e}") + verbose_logger.exception("Error getting tools from managed MCP servers: %s", e) # Continue with an empty listing instead of failing completely return AggregateToolListing(tools=[], outcomes={}) @@ -2396,9 +2405,9 @@ if MCP_AVAILABLE: oauth2_headers=oauth2_headers, raw_headers=raw_headers, ) - verbose_logger.debug(f"Successfully fetched {len(managed_prompts)} prompts from managed MCP servers") + verbose_logger.debug("Successfully fetched %s prompts from managed MCP servers", len(managed_prompts)) except Exception as e: - verbose_logger.exception(f"Error getting tools from managed MCP servers: {e}") + verbose_logger.exception("Error getting tools from managed MCP servers: %s", e) # Continue with empty managed tools list instead of failing completely return managed_prompts @@ -2426,9 +2435,9 @@ if MCP_AVAILABLE: oauth2_headers=oauth2_headers, raw_headers=raw_headers, ) - verbose_logger.debug(f"Successfully fetched {len(managed_resources)} resources from managed MCP servers") + verbose_logger.debug("Successfully fetched %s resources from managed MCP servers", len(managed_resources)) except Exception as e: - verbose_logger.exception(f"Error getting resources from managed MCP servers: {e}") + verbose_logger.exception("Error getting resources from managed MCP servers: %s", e) return managed_resources @@ -2814,7 +2823,7 @@ if MCP_AVAILABLE: if isinstance(hook_result, dict) and "arguments" in hook_result: arguments = hook_result["arguments"] - verbose_logger.debug(f"Executing local registry tool: {name}") + verbose_logger.debug("Executing local registry tool: %s", name) # For BYOK servers the credential must be injected via a ContextVar # because the tool function has headers baked into its closure. # Pre-format the full Authorization header value using the server's @@ -3335,7 +3344,7 @@ if MCP_AVAILABLE: result = tool.handler(**arguments) return [TextContent(text=str(result), type="text")] except Exception as e: - verbose_logger.exception(f"Error executing local tool {name}: {e}") + verbose_logger.exception("Error executing local tool %s: %s", name, e) return [TextContent(text=f"Error: {e}", type="text")] def _get_mcp_servers_in_path(path: str) -> list[str] | None: @@ -3986,7 +3995,7 @@ if MCP_AVAILABLE: # to the appropriate response. return exc.response.status_code, exc.response.headers.get("www-authenticate") except Exception as exc: - verbose_logger.debug(f"_probe_upstream_auth: probe to {url} failed ({exc}), allowing request through") + verbose_logger.debug("_probe_upstream_auth: probe to %s failed (%s), allowing request through", url, exc) return 200, None async def _check_passthrough_upstream_auth( @@ -4124,9 +4133,9 @@ if MCP_AVAILABLE: # Extract client IP for MCP access control _client_ip = IPAddressUtils.get_mcp_client_ip(StarletteRequest(scope)) - verbose_logger.debug(f"MCP request mcp_servers (header/path): {mcp_servers}") + verbose_logger.debug("MCP request mcp_servers (header/path): %s", mcp_servers) verbose_logger.debug( - f"MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" + "MCP server auth headers: %s", list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None ) # Strip any client-supplied x-mcp-toolset-id to prevent forgery. @@ -4413,7 +4422,7 @@ if MCP_AVAILABLE: # 500 that surfaces as a cancelled tool call. raise _proxy_exception_to_http_exception(e) except Exception as e: - verbose_logger.exception(f"Error handling MCP request: {e}") + verbose_logger.exception("Error handling MCP request: %s", e) # Try to send a graceful error response for non-HTTP exceptions try: from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR @@ -4424,7 +4433,7 @@ if MCP_AVAILABLE: ) await error_response(scope, receive, send) except Exception as response_error: - verbose_logger.exception(f"Failed to send error response: {response_error}") + verbose_logger.exception("Failed to send error response: %s", response_error) # If we can't send a proper response, re-raise the original error raise e @@ -4445,9 +4454,9 @@ if MCP_AVAILABLE: # Extract client IP for MCP access control _sse_client_ip = IPAddressUtils.get_mcp_client_ip(StarletteRequest(scope)) - verbose_logger.debug(f"MCP request mcp_servers (header/path): {mcp_servers}") + verbose_logger.debug("MCP request mcp_servers (header/path): %s", mcp_servers) verbose_logger.debug( - f"MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" + "MCP server auth headers: %s", list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None ) # Strip any client-supplied x-mcp-toolset-id to prevent forgery. @@ -4524,7 +4533,7 @@ if MCP_AVAILABLE: # 500 that surfaces as a cancelled tool call. raise _proxy_exception_to_http_exception(e) except Exception as e: - verbose_logger.exception(f"Error handling MCP request: {e}") + verbose_logger.exception("Error handling MCP request: %s", e) # Try to send a graceful error response for non-HTTP exceptions try: # Send a proper HTTP error response instead of letting the exception bubble up @@ -4537,7 +4546,7 @@ if MCP_AVAILABLE: ) await error_response(scope, receive, send) except Exception as response_error: - verbose_logger.exception(f"Failed to send error response: {response_error}") + verbose_logger.exception("Failed to send error response: %s", response_error) # If we can't send a proper response, re-raise the original error raise e diff --git a/litellm/proxy/_experimental/mcp_server/sse_transport.py b/litellm/proxy/_experimental/mcp_server/sse_transport.py index 09863a7d391..fd90cb59d97 100644 --- a/litellm/proxy/_experimental/mcp_server/sse_transport.py +++ b/litellm/proxy/_experimental/mcp_server/sse_transport.py @@ -46,7 +46,7 @@ class SseServerTransport: super().__init__() self._endpoint = endpoint self._read_stream_writers = {} - verbose_logger.debug(f"SseServerTransport initialized with endpoint: {endpoint}") + verbose_logger.debug("SseServerTransport initialized with endpoint: %s", endpoint) @asynccontextmanager async def connect_sse(self, request: Request): @@ -67,7 +67,7 @@ class SseServerTransport: session_id = uuid4() session_uri = f"{quote(self._endpoint)}?session_id={session_id.hex}" self._read_stream_writers[session_id] = read_stream_writer - verbose_logger.debug(f"Created new session with ID: {session_id}") + verbose_logger.debug("Created new session with ID: %s", session_id) sse_stream_writer: MemoryObjectSendStream[dict[str, Any]] sse_stream_reader: MemoryObjectReceiveStream[dict[str, Any]] @@ -77,10 +77,10 @@ class SseServerTransport: verbose_logger.debug("Starting SSE writer") async with sse_stream_writer, write_stream_reader: await sse_stream_writer.send({"event": "endpoint", "data": session_uri}) - verbose_logger.debug(f"Sent endpoint event: {session_uri}") + verbose_logger.debug("Sent endpoint event: %s", session_uri) async for message in write_stream_reader: - verbose_logger.debug(f"Sending message via SSE: {message}") + verbose_logger.debug("Sending message via SSE: %s", message) await sse_stream_writer.send( { "event": "message", @@ -108,31 +108,31 @@ class SseServerTransport: try: session_id = UUID(hex=session_id_param) - verbose_logger.debug(f"Parsed session ID: {session_id}") + verbose_logger.debug("Parsed session ID: %s", session_id) except ValueError: - verbose_logger.warning(f"Received invalid session ID: {session_id_param}") + verbose_logger.warning("Received invalid session ID: %s", session_id_param) response = Response("Invalid session ID", status_code=400) return response writer = self._read_stream_writers.get(session_id) if not writer: - verbose_logger.warning(f"Could not find session for ID: {session_id}") + verbose_logger.warning("Could not find session for ID: %s", session_id) response = Response("Could not find session", status_code=404) return response json = await request.json() - verbose_logger.debug(f"Received JSON: {json}") + verbose_logger.debug("Received JSON: %s", json) try: message = types.JSONRPCMessage.model_validate(json) - verbose_logger.debug(f"Validated client message: {message}") + verbose_logger.debug("Validated client message: %s", message) except ValidationError as err: - verbose_logger.error(f"Failed to parse message: {err}") + verbose_logger.error("Failed to parse message: %s", err) response = Response("Could not parse message", status_code=400) await writer.send(err) return response - verbose_logger.debug(f"Sending message to writer: {message}") + verbose_logger.debug("Sending message to writer: %s", message) response = Response("Accepted", status_code=202) await writer.send(message) return response diff --git a/litellm/proxy/_experimental/mcp_server/tool_registry.py b/litellm/proxy/_experimental/mcp_server/tool_registry.py index 1ebeac9993a..f2ef94f412b 100644 --- a/litellm/proxy/_experimental/mcp_server/tool_registry.py +++ b/litellm/proxy/_experimental/mcp_server/tool_registry.py @@ -40,7 +40,7 @@ class MCPToolRegistry: input_schema=input_schema, handler=handler, ) - verbose_logger.debug(f"Registered tool: {name}") + verbose_logger.debug("Registered tool: %s", name) def get_tool(self, name: str) -> MCPTool | None: """ @@ -122,7 +122,7 @@ class MCPToolRegistry: handler = get_instance_fn(handler_name, config_file_path) if handler is None: - verbose_logger.warning(f"Warning: Could not find handler {handler_name} for tool {name}") + verbose_logger.warning("Warning: Could not find handler %s for tool %s", handler_name, name) continue # Register the tool diff --git a/litellm/proxy/_experimental/mcp_server/toolset_db.py b/litellm/proxy/_experimental/mcp_server/toolset_db.py index a321c40b9e2..26c830d5d50 100644 --- a/litellm/proxy/_experimental/mcp_server/toolset_db.py +++ b/litellm/proxy/_experimental/mcp_server/toolset_db.py @@ -55,7 +55,7 @@ async def list_mcp_toolsets( rows = await MCPToolsetRepository(prisma_client).table.find_many(where=where) return [_toolset_from_row(r) for r in rows] except Exception as e: - verbose_proxy_logger.warning(f"litellm.proxy._experimental.mcp_server.toolset_db::list_mcp_toolsets - {e}") + verbose_proxy_logger.warning("litellm.proxy._experimental.mcp_server.toolset_db::list_mcp_toolsets - %s", e) return [] diff --git a/litellm/proxy/_experimental/mcp_server/ui_session_utils.py b/litellm/proxy/_experimental/mcp_server/ui_session_utils.py index 14726cba3a7..0e750453be6 100644 --- a/litellm/proxy/_experimental/mcp_server/ui_session_utils.py +++ b/litellm/proxy/_experimental/mcp_server/ui_session_utils.py @@ -91,7 +91,7 @@ async def admitted_user_context(user_api_key_auth: UserAPIKeyAuth) -> UserAPIKey try: admitted = await MCPRequestHandler._reload_admitted_user(user_id) except HTTPException as e: - verbose_logger.warning(f"MCP dashboard session: admitted-subject reload failed for {user_id}: {e.detail}") + verbose_logger.warning("MCP dashboard session: admitted-subject reload failed for %s: %s", user_id, e.detail) return None return admitted.model_copy(update={"parent_otel_span": user_api_key_auth.parent_otel_span}) diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py index 6d48b31658b..c10b3d95cd3 100644 --- a/litellm/proxy/agent_endpoints/a2a_endpoints.py +++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py @@ -468,7 +468,7 @@ async def _handle_stream_message( obj = normalize_stream_event(obj, served_version, request_id=request_id) yield json.dumps(obj) + "\n" except Exception as e: - verbose_proxy_logger.exception(f"Error streaming A2A response: {e}") + verbose_proxy_logger.exception("Error streaming A2A response: %s", e) if ( use_proxy_hooks and proxy_logging_obj is not None @@ -561,13 +561,13 @@ async def get_agent_card( served_version = _served_version(agent, request) agent_card = normalize_agent_card(agent_card, served_version) - verbose_proxy_logger.debug(f"Returning agent card for '{agent_id}' with proxy URL: {proxy_url}") + verbose_proxy_logger.debug("Returning agent card for '%s' with proxy URL: %s", agent_id, proxy_url) return JSONResponse(content=agent_card) except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error getting agent card: {e}") + verbose_proxy_logger.exception("Error getting agent card: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -615,7 +615,7 @@ async def invoke_agent_a2a( body = await request.json() request_data = body - verbose_proxy_logger.debug(f"A2A request for agent '{agent_id}': {body}") + verbose_proxy_logger.debug("A2A request for agent '%s': %s", agent_id, body) # Validate JSON-RPC format if body.get("jsonrpc") != "2.0": @@ -690,7 +690,9 @@ async def invoke_agent_a2a( if not agent_url and not custom_llm_provider: return _jsonrpc_error(request_id, -32000, f"Agent '{agent_id}' has no URL configured", 500) - verbose_proxy_logger.info(f"Proxying A2A request to agent '{agent_id}' at {agent_url or 'completion-bridge'}") + verbose_proxy_logger.info( + "Proxying A2A request to agent '%s' at %s", agent_id, agent_url or "completion-bridge" + ) # Set up data dict for litellm processing if "metadata" not in body: @@ -965,7 +967,7 @@ async def invoke_agent_a2a( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error invoking agent: {e}") + verbose_proxy_logger.exception("Error invoking agent: %s", e) try: await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, diff --git a/litellm/proxy/agent_endpoints/a2a_routing.py b/litellm/proxy/agent_endpoints/a2a_routing.py index 0410f067560..6446a9ad221 100644 --- a/litellm/proxy/agent_endpoints/a2a_routing.py +++ b/litellm/proxy/agent_endpoints/a2a_routing.py @@ -46,7 +46,7 @@ async def route_a2a_agent_request( # Look up agent in registry agent = global_agent_registry.get_agent_by_name(agent_name) if agent is None: - verbose_proxy_logger.error(f"[A2A] Agent '{agent_name}' not found in registry") + verbose_proxy_logger.error("[A2A] Agent '%s' not found in registry", agent_name) route_name = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type) raise ProxyModelNotFoundError(route=route_name, model_name=model_name) @@ -68,12 +68,12 @@ async def route_a2a_agent_request( # Get API base URL from agent config if not agent.agent_card_params or "url" not in agent.agent_card_params: - verbose_proxy_logger.error(f"[A2A] Agent '{agent_name}' has no URL configured") + verbose_proxy_logger.error("[A2A] Agent '%s' has no URL configured", agent_name) route_name = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type) raise ProxyModelNotFoundError(route=route_name, model_name=model_name) # Inject API base and route to litellm data["api_base"] = agent.agent_card_params["url"] - verbose_proxy_logger.debug(f"[A2A] Routing {model_name} to {data['api_base']}") + verbose_proxy_logger.debug("[A2A] Routing %s to %s", model_name, data["api_base"]) return getattr(litellm, f"{route_type}")(**data) diff --git a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py index 6999228c83d..e0b8b24de97 100644 --- a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py +++ b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py @@ -59,7 +59,7 @@ class AgentRequestHandler: return list(set(allowed_agents)) except Exception as e: - verbose_logger.warning(f"Failed to get allowed agents: {e}") + verbose_logger.warning("Failed to get allowed agents: %s", e) return [] @staticmethod @@ -179,7 +179,7 @@ class AgentRequestHandler: return list(set(all_agents)) except Exception as e: - verbose_logger.warning(f"Failed to get allowed agents for key: {e}") + verbose_logger.warning("Failed to get allowed agents for key: %s", e) return [] @staticmethod @@ -255,7 +255,7 @@ class AgentRequestHandler: # litellm-dashboard is the default UI team and will never have agents; # skip noisy warnings for it. if user_api_key_auth.team_id != UI_TEAM_ID: - verbose_logger.warning(f"Failed to get allowed agents for team: {e}") + verbose_logger.warning("Failed to get allowed agents for team: %s", e) return [] @staticmethod @@ -285,7 +285,7 @@ class AgentRequestHandler: for agent in agents: agent_ids.add(agent.agent_id) except Exception as e: - verbose_logger.debug(f"Error getting agents from access groups: {e}") + verbose_logger.debug("Error getting agents from access groups: %s", e) return agent_ids @staticmethod @@ -310,7 +310,7 @@ class AgentRequestHandler: return list(agent_ids) except Exception as e: - verbose_logger.warning(f"Failed to get agents from access groups: {e}") + verbose_logger.warning("Failed to get agents from access groups: %s", e) return [] @staticmethod @@ -369,7 +369,7 @@ class AgentRequestHandler: return key_object_permission.agent_access_groups or [] except Exception as e: - verbose_logger.warning(f"Failed to get agent access groups for key: {e}") + verbose_logger.warning("Failed to get agent access groups for key: %s", e) return [] @staticmethod @@ -412,5 +412,5 @@ class AgentRequestHandler: return object_permissions.agent_access_groups or [] except Exception as e: - verbose_logger.warning(f"Failed to get agent access groups for team: {e}") + verbose_logger.warning("Failed to get agent access groups for team: %s", e) return [] diff --git a/litellm/proxy/agent_endpoints/endpoints.py b/litellm/proxy/agent_endpoints/endpoints.py index db5341dbe5a..b4c0675c5ef 100644 --- a/litellm/proxy/agent_endpoints/endpoints.py +++ b/litellm/proxy/agent_endpoints/endpoints.py @@ -316,7 +316,7 @@ async def get_agents( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.agent_endpoints.get_agents(): Exception occurred - {e}") + verbose_proxy_logger.exception("litellm.proxy.agent_endpoints.get_agents(): Exception occurred - %s", e) raise HTTPException(status_code=500, detail={"error": f"Internal server error: {e}"}) @@ -434,10 +434,10 @@ async def create_agent( # Also register in memory try: AGENT_REGISTRY.register_agent(agent_config=result) - verbose_proxy_logger.info(f"Successfully registered agent '{agent_name}' (ID: {agent_id}) in memory") + verbose_proxy_logger.info("Successfully registered agent '%s' (ID: %s) in memory", agent_name, agent_id) except Exception as reg_error: verbose_proxy_logger.warning( - f"Failed to register agent '{agent_name}' (ID: {agent_id}) in memory: {reg_error}" + "Failed to register agent '%s' (ID: %s) in memory: %s", agent_name, agent_id, reg_error ) return result @@ -445,7 +445,7 @@ async def create_agent( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error adding agent to db: {e}") + verbose_proxy_logger.exception("Error adding agent to db: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -529,7 +529,7 @@ async def get_agent_by_id( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error getting agent from db: {e}") + verbose_proxy_logger.exception("Error getting agent from db: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -624,14 +624,14 @@ async def update_agent( AGENT_REGISTRY.register_agent(agent_config=result) verbose_proxy_logger.info( - f"Successfully updated agent '{existing_agent.get('agent_name')}' (ID: {agent_id}) in memory" + "Successfully updated agent '%s' (ID: %s) in memory", existing_agent.get("agent_name"), agent_id ) return result except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error updating agent: {e}") + verbose_proxy_logger.exception("Error updating agent: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -727,14 +727,14 @@ async def patch_agent( AGENT_REGISTRY.register_agent(agent_config=result) verbose_proxy_logger.info( - f"Successfully updated agent '{existing_agent.get('agent_name')}' (ID: {agent_id}) in memory" + "Successfully updated agent '%s' (ID: %s) in memory", existing_agent.get("agent_name"), agent_id ) return result except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error updating agent: {e}") + verbose_proxy_logger.exception("Error updating agent: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -789,7 +789,7 @@ async def delete_agent( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error deleting agent: {e}") + verbose_proxy_logger.exception("Error deleting agent: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -884,7 +884,7 @@ async def make_agent_public( await proxy_config.save_config(new_config=config) verbose_proxy_logger.debug( - f"Updated public agent groups to: {litellm.public_agent_groups} by user: {user_api_key_dict.user_id}" + "Updated public agent groups to: %s by user: %s", litellm.public_agent_groups, user_api_key_dict.user_id ) return { @@ -895,7 +895,7 @@ async def make_agent_public( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error making agent public: {e}") + verbose_proxy_logger.exception("Error making agent public: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -988,7 +988,7 @@ async def make_agents_public( await proxy_config.save_config(new_config=config) verbose_proxy_logger.debug( - f"Updated public agent groups to: {litellm.public_agent_groups} by user: {user_api_key_dict.user_id}" + "Updated public agent groups to: %s by user: %s", litellm.public_agent_groups, user_api_key_dict.user_id ) return { @@ -999,7 +999,7 @@ async def make_agents_public( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error making agent public: {e}") + verbose_proxy_logger.exception("Error making agent public: %s", e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/litellm/proxy/agent_endpoints/model_list_helpers.py b/litellm/proxy/agent_endpoints/model_list_helpers.py index 56053f59c85..fc70ca75ae6 100644 --- a/litellm/proxy/agent_endpoints/model_list_helpers.py +++ b/litellm/proxy/agent_endpoints/model_list_helpers.py @@ -40,7 +40,7 @@ async def append_agents_to_model_group( ) ) except Exception as e: - verbose_proxy_logger.debug(f"Error appending agents to model_group/info: {e}") + verbose_proxy_logger.debug("Error appending agents to model_group/info: %s", e) return model_groups @@ -84,6 +84,6 @@ async def append_agents_to_model_info( } ) except Exception as e: - verbose_proxy_logger.debug(f"Error appending agents to v2/model/info: {e}") + verbose_proxy_logger.debug("Error appending agents to v2/model/info: %s", e) return models diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py index bf797b92850..a6c45cf736d 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py @@ -79,12 +79,12 @@ async def get_marketplace(): try: manifest = json.loads(plugin.manifest_json) except json.JSONDecodeError: - verbose_proxy_logger.warning(f"Plugin {plugin.name} has invalid manifest JSON, skipping") + verbose_proxy_logger.warning("Plugin %s has invalid manifest JSON, skipping", plugin.name) continue # Source must be specified for URL-based marketplaces if "source" not in manifest: - verbose_proxy_logger.warning(f"Plugin {plugin.name} has no source field, skipping") + verbose_proxy_logger.warning("Plugin %s has no source field, skipping", plugin.name) continue entry: dict[str, Any] = { @@ -118,7 +118,7 @@ async def get_marketplace(): except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error generating marketplace: {e}") + verbose_proxy_logger.exception("Error generating marketplace: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to generate marketplace: {e}"}, @@ -283,7 +283,7 @@ async def register_plugin( ) action = "created" - verbose_proxy_logger.info(f"Plugin {request.name} {action} successfully") + verbose_proxy_logger.info("Plugin %s %s successfully", request.name, action) return { "status": "success", @@ -301,7 +301,7 @@ async def register_plugin( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error registering plugin: {e}") + verbose_proxy_logger.exception("Error registering plugin: %s", e) raise HTTPException( status_code=500, detail={"error": f"Registration failed: {e}"}, @@ -368,7 +368,7 @@ async def list_plugins( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error listing plugins: {e}") + verbose_proxy_logger.exception("Error listing plugins: %s", e) raise HTTPException( status_code=500, detail={"error": str(e)}, @@ -425,7 +425,7 @@ async def get_plugin( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error getting plugin: {e}") + verbose_proxy_logger.exception("Error getting plugin: %s", e) raise HTTPException( status_code=500, detail={"error": str(e)}, @@ -462,13 +462,13 @@ async def enable_plugin( data={"enabled": True, "updated_at": datetime.now(timezone.utc)}, ) - verbose_proxy_logger.info(f"Plugin {plugin_name} enabled") + verbose_proxy_logger.info("Plugin %s enabled", plugin_name) return {"status": "success", "message": f"Plugin '{plugin_name}' enabled"} except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error enabling plugin: {e}") + verbose_proxy_logger.exception("Error enabling plugin: %s", e) raise HTTPException( status_code=500, detail={"error": str(e)}, @@ -505,13 +505,13 @@ async def disable_plugin( data={"enabled": False, "updated_at": datetime.now(timezone.utc)}, ) - verbose_proxy_logger.info(f"Plugin {plugin_name} disabled") + verbose_proxy_logger.info("Plugin %s disabled", plugin_name) return {"status": "success", "message": f"Plugin '{plugin_name}' disabled"} except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error disabling plugin: {e}") + verbose_proxy_logger.exception("Error disabling plugin: %s", e) raise HTTPException( status_code=500, detail={"error": str(e)}, @@ -545,13 +545,13 @@ async def delete_plugin( await ClaudeCodePluginRepository(prisma_client).table.delete(where={"name": plugin_name}) - verbose_proxy_logger.info(f"Plugin {plugin_name} deleted") + verbose_proxy_logger.info("Plugin %s deleted", plugin_name) return {"status": "success", "message": f"Plugin '{plugin_name}' deleted"} except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error deleting plugin: {e}") + verbose_proxy_logger.exception("Error deleting plugin: %s", e) raise HTTPException( status_code=500, detail={"error": str(e)}, diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 5535928dfac..397ee64d399 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -189,7 +189,7 @@ async def anthropic_response( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.anthropic_response(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.anthropic_response(): Exception occured - %s", e) # Extract model_id from request metadata (same as success path) litellm_metadata = data.get("litellm_metadata", {}) or {} @@ -301,7 +301,7 @@ async def count_tokens( detail=detail, ) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.anthropic_endpoints.count_tokens(): Exception occurred - {e}") + verbose_proxy_logger.exception("litellm.proxy.anthropic_endpoints.count_tokens(): Exception occurred - %s", e) raise HTTPException(status_code=500, detail={"error": f"Internal server error: {e}"}) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 6b1d845ec95..c186bbc15ee 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -136,8 +136,10 @@ def _log_budget_lookup_failure(entity: str, error: Exception) -> None: if any(x in err_str for x in ("column", "schema", "does not exist", "prisma", "migrate")): hint = " Run `prisma db push` or `prisma migrate deploy` to fix schema mismatches." verbose_proxy_logger.error( - f"Budget lookup failed for {entity}; cache will not be populated. " - f"Each request will hit the database. Error: {error}.{hint}" + "Budget lookup failed for %s; cache will not be populated. Each request will hit the database. Error: %s.%s", + entity, + error, + hint, ) @@ -192,7 +194,7 @@ def _is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None if model_group_info is None: # Model not found or no pricing info available # Conservative approach: assume it has cost - verbose_proxy_logger.debug(f"No model group info found for {model_name}, assuming it has cost") + verbose_proxy_logger.debug("No model group info found for %s, assuming it has cost", model_name) if zero_cost_cache is not None: zero_cost_cache[model_name] = False return False @@ -205,7 +207,10 @@ def _is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None # If costs are not explicitly configured (None), assume it has cost if input_cost is None or output_cost is None: verbose_proxy_logger.debug( - f"Model {model_name} has undefined cost (input: {input_cost}, output: {output_cost}), assuming it has cost" + "Model %s has undefined cost (input: %s, output: %s), assuming it has cost", + model_name, + input_cost, + output_cost, ) if zero_cost_cache is not None: zero_cost_cache[model_name] = False @@ -214,7 +219,7 @@ def _is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None # If either cost is non-zero, return False if input_cost > 0 or output_cost > 0: verbose_proxy_logger.debug( - f"Model {model_name} has non-zero cost (input: {input_cost}, output: {output_cost})" + "Model %s has non-zero cost (input: %s, output: %s)", model_name, input_cost, output_cost ) if zero_cost_cache is not None: zero_cost_cache[model_name] = False @@ -246,7 +251,7 @@ def _is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None except Exception as e: # If we can't determine the cost, assume it has cost (conservative approach) - verbose_proxy_logger.debug(f"Error checking cost for model {model_name}: {e}, assuming it has cost") + verbose_proxy_logger.debug("Error checking cost for model %s: %s, assuming it has cost", model_name, e) return False # All models checked have zero cost @@ -957,7 +962,7 @@ async def get_default_end_user_budget( if budget_record is None: verbose_proxy_logger.warning( - f"Default end user budget not found in database: {litellm.max_end_user_budget_id}" + "Default end user budget not found in database: %s", litellm.max_end_user_budget_id ) return None @@ -973,7 +978,7 @@ async def get_default_end_user_budget( return _budget_obj except Exception as e: - verbose_proxy_logger.error(f"Error fetching default end user budget: {e}") + verbose_proxy_logger.error("Error fetching default end user budget: %s", e) return None @@ -1013,7 +1018,7 @@ async def get_team_member_default_budget( budget_record = await BudgetRepository(prisma_client).table.find_unique(where={"budget_id": budget_id}) if budget_record is None: - verbose_proxy_logger.warning(f"Team-default member budget not found in database: {budget_id}") + verbose_proxy_logger.warning("Team-default member budget not found in database: %s", budget_id) return None await user_api_key_cache.async_set_cache( @@ -1025,7 +1030,7 @@ async def get_team_member_default_budget( return LiteLLM_BudgetTable.model_validate(budget_record.dict()) except Exception: - verbose_proxy_logger.exception(f"Error fetching team-default member budget {budget_id}") + verbose_proxy_logger.exception("Error fetching team-default member budget %s", budget_id) return None @@ -1066,7 +1071,7 @@ async def _apply_default_budget_to_end_user( # Apply default budget to end user object end_user_obj.litellm_budget_table = default_budget verbose_proxy_logger.debug( - f"Applied default budget {litellm.max_end_user_budget_id} to end user {end_user_obj.user_id}" + "Applied default budget %s to end user %s", litellm.max_end_user_budget_id, end_user_obj.user_id ) return end_user_obj @@ -1289,7 +1294,7 @@ async def _end_user_id_exists_in_db( if end_user_obj is not None: return True except Exception as e: - verbose_proxy_logger.debug(f"end_user validation: get_end_user_object lookup failed: {e}") + verbose_proxy_logger.debug("end_user validation: get_end_user_object lookup failed: %s", e) try: user_obj = await get_user_object( @@ -1305,7 +1310,7 @@ async def _end_user_id_exists_in_db( if user_obj is not None: return True except Exception as e: - verbose_proxy_logger.debug(f"end_user validation: get_user_object lookup failed: {e}") + verbose_proxy_logger.debug("end_user validation: get_user_object lookup failed: %s", e) return False @@ -1376,7 +1381,7 @@ async def get_tag_objects_batch( ) tag_objects[tag_name] = _tag_obj except Exception as e: - verbose_proxy_logger.debug(f"Error batch fetching tags from database: {e}") + verbose_proxy_logger.debug("Error batch fetching tags from database: %s", e) return tag_objects @@ -1974,7 +1979,10 @@ async def _get_team_object_from_user_api_key_cache( ) except Exception as e: verbose_proxy_logger.debug( - f"Failed to load object_permission for team {team_id} with object_permission_id={_response.object_permission_id}: {e}" + "Failed to load object_permission for team %s with object_permission_id=%s: %s", + team_id, + _response.object_permission_id, + e, ) # save the team object to cache @@ -2250,7 +2258,10 @@ async def get_team_object_by_alias( ) except Exception as e: verbose_proxy_logger.debug( - f"Failed to load object_permission for team {team_obj.team_id} with object_permission_id={team_obj.object_permission_id}: {e}" + "Failed to load object_permission for team %s with object_permission_id=%s: %s", + team_obj.team_id, + team_obj.object_permission_id, + e, ) # Cache the result by both alias and team_id @@ -2610,7 +2621,9 @@ async def get_key_object( ) except Exception as e: verbose_proxy_logger.debug( - f"Failed to load object_permission for key with object_permission_id={_response.object_permission_id}: {e}" + "Failed to load object_permission for key with object_permission_id=%s: %s", + _response.object_permission_id, + e, ) # save the key object to cache diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index 681647814e7..b79e01eda21 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -95,7 +95,9 @@ class UserAPIKeyAuthExceptionHandler: use_x_forwarded_for=general_settings.get("use_x_forwarded_for", False), ) verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - {e}\nRequester IP Address:{requester_ip}", + "litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - %s\nRequester IP Address:%s", + e, + requester_ip, extra={"requester_ip": requester_ip}, ) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index a03ed13180c..df4f4caa78f 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -529,10 +529,11 @@ async def pre_db_read_auth_checks( _allowed_routes = general_settings["allowed_routes"] if premium_user is not True: verbose_proxy_logger.error( - f"Trying to set allowed_routes. This is an Enterprise feature. {CommonProxyErrors.not_premium_user.value}" + "Trying to set allowed_routes. This is an Enterprise feature. %s", + CommonProxyErrors.not_premium_user.value, ) if route not in _allowed_routes: - verbose_proxy_logger.error(f"Route {route} not in allowed_routes={_allowed_routes}") + verbose_proxy_logger.error("Route %s not in allowed_routes=%s", route, _allowed_routes) raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=f"Access forbidden: Route {route} not allowed", @@ -582,7 +583,7 @@ def route_in_additonal_public_routes(current_route: str): return False except Exception as e: - verbose_proxy_logger.error(f"route_in_additonal_public_routes: {e}") + verbose_proxy_logger.error("route_in_additonal_public_routes: %s", e) return False @@ -619,7 +620,7 @@ def get_request_route(request: Request) -> str: return raw_path except Exception as e: verbose_proxy_logger.debug( - f"error on get_request_route: {e}, defaulting to request.url.path={request.url.path}" + "error on get_request_route: %s, defaulting to request.url.path=%s", e, request.url.path ) return str(request.url.path) @@ -639,7 +640,7 @@ def get_request_route_template(request: Request) -> str | None: template = getattr(route, "path", None) return template if isinstance(template, str) and template else None except Exception as e: - verbose_proxy_logger.debug(f"error on get_request_route_template: {e}") + verbose_proxy_logger.debug("error on get_request_route_template: %s", e) return None @@ -781,7 +782,8 @@ async def check_if_request_size_is_safe(request: Request) -> bool: # Check if premium user if premium_user is not True: verbose_proxy_logger.warning( - f"using max_request_size_mb - not checking - this is an enterprise only feature. {CommonProxyErrors.not_premium_user.value}" + "using max_request_size_mb - not checking - this is an enterprise only feature. %s", + CommonProxyErrors.not_premium_user.value, ) return True @@ -791,7 +793,7 @@ async def check_if_request_size_is_safe(request: Request) -> bool: if content_length: header_size = int(content_length) header_size_mb = bytes_to_mb(bytes_value=header_size) - verbose_proxy_logger.debug(f"content_length request size in MB={header_size_mb}") + verbose_proxy_logger.debug("content_length request size in MB=%s", header_size_mb) if header_size_mb > max_request_size_mb: raise ProxyException( @@ -806,7 +808,7 @@ async def check_if_request_size_is_safe(request: Request) -> bool: body_size = len(body) request_size_mb = bytes_to_mb(bytes_value=body_size) - verbose_proxy_logger.debug(f"request body request size in MB={request_size_mb}") + verbose_proxy_logger.debug("request body request size in MB=%s", request_size_mb) if request_size_mb > max_request_size_mb: raise ProxyException( message=f"Request size is too large. Request size is {request_size_mb} MB. Max size is {max_request_size_mb} MB", @@ -841,12 +843,13 @@ async def check_response_size_is_safe(response: Any) -> bool: # Check if premium user if premium_user is not True: verbose_proxy_logger.warning( - f"using max_response_size_mb - not checking - this is an enterprise only feature. {CommonProxyErrors.not_premium_user.value}" + "using max_response_size_mb - not checking - this is an enterprise only feature. %s", + CommonProxyErrors.not_premium_user.value, ) return True response_size_mb = bytes_to_mb(bytes_value=sys.getsizeof(response)) - verbose_proxy_logger.debug(f"response size in MB={response_size_mb}") + verbose_proxy_logger.debug("response size in MB=%s", response_size_mb) if response_size_mb > max_response_size_mb: raise ProxyException( message=f"Response size is too large. Response size is {response_size_mb} MB. Max size is {max_response_size_mb} MB", diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index cf4b47e3180..cb860f5df4e 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -379,8 +379,10 @@ class JWTHandler: if not team_id: return default_value verbose_proxy_logger.debug( - f"JWT Auth: team_id_jwt_field '{self.litellm_jwtauth.team_id_jwt_field}' " - f"returned a list {team_id}; using first element '{team_id[0]}' automatically." + "JWT Auth: team_id_jwt_field '%s' returned a list %s; using first element '%s' automatically.", + self.litellm_jwtauth.team_id_jwt_field, + team_id, + team_id[0], ) team_id = team_id[0] return team_id # type: ignore[return-value] @@ -614,7 +616,7 @@ class JWTHandler: if cached_jwks_uri is not None: return cached_jwks_uri - verbose_proxy_logger.debug(f"JWT Auth: Fetching OIDC discovery document from {url}") + verbose_proxy_logger.debug("JWT Auth: Fetching OIDC discovery document from %s", url) response = await self.http_handler.get(url) if response.status_code != 200: raise Exception( @@ -629,7 +631,7 @@ class JWTHandler: if not jwks_uri: raise Exception(f"JWT Auth: OIDC discovery document at {url} does not contain a 'jwks_uri' field.") - verbose_proxy_logger.debug(f"JWT Auth: Resolved OIDC discovery {url} -> jwks_uri={jwks_uri}") + verbose_proxy_logger.debug("JWT Auth: Resolved OIDC discovery %s -> jwks_uri=%s", url, jwks_uri) await self.user_api_key_cache.async_set_cache( key=cache_key, value=jwks_uri, @@ -655,7 +657,7 @@ class JWTHandler: try: response_json = response.json() except Exception as e: - verbose_proxy_logger.error(f"Error parsing response: {e}. Original Response: {response.text}") + verbose_proxy_logger.error("Error parsing response: %s. Original Response: %s", e, response.text) raise Exception(f"Error parsing response: {e}. Check server logs for original response.") if "keys" in response_json: @@ -749,7 +751,7 @@ class JWTHandler: verbose_proxy_logger.debug("Returning cached OIDC UserInfo") return cached_userinfo - verbose_proxy_logger.debug(f"Calling OIDC UserInfo endpoint: {self.litellm_jwtauth.oidc_userinfo_endpoint}") + verbose_proxy_logger.debug("Calling OIDC UserInfo endpoint: %s", self.litellm_jwtauth.oidc_userinfo_endpoint) try: # Call the UserInfo endpoint with the access token @@ -765,7 +767,7 @@ class JWTHandler: raise Exception(f"OIDC UserInfo endpoint returned status {response.status_code}: {response.text}") userinfo = response.json() - verbose_proxy_logger.debug(f"Received OIDC UserInfo: {userinfo}") + verbose_proxy_logger.debug("Received OIDC UserInfo: %s", userinfo) # Cache the userinfo response await self.user_api_key_cache.async_set_cache( @@ -777,7 +779,7 @@ class JWTHandler: return userinfo except Exception as e: - verbose_proxy_logger.error(f"Error fetching OIDC UserInfo: {e}") + verbose_proxy_logger.error("Error fetching OIDC UserInfo: %s", e) raise Exception(f"Failed to fetch OIDC UserInfo: {e}") _unscoped_jwt_warning_emitted = False @@ -1239,7 +1241,7 @@ class JWTAuthManager: return None, None if team_alias: - verbose_proxy_logger.info(f"JWT Auth: Resolving team by alias: '{team_alias}'") + verbose_proxy_logger.info("JWT Auth: Resolving team by alias: '%s'", team_alias) team_object = await get_team_object_by_alias( team_alias=team_alias, prisma_client=prisma_client, @@ -1250,7 +1252,7 @@ class JWTAuthManager: if team_object: individual_team_id = team_object.team_id verbose_proxy_logger.info( - f"JWT Auth: Resolved team_alias='{team_alias}' to team_id='{individual_team_id}'" + "JWT Auth: Resolved team_alias='%s' to team_id='%s'", team_alias, individual_team_id ) return individual_team_id, team_object @@ -1391,7 +1393,7 @@ class JWTAuthManager: is_allowed = False denied_auth_enforced_pass_through_route = True verbose_proxy_logger.debug( - f"JWT team route check: team_id={team_id}, route={route}, is_allowed={is_allowed}" + "JWT team route check: team_id=%s, route=%s, is_allowed=%s", team_id, route, is_allowed ) if is_allowed: return team_id, team_object @@ -1482,7 +1484,7 @@ class JWTAuthManager: else None ) elif org_alias: - verbose_proxy_logger.info(f"JWT Auth: Resolving org by alias: '{org_alias}'") + verbose_proxy_logger.info("JWT Auth: Resolving org by alias: '%s'", org_alias) org_object = await get_org_object_by_alias( org_alias=org_alias, prisma_client=prisma_client, @@ -1492,7 +1494,7 @@ class JWTAuthManager: ) if org_object: verbose_proxy_logger.info( - f"JWT Auth: Resolved org_alias='{org_alias}' to org_id='{org_object.organization_id}'" + "JWT Auth: Resolved org_alias='%s' to org_id='%s'", org_alias, org_object.organization_id ) # Check if email domain is allowed before attempting to get/create user @@ -1625,7 +1627,7 @@ class JWTAuthManager: detail=f"Team '{header_team_id}' from x-litellm-team-id header is not in your JWT's allowed teams. Allowed teams: {list(allowed_team_ids)}", ) - verbose_proxy_logger.debug(f"Using team_id from x-litellm-team-id header: {header_team_id}") + verbose_proxy_logger.debug("Using team_id from x-litellm-team-id header: %s", header_team_id) return header_team_id @staticmethod @@ -1666,11 +1668,13 @@ class JWTAuthManager: user_role=LitellmUserRoles.PROXY_ADMIN ), # [TODO]: expose an internal service role, for better tracking ) - verbose_proxy_logger.debug(f"Successfully added user {user_object.user_id} to team {team_object.team_id}") + verbose_proxy_logger.debug( + "Successfully added user %s to team %s", user_object.user_id, team_object.team_id + ) except ProxyException as e: if e.type == ProxyErrorTypes.team_member_already_in_team: verbose_proxy_logger.debug( - f"User {user_object.user_id} is already a member of team {team_object.team_id}" + "User %s is already a member of team %s", user_object.user_id, team_object.team_id ) return else: diff --git a/litellm/proxy/auth/litellm_license.py b/litellm/proxy/auth/litellm_license.py index 1f61ef7ea28..357ea284103 100644 --- a/litellm/proxy/auth/litellm_license.py +++ b/litellm/proxy/auth/litellm_license.py @@ -26,7 +26,7 @@ class LicenseCheck: def __init__(self) -> None: self.license_str = os.getenv("LITELLM_LICENSE", None) - verbose_proxy_logger.debug(f"License Str value - {self.license_str}") + verbose_proxy_logger.debug("License Str value - %s", self.license_str) self.http_handler = HTTPHandler(timeout=NON_LLM_CONNECTION_TIMEOUT) self._premium_check_logged = False self.public_key = None @@ -48,11 +48,13 @@ class LicenseCheck: else: self.public_key = None except Exception as e: - verbose_proxy_logger.error(f"Error reading public key: {e}") + verbose_proxy_logger.error("Error reading public key: %s", e) def _verify(self, license_str: str) -> bool: verbose_proxy_logger.debug( - f"litellm.proxy.auth.litellm_license.py::_verify - Checking license against {self.base_url}/verify_license - {license_str}" + "litellm.proxy.auth.litellm_license.py::_verify - Checking license against %s/verify_license - %s", + self.base_url, + license_str, ) url = f"{self.base_url}/verify_license/{license_str}" @@ -79,12 +81,14 @@ class LicenseCheck: assert isinstance(premium, bool) verbose_proxy_logger.debug( - f"litellm.proxy.auth.litellm_license.py::_verify - License={license_str} is premium={premium}" + "litellm.proxy.auth.litellm_license.py::_verify - License=%s is premium=%s", license_str, premium ) return premium except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.auth.litellm_license.py::_verify - Unable to verify License={license_str} via api. - {e}" + "litellm.proxy.auth.litellm_license.py::_verify - Unable to verify License=%s via api. - %s", + license_str, + e, ) return False @@ -96,7 +100,8 @@ class LicenseCheck: try: if not self._premium_check_logged: verbose_proxy_logger.debug( - f"litellm.proxy.auth.litellm_license.py::is_premium() - ENTERING 'IS_PREMIUM' - LiteLLM License={self.license_str}" + "litellm.proxy.auth.litellm_license.py::is_premium() - ENTERING 'IS_PREMIUM' - LiteLLM License=%s", + self.license_str, ) if self.license_str is None: @@ -104,7 +109,8 @@ class LicenseCheck: if not self._premium_check_logged: verbose_proxy_logger.debug( - f"litellm.proxy.auth.litellm_license.py::is_premium() - Updated 'self.license_str' - {self.license_str}" + "litellm.proxy.auth.litellm_license.py::is_premium() - Updated 'self.license_str' - %s", + self.license_str, ) self._premium_check_logged = True @@ -187,6 +193,7 @@ class LicenseCheck: except Exception as e: verbose_proxy_logger.debug( - f"litellm.proxy.auth.litellm_license.py::verify_license_without_api_request - Unable to verify License locally. - {e}" + "litellm.proxy.auth.litellm_license.py::verify_license_without_api_request - Unable to verify License locally. - %s", + e, ) return False diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 24875dae9ab..fa0abe71081 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -132,7 +132,7 @@ def get_key_models( # deduplicate while preserving order all_models = list(dict.fromkeys(all_models)) - verbose_proxy_logger.debug(f"ALL KEY MODELS - {len(all_models)}") + verbose_proxy_logger.debug("ALL KEY MODELS - %s", len(all_models)) return all_models @@ -173,7 +173,7 @@ def get_team_models( # deduplicate while preserving order all_models = list(dict.fromkeys(all_models)) - verbose_proxy_logger.debug(f"ALL TEAM MODELS - {len(all_models)}") + verbose_proxy_logger.debug("ALL TEAM MODELS - %s", len(all_models)) return all_models @@ -448,7 +448,7 @@ def get_all_fallbacks( elif fallback_type == "content_policy": fallbacks_config = getattr(llm_router, "content_policy_fallbacks", []) else: - verbose_proxy_logger.warning(f"Unknown fallback_type: {fallback_type}") + verbose_proxy_logger.warning("Unknown fallback_type: %s", fallback_type) return [] if not fallbacks_config: @@ -463,5 +463,5 @@ def get_all_fallbacks( return fallback_model_group except Exception as e: - verbose_proxy_logger.error(f"Error getting fallbacks for model {model}: {e}") + verbose_proxy_logger.error("Error getting fallbacks for model %s: %s", model, e) return [] diff --git a/litellm/proxy/auth/oauth2_proxy_hook.py b/litellm/proxy/auth/oauth2_proxy_hook.py index a7e34072712..9c0bf11a851 100644 --- a/litellm/proxy/auth/oauth2_proxy_hook.py +++ b/litellm/proxy/auth/oauth2_proxy_hook.py @@ -65,7 +65,7 @@ async def handle_oauth2_proxy_request(request: Request) -> UserAPIKeyAuth: ) oauth2_config_mappings: dict[str, str] = general_settings.get("oauth2_config_mappings") or {} - verbose_proxy_logger.debug(f"Oauth2 config mappings: {oauth2_config_mappings}") + verbose_proxy_logger.debug("Oauth2 config mappings: %s", oauth2_config_mappings) if not oauth2_config_mappings: raise ValueError("Oauth2 config mappings not found in general_settings") diff --git a/litellm/proxy/auth/resolvers/store.py b/litellm/proxy/auth/resolvers/store.py index 0702f47fa99..e15a5668701 100644 --- a/litellm/proxy/auth/resolvers/store.py +++ b/litellm/proxy/auth/resolvers/store.py @@ -132,7 +132,9 @@ class IdentityStore: ) except Exception as e: verbose_proxy_logger.debug( - f"Failed to load object_permission for key with object_permission_id={key.object_permission_id}: {e}" + "Failed to load object_permission for key with object_permission_id=%s: %s", + key.object_permission_id, + e, ) await _cache_key_object( diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 1e63d9746a4..6c485504a0f 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -258,7 +258,7 @@ class RouteChecks: # check if user can access this route query_params = request.query_params user_id = query_params.get("user_id") - verbose_proxy_logger.debug(f"user_id: {user_id} & valid_token.user_id: {valid_token.user_id}") + verbose_proxy_logger.debug("user_id: %s & valid_token.user_id: %s", user_id, valid_token.user_id) if user_id and user_id != valid_token.user_id: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, @@ -326,7 +326,8 @@ class RouteChecks: if "admin_only_routes" in general_settings: if premium_user is not True: verbose_proxy_logger.error( - f"Trying to use 'admin_only_routes' this is an Enterprise only feature. {CommonProxyErrors.not_premium_user.value}" + "Trying to use 'admin_only_routes' this is an Enterprise only feature. %s", + CommonProxyErrors.not_premium_user.value, ) return if route in general_settings["admin_only_routes"]: diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 286837c8909..b295a25ab9f 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -102,7 +102,7 @@ try: enterprise_custom_auth: Callable | None = _enterprise_custom_auth except ImportError as e: - verbose_proxy_logger.debug(f"Error in enterprise custom auth: {e}") + verbose_proxy_logger.debug("Error in enterprise custom auth: %s", e) enterprise_custom_auth = None user_api_key_service_logger_obj = ServiceLogging() # used for tracking latency on OTEL @@ -406,7 +406,7 @@ def _apply_budget_limits_to_end_user_params( if budget_info.model_max_budget is not None: end_user_params["end_user_model_max_budget"] = budget_info.model_max_budget - verbose_proxy_logger.debug(f"Applied budget limits to end user {end_user_id}") + verbose_proxy_logger.debug("Applied budget limits to end user %s", end_user_id) async def user_api_key_auth_websocket(websocket: WebSocket): @@ -865,7 +865,9 @@ async def _resolve_jwt_to_virtual_key( ) if claim_value is None: - verbose_proxy_logger.debug(f"JWT Key Mapping: Claim field '{virtual_key_claim_field}' not found in JWT claims.") + verbose_proxy_logger.debug( + "JWT Key Mapping: Claim field '%s' not found in JWT claims.", virtual_key_claim_field + ) # A missing claim is an unmapped client — apply the no-match policy # rather than returning early. Otherwise a caller can bypass REJECT # simply by presenting a JWT that omits the configured field. For @@ -1248,7 +1250,7 @@ async def _user_api_key_auth_builder( user_email=mapped_user_email, ) except Exception as e: - verbose_proxy_logger.debug(f"JWT mapped-key user_email backfill skipped: {e}") + verbose_proxy_logger.debug("JWT mapped-key user_email backfill skipped: %s", e) else: if mapped_user_obj is not None: valid_token.user_email = mapped_user_obj.user_email @@ -1390,7 +1392,7 @@ async def _user_api_key_auth_builder( skip_budget_checks = _is_model_cost_zero(model=model, llm_router=llm_router) if skip_budget_checks: - verbose_proxy_logger.info(f"Skipping all budget checks for zero-cost model: {model}") + verbose_proxy_logger.info("Skipping all budget checks for zero-cost model: %s", model) # Fetch project object for JWT path if project_id is set _jwt_project_obj = None @@ -1501,7 +1503,7 @@ async def _user_api_key_auth_builder( except Exception as e: if isinstance(e, litellm.BudgetExceededError): raise e - verbose_proxy_logger.debug(f"Unable to find user in db. Error - {e}") + verbose_proxy_logger.debug("Unable to find user in db. Error - %s", e) ### CHECK IF ADMIN ### # note: never string compare api keys, this is vulenerable to a time attack. Use secrets.compare_digest instead @@ -1749,7 +1751,8 @@ async def _user_api_key_auth_builder( ) except Exception as e: verbose_logger.debug( - f"litellm.proxy.auth.user_api_key_auth.py::user_api_key_auth() - Unable to get user from db/cache. Setting user_obj to None. Exception received - {e}" + "litellm.proxy.auth.user_api_key_auth.py::user_api_key_auth() - Unable to get user from db/cache. Setting user_obj to None. Exception received - %s", + e, ) user_obj = None @@ -1775,7 +1778,7 @@ async def _user_api_key_auth_builder( skip_budget_checks = _is_model_cost_zero(model=model, llm_router=llm_router) if skip_budget_checks: - verbose_proxy_logger.info(f"Skipping all budget checks for zero-cost model: {model}") + verbose_proxy_logger.info("Skipping all budget checks for zero-cost model: %s", model) # Check 3. Check if user is in their team budget if not skip_budget_checks and valid_token.team_member_spend is not None: @@ -1839,7 +1842,7 @@ async def _user_api_key_auth_builder( if expiry_time.tzinfo is None or expiry_time.tzinfo.utcoffset(expiry_time) is None: expiry_time = expiry_time.replace(tzinfo=timezone.utc) verbose_proxy_logger.debug( - f"Checking if token expired, expiry time {expiry_time} and current time {current_time}" + "Checking if token expired, expiry time %s and current time %s", expiry_time, current_time ) if expiry_time < current_time: # Token exists but is expired. @@ -2689,11 +2692,13 @@ def get_api_key_from_custom_header(request: Request, custom_litellm_key_header_n if custom_api_key: api_key = _get_bearer_token(api_key=custom_api_key) verbose_proxy_logger.debug( - f"Found custom API key using header: {custom_litellm_key_header_name}, setting api_key={abbreviate_api_key(api_key)}" + "Found custom API key using header: %s, setting api_key=%s", + custom_litellm_key_header_name, + abbreviate_api_key(api_key), ) else: verbose_proxy_logger.exception( - f"No LiteLLM Virtual Key pass. Please set header={custom_litellm_key_header_name}: Bearer " + "No LiteLLM Virtual Key pass. Please set header=%s: Bearer ", custom_litellm_key_header_name ) return api_key @@ -2774,7 +2779,7 @@ async def _lookup_end_user_and_apply_budget( except Exception as e: if isinstance(e, litellm.BudgetExceededError): raise e - verbose_proxy_logger.debug(f"Unable to find user in db. Error - {e}") + verbose_proxy_logger.debug("Unable to find user in db. Error - %s", e) return valid_token, end_user_object @@ -2796,7 +2801,7 @@ async def _enforce_key_and_fallback_model_access( if config != {}: model_list = config.get("model_list", []) new_model_list = model_list - verbose_proxy_logger.debug(f"\n new llm router model list {new_model_list}") + verbose_proxy_logger.debug("\n new llm router model list %s", new_model_list) elif isinstance(valid_token.models, list) and "all-team-models" in valid_token.models: pass else: diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index 0c2764db33f..0d815d370e7 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -120,7 +120,8 @@ async def create_batch( try: data = await _read_request_body(request=request) verbose_proxy_logger.debug( - f"Request received by LiteLLM:\n{json.dumps(data, indent=4)}", + "Request received by LiteLLM:\n%s", + json.dumps(data, indent=4), ) base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) ( @@ -225,8 +226,10 @@ async def create_batch( ) verbose_proxy_logger.debug( - f"Created batch using model: {model_from_file_id}, " - f"original_batch_id: {original_batch_id}, encoded: {encoded_batch_id}" + "Created batch using model: %s, original_batch_id: %s, encoded: %s", + model_from_file_id, + original_batch_id, + encoded_batch_id, ) response.input_file_id = input_file_id @@ -293,7 +296,7 @@ async def create_batch( encode_batch_response_ids(response, model=model_param) - verbose_proxy_logger.debug(f"Created batch using model: {model_param}") + verbose_proxy_logger.debug("Created batch using model: %s", model_param) else: # SCENARIO 3: Fallback to custom_llm_provider (uses env variables) apply_team_provider_credentials( @@ -340,7 +343,7 @@ async def create_batch( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.create_batch(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.create_batch(): Exception occured - %s", e) raise handle_exception_on_proxy(e) @@ -471,7 +474,7 @@ async def retrieve_batch( # If batch is still processing, sync with provider to get latest state if response is not None: verbose_proxy_logger.debug( - f"Batch {batch_id} is in non-terminal state {response.status}, syncing with provider" + "Batch %s is in non-terminal state %s, syncing with provider", batch_id, response.status ) # Retrieve from provider (for non-terminal states or if DB lookup failed) @@ -505,7 +508,7 @@ async def retrieve_batch( encode_batch_response_ids(response, model=model_from_id) verbose_proxy_logger.debug( - f"Retrieved batch using model: {model_from_id}, original_id: {original_batch_id}" + "Retrieved batch using model: %s, original_id: %s", model_from_id, original_batch_id ) elif litellm.enable_loadbalancing_on_batch_endpoints is True or unified_batch_id: @@ -592,7 +595,7 @@ async def retrieve_batch( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.retrieve_batch(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.retrieve_batch(): Exception occured - %s", e) raise handle_exception_on_proxy(e) @@ -641,7 +644,7 @@ async def list_batches( version, ) - verbose_proxy_logger.debug(f"GET /v1/batches after={after} limit={limit}") + verbose_proxy_logger.debug("GET /v1/batches after=%s limit=%s", after, limit) try: if llm_router is None: raise HTTPException( @@ -703,7 +706,7 @@ async def list_batches( for batch in response_data: encode_batch_response_ids(batch, model=model_param) - verbose_proxy_logger.debug(f"Listed batches using model: {model_param}") + verbose_proxy_logger.debug("Listed batches using model: %s", model_param) # SCENARIO 2 (alternative): target_model_names based routing elif target_model_names or data.get("target_model_names", None): @@ -773,7 +776,7 @@ async def list_batches( original_exception=e, request_data={"after": after, "limit": limit}, ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.retrieve_batch(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.retrieve_batch(): Exception occured - %s", e) raise handle_exception_on_proxy(e) @@ -886,7 +889,7 @@ async def cancel_batch( encode_batch_response_ids(response, model=model_from_id) verbose_proxy_logger.debug( - f"Cancelled batch using model: {model_from_id}, original_id: {original_batch_id}" + "Cancelled batch using model: %s, original_id: %s", model_from_id, original_batch_id ) # SCENARIO 2: target_model_names based routing @@ -982,7 +985,7 @@ async def cancel_batch( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.create_batch(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.create_batch(): Exception occured - %s", e) raise handle_exception_on_proxy(e) diff --git a/litellm/proxy/caching_routes.py b/litellm/proxy/caching_routes.py index 50b2f63e18a..d823fa2d532 100644 --- a/litellm/proxy/caching_routes.py +++ b/litellm/proxy/caching_routes.py @@ -43,7 +43,7 @@ def _extract_cache_params() -> dict[str, Any]: cleaned_params = HealthCheckCacheParams(**cache_params).model_dump() if cache_params else {} return masker.mask_dict(cleaned_params) except (AttributeError, TypeError) as e: - verbose_proxy_logger.debug(f"Error extracting cache params: {e}") + verbose_proxy_logger.debug("Error extracting cache params: %s", e) return {} @@ -173,7 +173,7 @@ def _get_redis_client_info(cache_instance) -> tuple[list, int]: client_list = cache_instance.client_list() return client_list, len(client_list) except Exception as e: - verbose_proxy_logger.warning(f"CLIENT LIST command failed (likely restricted on managed Redis): {e}") + verbose_proxy_logger.warning("CLIENT LIST command failed (likely restricted on managed Redis): %s", e) return ["CLIENT LIST command not available on this Redis instance"], -1 diff --git a/litellm/proxy/client/cli/interface.py b/litellm/proxy/client/cli/interface.py index 9f42c3b06bb..ea20cbb5ea4 100644 --- a/litellm/proxy/client/cli/interface.py +++ b/litellm/proxy/client/cli/interface.py @@ -20,7 +20,7 @@ def styled_prompt(): click.echo("\n" * 3) except Exception as e: # Fallback if we can't get terminal size - verbose_logger.debug(f"Error getting terminal size: {e}") + verbose_logger.debug("Error getting terminal size: %s", e) click.echo("\n" * 3) # ASCII box drawing characters diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index bc3bcd233f0..f9cad283166 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -439,7 +439,7 @@ async def _parse_event_data_for_error(event_line: str | bytes) -> int | None: error_code = int(error_code_raw) except ValueError: verbose_proxy_logger.warning( - f"Error code is a string but not a valid integer: {error_code_raw}" + "Error code is a string but not a valid integer: %s", error_code_raw ) # Not a valid integer string, treat as if no valid code was found for this check @@ -447,7 +447,7 @@ async def _parse_event_data_for_error(event_line: str | bytes) -> int | None: if error_code is not None and 100 <= error_code <= 599: return error_code elif error_code_raw is not None: # Log if original code was present but not valid - verbose_proxy_logger.warning(f"Error has invalid or non-convertible code: {error_code_raw}") + verbose_proxy_logger.warning("Error has invalid or non-convertible code: %s", error_code_raw) except (orjson.JSONDecodeError, json.JSONDecodeError): # not a known error chunk pass @@ -644,7 +644,8 @@ async def create_response( # Should return standard JSON error response instead of SSE format final_status_code = error_code_from_chunk verbose_proxy_logger.debug( - f"Error detected in first stream chunk. Returning JSON error response with status code: {final_status_code}" + "Error detected in first stream chunk. Returning JSON error response with status code: %s", + final_status_code, ) # Parse error content @@ -663,7 +664,7 @@ async def create_response( headers=headers, ) except Exception as e: - verbose_proxy_logger.debug(f"Error parsing first chunk value: {e}") + verbose_proxy_logger.debug("Error parsing first chunk value: %s", e) except _ClientDisconnectedBeforeFirstChunk: # Client vanished during the time-to-first-token wait; the upstream @@ -694,7 +695,7 @@ async def create_response( ) except Exception as e: # Unexpected error consuming first chunk. - verbose_proxy_logger.exception(f"Error consuming first chunk from generator: {e}") + verbose_proxy_logger.exception("Error consuming first chunk from generator: %s", e) # Preserve status code from HTTPException (e.g., guardrail blocks) error_status = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR) @@ -943,7 +944,7 @@ def _log_llm_api_exception(e: Exception) -> None: "litellm.proxy.proxy_server._handle_llm_api_exception(): client disconnected, upstream LLM request cancelled" ) return - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - %s", e) async def _cancel_llm_call_on_client_disconnect( @@ -1083,7 +1084,7 @@ class ProxyBaseLLMRequestProcessing: try: return {key: str(value) for key, value in headers.items() if value not in exclude_values} except Exception as e: - verbose_proxy_logger.error(f"Error setting custom headers: {e}") + verbose_proxy_logger.error("Error setting custom headers: %s", e) return {} @staticmethod @@ -2945,7 +2946,7 @@ class ProxyBaseLLMRequestProcessing: raise except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.async_data_generator(): Exception occured - {e}" + "litellm.proxy.proxy_server.async_data_generator(): Exception occured - %s", e ) transformed_exception = await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, @@ -2955,7 +2956,8 @@ class ProxyBaseLLMRequestProcessing: if transformed_exception is not None: e = transformed_exception verbose_proxy_logger.debug( - f"\033[1;31mAn error occurred: {e}\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`" + "\x1b[1;31mAn error occurred: %s\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`", + e, ) if isinstance(e, HTTPException): diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 1eca0eb768c..6325a27b7d1 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -61,7 +61,7 @@ def initialize_callbacks_on_proxy( ) from litellm.proxy.proxy_server import prisma_client - verbose_proxy_logger.debug(f"{blue_color_code}initializing callbacks={value} on proxy{reset_color_code}") + verbose_proxy_logger.debug("%sinitializing callbacks=%s on proxy%s", blue_color_code, value, reset_color_code) if isinstance(value, list): imported_list: list[Any] = [] for callback in value: # ["presidio", ] @@ -298,7 +298,7 @@ def initialize_callbacks_on_proxy( imported_list.append(callback) else: verbose_proxy_logger.debug( - f"{blue_color_code} attempting to import custom calback={callback} {reset_color_code}" + "%s attempting to import custom calback=%s %s", blue_color_code, callback, reset_color_code ) imported_list.append( get_instance_fn( @@ -322,7 +322,7 @@ def initialize_callbacks_on_proxy( config_file_path=config_file_path, ) ] - verbose_proxy_logger.debug(f"{blue_color_code} Initialized Callbacks - {litellm.callbacks} {reset_color_code}") + verbose_proxy_logger.debug("%s Initialized Callbacks - %s %s", blue_color_code, litellm.callbacks, reset_color_code) def get_model_group_from_litellm_kwargs(kwargs: dict) -> str | None: diff --git a/litellm/proxy/common_utils/custom_openapi_spec.py b/litellm/proxy/common_utils/custom_openapi_spec.py index a884eab462a..de4f9ceaa63 100644 --- a/litellm/proxy/common_utils/custom_openapi_spec.py +++ b/litellm/proxy/common_utils/custom_openapi_spec.py @@ -49,7 +49,7 @@ class CustomOpenAPISpec: except Exception as e: # FastAPI 0.120+ may fail schema generation for certain types (e.g., openai.Timeout) # Log the error and return None to skip schema generation for this model - verbose_proxy_logger.debug(f"Failed to generate schema for {model_class}: {e}") + verbose_proxy_logger.debug("Failed to generate schema for %s: %s", model_class, e) return None @staticmethod @@ -267,13 +267,13 @@ class CustomOpenAPISpec: openapi_schema, paths, f"#/components/schemas/{schema_name}" ) - verbose_proxy_logger.debug(f"Successfully added {schema_name} schema to OpenAPI spec") + verbose_proxy_logger.debug("Successfully added %s schema to OpenAPI spec", schema_name) else: - verbose_proxy_logger.debug(f"Could not get schema for {schema_name}") + verbose_proxy_logger.debug("Could not get schema for %s", schema_name) except Exception as e: # If schema addition fails, continue without it - verbose_proxy_logger.debug(f"Failed to add {operation_name} request schema: {e}") + verbose_proxy_logger.debug("Failed to add %s request schema: %s", operation_name, e) return openapi_schema @@ -302,7 +302,7 @@ class CustomOpenAPISpec: operation_name="chat completion", ) except ImportError as e: - verbose_proxy_logger.debug(f"Failed to import ProxyChatCompletionRequest: {e}") + verbose_proxy_logger.debug("Failed to import ProxyChatCompletionRequest: %s", e) return openapi_schema @staticmethod @@ -328,7 +328,7 @@ class CustomOpenAPISpec: operation_name="embedding", ) except ImportError as e: - verbose_proxy_logger.debug(f"Failed to import EmbeddingRequest: {e}") + verbose_proxy_logger.debug("Failed to import EmbeddingRequest: %s", e) return openapi_schema @staticmethod @@ -356,7 +356,7 @@ class CustomOpenAPISpec: operation_name="responses API", ) except ImportError as e: - verbose_proxy_logger.debug(f"Failed to import ResponsesAPIRequestParams: {e}") + verbose_proxy_logger.debug("Failed to import ResponsesAPIRequestParams: %s", e) return openapi_schema @staticmethod diff --git a/litellm/proxy/common_utils/debug_utils.py b/litellm/proxy/common_utils/debug_utils.py index 7d2b303a7ca..7cbf8d4527e 100644 --- a/litellm/proxy/common_utils/debug_utils.py +++ b/litellm/proxy/common_utils/debug_utils.py @@ -29,18 +29,21 @@ def configure_gc_thresholds(): thresholds = [int(x.strip()) for x in gc_threshold_env.split(",")] if len(thresholds) == 3: gc.set_threshold(*thresholds) - verbose_proxy_logger.info(f"GC thresholds set to: {thresholds}") + verbose_proxy_logger.info("GC thresholds set to: %s", thresholds) else: verbose_proxy_logger.warning( - f"GC threshold not set: {gc_threshold_env}. Expected format: 'gen0,gen1,gen2'" + "GC threshold not set: %s. Expected format: 'gen0,gen1,gen2'", gc_threshold_env ) except ValueError as e: - verbose_proxy_logger.warning(f"Failed to parse GC threshold: {gc_threshold_env}. Error: {e}") + verbose_proxy_logger.warning("Failed to parse GC threshold: %s. Error: %s", gc_threshold_env, e) # Log current thresholds current_thresholds = gc.get_threshold() verbose_proxy_logger.info( - f"Current GC thresholds: gen0={current_thresholds[0]}, gen1={current_thresholds[1]}, gen2={current_thresholds[2]}" + "Current GC thresholds: gen0=%s, gen1=%s, gen2=%s", + current_thresholds[0], + current_thresholds[1], + current_thresholds[2], ) @@ -425,12 +428,12 @@ def _get_cache_memory_stats(user_api_key_cache, llm_router, proxy_logging_obj, r ), } except Exception as e: - verbose_proxy_logger.debug(f"Error getting Redis pool info: {e}") + verbose_proxy_logger.debug("Error getting Redis pool info: %s", e) else: cache_stats["redis_usage_cache"] = {"enabled": False} except Exception as e: - verbose_proxy_logger.debug(f"Error calculating cache stats: {e}") + verbose_proxy_logger.debug("Error calculating cache stats: %s", e) cache_stats["error"] = str(e) return cache_stats @@ -496,7 +499,7 @@ def _get_router_memory_stats(llm_router) -> dict[str, Any]: else: litellm_router_memory = {"note": "Router not initialized"} except Exception as e: - verbose_proxy_logger.debug(f"Error getting router memory info: {e}") + verbose_proxy_logger.debug("Error getting router memory info: %s", e) litellm_router_memory = {"error": str(e)} return litellm_router_memory @@ -546,7 +549,7 @@ def _get_process_memory_info(worker_pid: int, include_process_info: bool) -> dic "error": "psutil not installed. Install with: pip install psutil", } except Exception as e: - verbose_proxy_logger.debug(f"Error getting process info: {e}") + verbose_proxy_logger.debug("Error getting process info: %s", e) return {"pid": worker_pid, "error": str(e)} @@ -649,10 +652,10 @@ async def configure_gc_thresholds_endpoint( try: gc.set_threshold(generation_0, generation_1, generation_2) verbose_proxy_logger.info( - f"GC thresholds updated from {old_thresholds} to ({generation_0}, {generation_1}, {generation_2})" + "GC thresholds updated from %s to (%s, %s, %s)", old_thresholds, generation_0, generation_1, generation_2 ) except Exception as e: - verbose_proxy_logger.error(f"Failed to set GC thresholds: {e}") + verbose_proxy_logger.error("Failed to set GC thresholds: %s", e) raise HTTPException(status_code=500, detail=f"Failed to set GC thresholds: {e}") # Get current object count to show immediate impact @@ -783,4 +786,4 @@ def init_verbose_loggers(): except Exception as e: import logging - logging.warning(f"Failed to init verbose loggers: {e}") + logging.warning("Failed to init verbose loggers: %s", e) diff --git a/litellm/proxy/common_utils/encrypt_decrypt_utils.py b/litellm/proxy/common_utils/encrypt_decrypt_utils.py index 651e59ef959..e288de6ec44 100644 --- a/litellm/proxy/common_utils/encrypt_decrypt_utils.py +++ b/litellm/proxy/common_utils/encrypt_decrypt_utils.py @@ -108,7 +108,7 @@ def encrypt_value_helper(value: str, new_encryption_key: str | None = None): return encrypted_value verbose_proxy_logger.debug( - f"Invalid value type passed to encrypt_value: {type(value)} for Value: {value}\n Value must be a string" + "Invalid value type passed to encrypt_value: %s for Value: %s\n Value must be a string", type(value), value ) # if it's not a string - do not encrypt it and return the value return value @@ -150,7 +150,7 @@ def decrypt_value_helper( verbose_proxy_logger.debug(error_message) return value if return_original_value else None - verbose_proxy_logger.debug(f"Unable to decrypt value for key: {key}, returning None") + verbose_proxy_logger.debug("Unable to decrypt value for key: %s, returning None", key) if return_original_value: return value else: diff --git a/litellm/proxy/common_utils/expired_ui_session_key_cleanup_manager.py b/litellm/proxy/common_utils/expired_ui_session_key_cleanup_manager.py index a8acc28d9de..cabd1ca84ef 100644 --- a/litellm/proxy/common_utils/expired_ui_session_key_cleanup_manager.py +++ b/litellm/proxy/common_utils/expired_ui_session_key_cleanup_manager.py @@ -101,7 +101,7 @@ class ExpiredUISessionKeyCleanupManager: e, ) return 0 - verbose_proxy_logger.error(f"Expired UI session key cleanup failed: {e}") + verbose_proxy_logger.error("Expired UI session key cleanup failed: %s", e) return 0 finally: if lock_acquired and self.pod_lock_manager and self.pod_lock_manager.redis_cache: diff --git a/litellm/proxy/common_utils/get_routes.py b/litellm/proxy/common_utils/get_routes.py index 4e3c908a8bb..4f320889fe6 100644 --- a/litellm/proxy/common_utils/get_routes.py +++ b/litellm/proxy/common_utils/get_routes.py @@ -70,5 +70,5 @@ class GetRoutes: else: return None except Exception: - verbose_logger.exception(f"Error getting endpoint name for route: {endpoint_function}") + verbose_logger.exception("Error getting endpoint name for route: %s", endpoint_function) return None diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 67212539cc4..f8cfb14326d 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -72,7 +72,7 @@ async def _read_request_body(request: Request | None) -> dict: # a later raw-body re-read sees the original payload — # banned-param checks must see the same body the handler # acts on. - verbose_proxy_logger.error(f"Invalid form payload: {e}") + verbose_proxy_logger.error("Invalid form payload: %s", e) raise ProxyException( message=f"Invalid form payload: {e}", type="invalid_request_error", @@ -98,7 +98,7 @@ async def _read_request_body(request: Request | None) -> dict: # Above the configured size, skip the repair and raise the 400 now. repair_limit_bytes = MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB * 1024 * 1024 if repair_limit_bytes > 0 and len(body) > repair_limit_bytes: - verbose_proxy_logger.error(f"Invalid JSON payload received: {e}") + verbose_proxy_logger.error("Invalid JSON payload received: %s", e) raise ProxyException( message=f"Invalid JSON payload: {e}", type="invalid_request_error", @@ -120,7 +120,7 @@ async def _read_request_body(request: Request | None) -> dict: parsed_body = json.loads(body_str) except json.JSONDecodeError: # If both orjson and json.loads fail, throw a proper error - verbose_proxy_logger.error(f"Invalid JSON payload received: {e}") + verbose_proxy_logger.error("Invalid JSON payload received: %s", e) raise ProxyException( message=f"Invalid JSON payload: {e}", type="invalid_request_error", @@ -134,11 +134,11 @@ async def _read_request_body(request: Request | None) -> dict: except (json.JSONDecodeError, orjson.JSONDecodeError, ProxyException) as e: # Re-raise ProxyException as-is - verbose_proxy_logger.error(f"Invalid JSON payload received: {e}") + verbose_proxy_logger.error("Invalid JSON payload received: %s", e) raise except Exception as e: # Catch unexpected errors to avoid crashes - verbose_proxy_logger.exception(f"Unexpected error reading request body - {e}") + verbose_proxy_logger.exception("Unexpected error reading request body - %s", e) return {} @@ -159,7 +159,7 @@ def _safe_get_request_query_params(request: Request | None) -> dict: return dict(request.query_params) return {} except Exception as e: - verbose_proxy_logger.debug(f"Unexpected error reading request query params - {e}") + verbose_proxy_logger.debug("Unexpected error reading request query params - %s", e) return {} @@ -172,7 +172,7 @@ def _safe_set_request_parsed_body( return request.scope["parsed_body"] = (tuple(parsed_body.keys()), parsed_body) except Exception as e: - verbose_proxy_logger.debug(f"Unexpected error setting request parsed body - {e}") + verbose_proxy_logger.debug("Unexpected error setting request parsed body - %s", e) def _safe_get_request_headers(request: Request | None) -> dict: @@ -190,11 +190,11 @@ def _safe_get_request_headers(request: Request | None) -> dict: if isinstance(cached, dict): return cached if cached is not None: - verbose_proxy_logger.debug(f"Unexpected cached request headers type - {type(cached)}") + verbose_proxy_logger.debug("Unexpected cached request headers type - %s", type(cached)) try: headers = dict(request.headers) except Exception as e: - verbose_proxy_logger.debug(f"Unexpected error reading request headers - {e}") + verbose_proxy_logger.debug("Unexpected error reading request headers - %s", e) headers = {} try: if state is not None: @@ -393,7 +393,7 @@ def extract_nested_form_metadata(form_data: dict[str, Any], prefix: str = "litel # Skip UploadFile objects - they should not be in metadata if isinstance(value, UploadFile): - verbose_proxy_logger.warning(f"Skipping UploadFile in metadata extraction for key: {key}") + verbose_proxy_logger.warning("Skipping UploadFile in metadata extraction for key: %s", key) continue # Extract the nested path from bracket notation @@ -406,7 +406,7 @@ def extract_nested_form_metadata(form_data: dict[str, Any], prefix: str = "litel parts = path_string.split("][") if not parts or not parts[0]: - verbose_proxy_logger.warning(f"Invalid metadata key format (empty path): {key}") + verbose_proxy_logger.warning("Invalid metadata key format (empty path): %s", key) continue # Navigate/create nested dictionary structure @@ -414,7 +414,7 @@ def extract_nested_form_metadata(form_data: dict[str, Any], prefix: str = "litel for part in parts[:-1]: if not isinstance(current, dict): verbose_proxy_logger.warning( - f"Cannot create nested path - intermediate value is not a dict at: {part}" + "Cannot create nested path - intermediate value is not a dict at: %s", part ) break current = current.setdefault(part, {}) @@ -423,10 +423,10 @@ def extract_nested_form_metadata(form_data: dict[str, Any], prefix: str = "litel if isinstance(current, dict): current[parts[-1]] = value else: - verbose_proxy_logger.warning(f"Cannot set value - parent is not a dict for key: {key}") + verbose_proxy_logger.warning("Cannot set value - parent is not a dict for key: %s", key) except Exception as e: - verbose_proxy_logger.error(f"Error parsing metadata key '{key}': {e}") + verbose_proxy_logger.error("Error parsing metadata key '%s': %s", key, e) continue return metadata @@ -505,7 +505,8 @@ def populate_request_with_path_params(request_data: dict, request: Request) -> d continue request_data.setdefault(key, value) verbose_proxy_logger.debug( - f"populate_request_with_path_params: Found path_params, vector_store_ids={request_data.get('vector_store_ids')}" + "populate_request_with_path_params: Found path_params, vector_store_ids=%s", + request_data.get("vector_store_ids"), ) return request_data @@ -533,7 +534,7 @@ def _add_vector_store_id_from_path(request_data: dict, request: Request) -> None if vector_store_match: vector_store_id = vector_store_match.group(1) verbose_proxy_logger.debug( - f"populate_request_with_path_params: Extracted vector_store_id={vector_store_id} from path={path}" + "populate_request_with_path_params: Extracted vector_store_id=%s from path=%s", vector_store_id, path ) request_data.setdefault("vector_store_id", vector_store_id) existing_ids = request_data.get("vector_store_ids") @@ -543,7 +544,8 @@ def _add_vector_store_id_from_path(request_data: dict, request: Request) -> None else: request_data["vector_store_ids"] = [vector_store_id] verbose_proxy_logger.debug( - f"populate_request_with_path_params: Updated request_data with vector_store_ids={request_data.get('vector_store_ids')}" + "populate_request_with_path_params: Updated request_data with vector_store_ids=%s", + request_data.get("vector_store_ids"), ) else: - verbose_proxy_logger.debug(f"populate_request_with_path_params: No vector_store_id present in path={path}") + verbose_proxy_logger.debug("populate_request_with_path_params: No vector_store_id present in path=%s", path) diff --git a/litellm/proxy/common_utils/key_rotation_manager.py b/litellm/proxy/common_utils/key_rotation_manager.py index 8e065a06979..a7e8f7aeb13 100644 --- a/litellm/proxy/common_utils/key_rotation_manager.py +++ b/litellm/proxy/common_utils/key_rotation_manager.py @@ -84,20 +84,20 @@ class KeyRotationManager: verbose_proxy_logger.debug("No keys are due for rotation at this time") return - verbose_proxy_logger.info(f"Found {len(keys_to_rotate)} keys due for rotation") + verbose_proxy_logger.info("Found %s keys due for rotation", len(keys_to_rotate)) # Rotate each key for key in keys_to_rotate: try: await self._rotate_key(key) key_identifier = key.key_name or (key.token[:8] + "..." if key.token else "unknown") - verbose_proxy_logger.info(f"Successfully rotated key: {key_identifier}") + verbose_proxy_logger.info("Successfully rotated key: %s", key_identifier) except Exception as e: key_identifier = key.key_name or (key.token[:8] + "..." if key.token else "unknown") - verbose_proxy_logger.error(f"Failed to rotate key {key_identifier}: {e}") + verbose_proxy_logger.error("Failed to rotate key %s: %s", key_identifier, e) except Exception as e: - verbose_proxy_logger.error(f"Key rotation process failed: {e}") + verbose_proxy_logger.error("Key rotation process failed: %s", e) finally: # Only release the lock if it was actually acquired if lock_acquired and self.pod_lock_manager and self.pod_lock_manager.redis_cache: diff --git a/litellm/proxy/common_utils/load_config_utils.py b/litellm/proxy/common_utils/load_config_utils.py index 56aedb76590..67b3baaf626 100644 --- a/litellm/proxy/common_utils/load_config_utils.py +++ b/litellm/proxy/common_utils/load_config_utils.py @@ -20,9 +20,9 @@ def get_file_contents_from_s3(bucket_name, object_key): aws_secret_access_key=credentials.secret_key, aws_session_token=credentials.token, # Optional, if using temporary credentials ) - verbose_proxy_logger.debug(f"Retrieving {object_key} from S3 bucket: {bucket_name}") + verbose_proxy_logger.debug("Retrieving %s from S3 bucket: %s", object_key, bucket_name) response = s3_client.get_object(Bucket=bucket_name, Key=object_key) - verbose_proxy_logger.debug(f"Response: {response}") + verbose_proxy_logger.debug("Response: %s", response) # Read the file contents and directly parse YAML file_contents = response["Body"].read().decode("utf-8") @@ -34,9 +34,9 @@ def get_file_contents_from_s3(bucket_name, object_key): except ImportError as e: # this is most likely if a user is not using the litellm docker container - verbose_proxy_logger.error(f"ImportError: {e}") + verbose_proxy_logger.error("ImportError: %s", e) except Exception as e: - verbose_proxy_logger.error(f"Error retrieving file contents: {e}") + verbose_proxy_logger.error("Error retrieving file contents: %s", e) return None @@ -57,7 +57,7 @@ async def get_config_file_contents_from_gcs(bucket_name, object_key): return config except Exception as e: - verbose_proxy_logger.error(f"Error retrieving file contents: {e}") + verbose_proxy_logger.error("Error retrieving file contents: %s", e) return None @@ -93,12 +93,12 @@ def download_python_file_from_s3( aws_session_token=credentials.token, ) - verbose_proxy_logger.debug(f"Downloading Python file {object_key} from S3 bucket: {bucket_name}") + verbose_proxy_logger.debug("Downloading Python file %s from S3 bucket: %s", object_key, bucket_name) response = s3_client.get_object(Bucket=bucket_name, Key=object_key) # Read the file contents file_contents = response["Body"].read().decode("utf-8") - verbose_proxy_logger.debug(f"File contents: {file_contents}") + verbose_proxy_logger.debug("File contents: %s", file_contents) # Ensure directory exists os.makedirs(os.path.dirname(local_file_path), exist_ok=True) @@ -107,14 +107,14 @@ def download_python_file_from_s3( with open(local_file_path, "w") as f: f.write(file_contents) - verbose_proxy_logger.debug(f"Python file downloaded successfully to {local_file_path}") + verbose_proxy_logger.debug("Python file downloaded successfully to %s", local_file_path) return True except ImportError as e: - verbose_proxy_logger.error(f"ImportError: {e}") + verbose_proxy_logger.error("ImportError: %s", e) return False except Exception as e: - verbose_proxy_logger.exception(f"Error downloading Python file: {e}") + verbose_proxy_logger.exception("Error downloading Python file: %s", e) return False @@ -154,11 +154,11 @@ async def download_python_file_from_gcs( with open(local_file_path, "w") as f: f.write(file_contents) - verbose_proxy_logger.debug(f"Python file downloaded successfully to {local_file_path}") + verbose_proxy_logger.debug("Python file downloaded successfully to %s", local_file_path) return True except Exception as e: - verbose_proxy_logger.exception(f"Error downloading Python file from GCS: {e}") + verbose_proxy_logger.exception("Error downloading Python file from GCS: %s", e) return False diff --git a/litellm/proxy/common_utils/openapi_schema_compat.py b/litellm/proxy/common_utils/openapi_schema_compat.py index 06a18524733..919a9fa3939 100644 --- a/litellm/proxy/common_utils/openapi_schema_compat.py +++ b/litellm/proxy/common_utils/openapi_schema_compat.py @@ -80,7 +80,7 @@ def get_openapi_schema_with_compat( except (ImportError, AttributeError) as e: # If patching fails, try normal generation with error handling - verbose_proxy_logger.debug(f"Could not patch Pydantic schema generation: {e}. Trying normal generation.") + verbose_proxy_logger.debug("Could not patch Pydantic schema generation: %s. Trying normal generation.", e) try: return get_openapi_func( title=title, @@ -97,7 +97,7 @@ def get_openapi_schema_with_compat( ): # If we still get the error, log it and return minimal schema verbose_proxy_logger.warning( - f"PydanticSchemaGenerationError during schema generation: {pydantic_error}" + "PydanticSchemaGenerationError during schema generation: %s", pydantic_error ) return { "openapi": "3.0.0", diff --git a/litellm/proxy/common_utils/performance_utils.py b/litellm/proxy/common_utils/performance_utils.py index 50de40480fd..09ef61b7116 100644 --- a/litellm/proxy/common_utils/performance_utils.py +++ b/litellm/proxy/common_utils/performance_utils.py @@ -55,7 +55,7 @@ def _start_profiling(profile_sampling_rate: float) -> None: if _profiler is None: _profiler = cProfile.Profile() _profiler.enable() - verbose_proxy_logger.info(f"Profiling started with sampling rate: {profile_sampling_rate}") + verbose_proxy_logger.info("Profiling started with sampling rate: %s", profile_sampling_rate) def _start_profiling_for_request(profile_sampling_rate: float) -> bool: @@ -77,9 +77,9 @@ def _save_stats(profile_file: PathLib) -> None: _profiler.dump_stats(str(profile_file)) # Re-enable profiler to continue profiling _profiler.enable() - verbose_proxy_logger.debug(f"Profiling stats saved to {profile_file}") + verbose_proxy_logger.debug("Profiling stats saved to %s", profile_file) except Exception as e: - verbose_proxy_logger.error(f"Error saving profiling stats: {e}") + verbose_proxy_logger.error("Error saving profiling stats: %s", e) # Make sure profiler is re-enabled even if there's an error try: _profiler.enable() @@ -178,7 +178,7 @@ def wrap_function_with_line_profiler(module: Any, function_name: str) -> bool: try: original_function = getattr(module, function_name, None) if original_function is None: - verbose_proxy_logger.warning(f"Function {function_name} not found in module {module.__name__}") + verbose_proxy_logger.warning("Function %s not found in module %s", function_name, module.__name__) return False # Store original function if not already wrapped @@ -189,10 +189,10 @@ def wrap_function_with_line_profiler(module: Any, function_name: str) -> bool: profiled_function = _line_profiler(original_function) setattr(module, function_name, profiled_function) - verbose_proxy_logger.info(f"Wrapped {module.__name__}.{function_name} with line_profiler") + verbose_proxy_logger.info("Wrapped %s.%s with line_profiler", module.__name__, function_name) return True except Exception as e: - verbose_proxy_logger.error(f"Error wrapping {function_name} with line_profiler: {e}") + verbose_proxy_logger.error("Error wrapping %s with line_profiler: %s", function_name, e) return False @@ -226,7 +226,7 @@ def wrap_function_directly(func: Callable) -> Callable: _line_profiler.add_function(func) profiled_function = _line_profiler(func) - verbose_proxy_logger.info(f"Wrapped function {func.__name__} with line_profiler") + verbose_proxy_logger.info("Wrapped function %s with line_profiler", func.__name__) return profiled_function @@ -251,7 +251,7 @@ def collect_line_profiler_stats(output_file: str | None = None) -> None: # Save to file output_path = PathLib(output_file) _line_profiler.dump_stats(str(output_path)) - verbose_proxy_logger.info(f"Line profiler stats saved to {output_path}") + verbose_proxy_logger.info("Line profiler stats saved to %s", output_path) else: # Print to stdout from io import StringIO @@ -261,7 +261,7 @@ def collect_line_profiler_stats(output_file: str | None = None) -> None: stats_output = stream.getvalue() verbose_proxy_logger.info("Line profiler stats:\n" + stats_output) except Exception as e: - verbose_proxy_logger.error(f"Error collecting line profiler stats: {e}") + verbose_proxy_logger.error("Error collecting line profiler stats: %s", e) def register_shutdown_handler(output_file: str | None = None) -> None: @@ -282,4 +282,4 @@ def register_shutdown_handler(output_file: str | None = None) -> None: collect_line_profiler_stats(output_file=output_file) atexit.register(shutdown_handler) - verbose_proxy_logger.debug(f"Registered line_profiler shutdown handler for {output_file}") + verbose_proxy_logger.debug("Registered line_profiler shutdown handler for %s", output_file) diff --git a/litellm/proxy/custom_prompt_management.py b/litellm/proxy/custom_prompt_management.py index 355edb69897..ff95e58a3b4 100644 --- a/litellm/proxy/custom_prompt_management.py +++ b/litellm/proxy/custom_prompt_management.py @@ -27,7 +27,10 @@ class X42PromptManagement(CustomPromptManagement): - non_default_params: dict - update with any optional params (e.g. temperature, max_tokens, etc.) to use (can be pulled from prompt management tool) """ verbose_logger.debug( - f"in async get chat completion prompt. Prompt ID: {prompt_id}, Prompt Variables: {prompt_variables}, Dynamic Callback Params: {dynamic_callback_params}" + "in async get chat completion prompt. Prompt ID: %s, Prompt Variables: %s, Dynamic Callback Params: %s", + prompt_id, + prompt_variables, + dynamic_callback_params, ) return model, messages, non_default_params diff --git a/litellm/proxy/db/check_migration.py b/litellm/proxy/db/check_migration.py index 5e53e118e45..62333324c90 100644 --- a/litellm/proxy/db/check_migration.py +++ b/litellm/proxy/db/check_migration.py @@ -97,5 +97,6 @@ def check_prisma_schema_diff(db_url: str | None = None) -> None: has_diff, message = check_prisma_schema_diff_helper(db_url) if has_diff: verbose_logger.exception( - f"🚨🚨🚨 prisma schema out of sync with db. Consider running these sql_commands to sync the two - {message}" + "🚨🚨🚨 prisma schema out of sync with db. Consider running these sql_commands to sync the two - %s", + message, ) diff --git a/litellm/proxy/db/create_views.py b/litellm/proxy/db/create_views.py index 3ced1589757..27d978f5b8c 100644 --- a/litellm/proxy/db/create_views.py +++ b/litellm/proxy/db/create_views.py @@ -238,7 +238,7 @@ async def should_create_missing_views(db: _db) -> bool: result = await db.query_raw(query=sql_query) - verbose_logger.debug(f"Estimated Row count of LiteLLM_SpendLogs = {result}") + verbose_logger.debug("Estimated Row count of LiteLLM_SpendLogs = %s", result) if ( result and isinstance(result, list) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index b24ed4b8282..9d494bf9402 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -163,7 +163,11 @@ class DBSpendUpdateWriter: try: verbose_proxy_logger.debug( - f"Enters prisma db call, response_cost: {response_cost}, token: {token}; user_id: {user_id}; team_id: {team_id}" + "Enters prisma db call, response_cost: %s, token: %s; user_id: %s; team_id: %s", + response_cost, + token, + user_id, + team_id, ) if ProxyUpdateSpend.disable_spend_updates() is True: return @@ -728,7 +732,7 @@ class DBSpendUpdateWriter: if isinstance(request_tags, str): tags = safe_json_loads(request_tags, default=[]) if not tags: - verbose_proxy_logger.debug(f"Failed to parse request_tags JSON: {request_tags}") + verbose_proxy_logger.debug("Failed to parse request_tags JSON: %s", request_tags) return elif isinstance(request_tags, list): tags = request_tags @@ -1120,7 +1124,7 @@ class DBSpendUpdateWriter: ### UPDATE USER TABLE ### user_list_transactions = db_spend_update_transactions["user_list_transactions"] - verbose_proxy_logger.debug(f"User Spend transactions: {user_list_transactions}") + verbose_proxy_logger.debug("User Spend transactions: %s", user_list_transactions) if user_list_transactions is not None and len(user_list_transactions.keys()) > 0: for i in range(n_retry_times + 1): start_time = time.time() @@ -1152,7 +1156,7 @@ class DBSpendUpdateWriter: ### UPDATE END-USER TABLE ### end_user_list_transactions = db_spend_update_transactions["end_user_list_transactions"] - verbose_proxy_logger.debug(f"End-User Spend transactions: {end_user_list_transactions}") + verbose_proxy_logger.debug("End-User Spend transactions: %s", end_user_list_transactions) if end_user_list_transactions is not None and len(end_user_list_transactions.keys()) > 0: await ProxyUpdateSpend.update_end_user_spend( n_retry_times=n_retry_times, @@ -1162,7 +1166,7 @@ class DBSpendUpdateWriter: ) ### UPDATE KEY TABLE ### key_list_transactions = db_spend_update_transactions["key_list_transactions"] - verbose_proxy_logger.debug(f"KEY Spend transactions: {key_list_transactions}") + verbose_proxy_logger.debug("KEY Spend transactions: %s", key_list_transactions) if key_list_transactions is not None and len(key_list_transactions.keys()) > 0: for i in range(n_retry_times + 1): start_time = time.time() @@ -1195,7 +1199,7 @@ class DBSpendUpdateWriter: ### UPDATE TEAM TABLE ### team_list_transactions = db_spend_update_transactions["team_list_transactions"] - verbose_proxy_logger.debug(f"Team Spend transactions: {team_list_transactions}") + verbose_proxy_logger.debug("Team Spend transactions: %s", team_list_transactions) if team_list_transactions is not None and len(team_list_transactions.keys()) > 0: for i in range(n_retry_times + 1): start_time = time.time() @@ -1204,7 +1208,9 @@ class DBSpendUpdateWriter: async with transaction.batch_() as batcher: # Sort by team_id for consistent lock ordering across pods to prevent deadlocks. for team_id, response_cost in sorted(team_list_transactions.items()): - verbose_proxy_logger.debug(f"Updating spend for team id={team_id} by {response_cost}") + verbose_proxy_logger.debug( + "Updating spend for team id=%s by %s", team_id, response_cost + ) batcher.litellm_teamtable.update_many( # 'update_many' prevents error from being raised if no row exists where={"team_id": team_id}, data={"spend": {"increment": response_cost}}, @@ -1226,7 +1232,7 @@ class DBSpendUpdateWriter: ### UPDATE TEAM Membership TABLE with spend ### team_member_list_transactions = db_spend_update_transactions["team_member_list_transactions"] - verbose_proxy_logger.debug(f"Team Membership Spend transactions: {team_member_list_transactions}") + verbose_proxy_logger.debug("Team Membership Spend transactions: %s", team_member_list_transactions) if team_member_list_transactions is not None and len(team_member_list_transactions.keys()) > 0: # Track which team memberships will be updated for cache invalidation team_memberships_to_invalidate: list[tuple[str, str]] = [] @@ -1280,12 +1286,12 @@ class DBSpendUpdateWriter: cache_key = f"team_membership:{user_id}:{team_id}" await user_api_key_cache.async_delete_cache(key=cache_key) verbose_proxy_logger.debug( - f"Invalidated team membership cache for user_id={user_id}, team_id={team_id}" + "Invalidated team membership cache for user_id=%s, team_id=%s", user_id, team_id ) ### UPDATE ORG TABLE ### org_list_transactions = db_spend_update_transactions["org_list_transactions"] - verbose_proxy_logger.debug(f"Org Spend transactions: {org_list_transactions}") + verbose_proxy_logger.debug("Org Spend transactions: %s", org_list_transactions) if org_list_transactions is not None and len(org_list_transactions.keys()) > 0: for i in range(n_retry_times + 1): start_time = time.time() @@ -1368,7 +1374,7 @@ class DBSpendUpdateWriter: """ from litellm.proxy.utils import _raise_failed_update_spend_exception - verbose_proxy_logger.debug(f"{entity_name} Spend transactions: {transactions}") + verbose_proxy_logger.debug("%s Spend transactions: %s", entity_name, transactions) if transactions is not None and len(transactions.keys()) > 0: for i in range(n_retry_times + 1): start_time = time.time() @@ -1378,7 +1384,11 @@ class DBSpendUpdateWriter: # Sort by entity_id for consistent lock ordering across pods to prevent deadlocks. for entity_id, response_cost in sorted(transactions.items()): verbose_proxy_logger.debug( - f"Updating spend for {entity_name} {where_field}={entity_id} by {response_cost}" + "Updating spend for %s %s=%s by %s", + entity_name, + where_field, + entity_id, + response_cost, ) getattr(batcher, table_accessor).update_many( where={where_field: entity_id}, @@ -1507,7 +1517,7 @@ class DBSpendUpdateWriter: from litellm.proxy.utils import _raise_failed_update_spend_exception verbose_proxy_logger.debug( - f"Daily {entity_type.capitalize()} Spend transactions: {len(daily_spend_transactions)}" + "Daily %s Spend transactions: %s", entity_type.capitalize(), len(daily_spend_transactions) ) BATCH_SIZE = 100 start_time = time.time() @@ -1541,7 +1551,7 @@ class DBSpendUpdateWriter: if len(transactions_to_process) == 0: verbose_proxy_logger.debug( - f"No new transactions to process for daily {entity_type} spend update" + "No new transactions to process for daily %s spend update", entity_type ) return @@ -1837,14 +1847,15 @@ class DBSpendUpdateWriter: raise ValueError(f"Invalid type: {type}") if not all(key in payload for key in expected_keys): verbose_proxy_logger.debug( - f"Missing expected keys: {expected_keys}, in payload, skipping from daily_user_spend_transactions" + "Missing expected keys: %s, in payload, skipping from daily_user_spend_transactions", expected_keys ) return None any_expected_keys = ["model", "mcp_namespaced_tool_name"] if not any(key in payload for key in any_expected_keys): verbose_proxy_logger.debug( - f"Missing any expected keys: {any_expected_keys}, in payload, skipping from daily_user_spend_transactions" + "Missing any expected keys: %s, in payload, skipping from daily_user_spend_transactions", + any_expected_keys, ) return None elif "mcp_namespaced_tool_name" in payload: @@ -1856,7 +1867,7 @@ class DBSpendUpdateWriter: return None request_status = prisma_client.get_request_status(payload) - verbose_proxy_logger.debug(f"Logged request status: {request_status}") + verbose_proxy_logger.debug("Logged request status: %s", request_status) _metadata: SpendLogsMetadata = json.loads(payload["metadata"]) usage_obj = _metadata.get("usage_object", {}) or {} if isinstance(payload["startTime"], datetime): @@ -1866,7 +1877,7 @@ class DBSpendUpdateWriter: date = payload["startTime"].split("T")[0] else: verbose_proxy_logger.debug( - f"Invalid start time: {payload['startTime']}, skipping from daily_user_spend_transactions" + "Invalid start time: %s, skipping from daily_user_spend_transactions", payload["startTime"] ) return None try: diff --git a/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py b/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py index 01f6a92485a..604cde4a88c 100644 --- a/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py +++ b/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py @@ -106,7 +106,7 @@ end ) return False except Exception as e: - verbose_proxy_logger.error(f"Error acquiring Redis lock for {cronjob_id}: {e}") + verbose_proxy_logger.error("Error acquiring Redis lock for %s: %s", cronjob_id, e) return False async def release_lock( @@ -148,7 +148,7 @@ end cronjob_id, ) except Exception as e: - verbose_proxy_logger.error(f"Error releasing Redis lock for {cronjob_id}: {e}") + verbose_proxy_logger.error("Error releasing Redis lock for %s: %s", cronjob_id, e) async def _compare_and_delete_lock(self, lock_key: str) -> int: """ diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py index d7c70bdb20c..6e5d83bc500 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py +++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py @@ -43,14 +43,14 @@ class SpendLogCleanup: pod_lock_manager = proxy_logging_obj.db_spend_update_writer.pod_lock_manager self.pod_lock_manager = pod_lock_manager - verbose_proxy_logger.info(f"SpendLogCleanup initialized with batch size: {self.batch_size}") + verbose_proxy_logger.info("SpendLogCleanup initialized with batch size: %s", self.batch_size) def _should_delete_spend_logs(self) -> bool: """ Determines if logs should be deleted based on the max retention period in settings. """ retention_setting = self.general_settings.get("maximum_spend_logs_retention_period") - verbose_proxy_logger.info(f"Checking retention setting: {retention_setting}") + verbose_proxy_logger.info("Checking retention setting: %s", retention_setting) if retention_setting is None: verbose_proxy_logger.info("No retention setting found") @@ -59,16 +59,16 @@ class SpendLogCleanup: try: if isinstance(retention_setting, int): verbose_proxy_logger.warning( - f"maximum_spend_logs_retention_period is an integer ({retention_setting}); treating as days. " - "Use a string like '3d' to be explicit." + "maximum_spend_logs_retention_period is an integer (%s); treating as days. Use a string like '3d' to be explicit.", + retention_setting, ) retention_setting = f"{retention_setting}d" self.retention_seconds = duration_in_seconds(retention_setting) - verbose_proxy_logger.info(f"Retention period set to {self.retention_seconds} seconds") + verbose_proxy_logger.info("Retention period set to %s seconds", self.retention_seconds) return True except ValueError as e: verbose_proxy_logger.warning( - f"Invalid maximum_spend_logs_retention_period value: {retention_setting}, error: {e}" + "Invalid maximum_spend_logs_retention_period value: %s, error: %s", retention_setting, e ) return False @@ -145,15 +145,16 @@ class SpendLogCleanup: deleted_count = deleted_result else: verbose_proxy_logger.error( - f"Unexpected execute_raw return type for {table_name} cleanup: {type(deleted_result)}; " - "aborting cleanup to avoid infinite loop" + "Unexpected execute_raw return type for %s cleanup: %s; aborting cleanup to avoid infinite loop", + table_name, + type(deleted_result), ) break - verbose_proxy_logger.info(f"Deleted {deleted_count} {table_name} rows in this batch") + verbose_proxy_logger.info("Deleted %s %s rows in this batch", deleted_count, table_name) if deleted_count == 0: - verbose_proxy_logger.info(f"No more {table_name} rows to delete. Total deleted: {total_deleted}") + verbose_proxy_logger.info("No more %s rows to delete. Total deleted: %s", table_name, total_deleted) break total_deleted += deleted_count @@ -192,7 +193,7 @@ class SpendLogCleanup: """ lock_acquired = False try: - verbose_proxy_logger.info(f"Cleanup job triggered at {datetime.now()}") + verbose_proxy_logger.info("Cleanup job triggered at %s", datetime.now()) if not self._should_delete_spend_logs(): return @@ -210,7 +211,7 @@ class SpendLogCleanup: or False ) verbose_proxy_logger.info( - f"Lock acquisition attempt: {'successful' if lock_acquired else 'failed'} at {datetime.now()}" + "Lock acquisition attempt: %s at %s", "successful" if lock_acquired else "failed", datetime.now() ) if not lock_acquired: @@ -218,7 +219,7 @@ class SpendLogCleanup: return cutoff_date = datetime.now(timezone.utc) - timedelta(seconds=float(self.retention_seconds)) - verbose_proxy_logger.info(f"Removing logs older than {cutoff_date.isoformat()}") + verbose_proxy_logger.info("Removing logs older than %s", cutoff_date.isoformat()) if self.general_settings.get( "use_spend_logs_partitioning", False @@ -235,13 +236,13 @@ class SpendLogCleanup: # or in a partition that spans the cutoff, so retention must # also delete those stragglers row-wise. total_deleted = await self._delete_old_logs(prisma_client, cutoff_date) - verbose_proxy_logger.info(f"Deleted {total_deleted} expired logs not covered by dropped partitions") + verbose_proxy_logger.info("Deleted %s expired logs not covered by dropped partitions", total_deleted) else: total_deleted = await self._delete_old_logs(prisma_client, cutoff_date) - verbose_proxy_logger.info(f"Deleted {total_deleted} logs") + verbose_proxy_logger.info("Deleted %s logs", total_deleted) index_deleted = await self._delete_old_tool_index_rows(prisma_client, cutoff_date) - verbose_proxy_logger.info(f"Deleted {index_deleted} expired tool index rows") + verbose_proxy_logger.info("Deleted %s expired tool index rows", index_deleted) except Exception as e: # .exception() captures the traceback; str(e) alone on a Prisma/DB diff --git a/litellm/proxy/db/dynamo_db.py b/litellm/proxy/db/dynamo_db.py index 6367c34341f..805d17b36ac 100644 --- a/litellm/proxy/db/dynamo_db.py +++ b/litellm/proxy/db/dynamo_db.py @@ -39,7 +39,7 @@ class DynamoDBWrapper(CustomDB): def set_env_vars_based_on_arn(self): if self.database_arguments.aws_role_name is None: return - verbose_proxy_logger.debug(f"DynamoDB: setting env vars based on arn={self.database_arguments.aws_role_name}") + verbose_proxy_logger.debug("DynamoDB: setting env vars based on arn=%s", self.database_arguments.aws_role_name) import os import boto3 @@ -63,7 +63,7 @@ class DynamoDBWrapper(CustomDB): aws_secret_access_key = assumed_role["Credentials"]["SecretAccessKey"] aws_session_token = assumed_role["Credentials"]["SessionToken"] - verbose_proxy_logger.debug(f"Got STS assumed Role, aws_access_key_id={aws_access_key_id}") + verbose_proxy_logger.debug("Got STS assumed Role, aws_access_key_id=%s", aws_access_key_id) # set these in the env so aiodynamo can use them os.environ["AWS_ACCESS_KEY_ID"] = aws_access_key_id os.environ["AWS_SECRET_ACCESS_KEY"] = aws_secret_access_key diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index 79f86d548c5..f12f29a03d0 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -398,7 +398,7 @@ class PrismaWrapper: return token_created + timedelta(seconds=expires_in) except Exception as e: - verbose_proxy_logger.debug(f"Failed to parse token expiration: {e}") + verbose_proxy_logger.debug("Failed to parse token expiration: %s", e) return None def _calculate_seconds_until_refresh(self) -> float: @@ -420,8 +420,8 @@ class PrismaWrapper: if expiration_time is None: # If we can't parse the token, use fallback interval verbose_proxy_logger.debug( - f"Could not parse token expiration, using fallback interval of " - f"{self.FALLBACK_REFRESH_INTERVAL_SECONDS}s" + "Could not parse token expiration, using fallback interval of %ss", + self.FALLBACK_REFRESH_INTERVAL_SECONDS, ) return self.FALLBACK_REFRESH_INTERVAL_SECONDS @@ -670,8 +670,9 @@ class PrismaWrapper: This is more efficient than polling, requiring only 1 wake-up per token cycle. """ verbose_proxy_logger.info( - f"{self._log_prefix}RDS IAM token refresh loop started. " - f"Tokens will be refreshed {self.TOKEN_REFRESH_BUFFER_SECONDS}s before expiration." + "%sRDS IAM token refresh loop started. Tokens will be refreshed %ss before expiration.", + self._log_prefix, + self.TOKEN_REFRESH_BUFFER_SECONDS, ) while True: @@ -695,8 +696,10 @@ class PrismaWrapper: break except Exception as e: verbose_proxy_logger.error( - f"{self._log_prefix}Error in RDS IAM token refresh loop: {e}. " - f"Retrying in {self.FALLBACK_REFRESH_INTERVAL_SECONDS}s..." + "%sError in RDS IAM token refresh loop: %s. Retrying in %ss...", + self._log_prefix, + e, + self.FALLBACK_REFRESH_INTERVAL_SECONDS, ) # On error, wait before retrying to avoid tight error loops try: @@ -874,7 +877,7 @@ class PrismaManager: try: from litellm_proxy_extras.utils import ProxyExtrasDBManager except ImportError as e: - verbose_proxy_logger.error(f"\033[1;31mLiteLLM: Failed to import proxy extras. Got {e}\033[0m") + verbose_proxy_logger.error("\x1b[1;31mLiteLLM: Failed to import proxy extras. Got %s\x1b[0m", e) return False prisma_dir = PrismaManager._get_prisma_dir() @@ -899,12 +902,12 @@ class PrismaManager: PrismaManager._apply_replica_identity_full_if_requested() return True except subprocess.TimeoutExpired: - verbose_proxy_logger.warning(f"Attempt {attempt + 1} timed out") + verbose_proxy_logger.warning("Attempt %s timed out", attempt + 1) time.sleep(random.randrange(5, 15)) except subprocess.CalledProcessError as e: attempts_left = 3 - attempt retry_msg = f" Retrying... ({attempts_left} attempts left)" if attempts_left > 0 else "" - verbose_proxy_logger.warning(f"The process failed to execute. Details: {e}.{retry_msg}") + verbose_proxy_logger.warning("The process failed to execute. Details: %s.%s", e, retry_msg) time.sleep(random.randrange(5, 15)) finally: os.chdir(original_dir) diff --git a/litellm/proxy/example_config_yaml/custom_guardrail.py b/litellm/proxy/example_config_yaml/custom_guardrail.py index a755390743e..d30ed839ce4 100644 --- a/litellm/proxy/example_config_yaml/custom_guardrail.py +++ b/litellm/proxy/example_config_yaml/custom_guardrail.py @@ -23,7 +23,7 @@ class GuardrailForLBTestingA(CustomGuardrail): call_type: CallTypesLiteral, ) -> Optional[Union[Exception, str, dict]]: guardrail_lb_call_count["A"] += 1 - verbose_proxy_logger.info(f"GuardrailForLBTestingA called. Total A calls: {guardrail_lb_call_count['A']}") + verbose_proxy_logger.info("GuardrailForLBTestingA called. Total A calls: %s", guardrail_lb_call_count["A"]) return data @@ -38,7 +38,7 @@ class GuardrailForLBTestingB(CustomGuardrail): call_type: CallTypesLiteral, ) -> Optional[Union[Exception, str, dict]]: guardrail_lb_call_count["B"] += 1 - verbose_proxy_logger.info(f"GuardrailForLBTestingB called. Total B calls: {guardrail_lb_call_count['B']}") + verbose_proxy_logger.info("GuardrailForLBTestingB called. Total B calls: %s", guardrail_lb_call_count["B"]) return data diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py index 4daab1caf96..3cdd57490c6 100644 --- a/litellm/proxy/fine_tuning_endpoints/endpoints.py +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -112,7 +112,8 @@ async def create_fine_tuning_job( # Convert Pydantic model to dict verbose_proxy_logger.debug( - f"Request received by LiteLLM:\n{json.dumps(data, indent=4)}", + "Request received by LiteLLM:\n%s", + json.dumps(data, indent=4), ) # Include original request and headers in the data @@ -199,7 +200,9 @@ async def create_fine_tuning_job( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.create_fine_tuning_job(): Exception occurred - {e}") + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.create_fine_tuning_job(): Exception occurred - %s", e + ) raise handle_exception_on_proxy(e) @@ -338,7 +341,7 @@ async def retrieve_fine_tuning_job( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.retrieve_fine_tuning_job(): Exception occurred - {e}" + "litellm.proxy.proxy_server.retrieve_fine_tuning_job(): Exception occurred - %s", e ) raise handle_exception_on_proxy(e) @@ -466,7 +469,7 @@ async def list_fine_tuning_jobs( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.list_fine_tuning_jobs(): Exception occurred - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.list_fine_tuning_jobs(): Exception occurred - %s", e) raise handle_exception_on_proxy(e) @@ -604,5 +607,7 @@ async def cancel_fine_tuning_job( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.cancel_fine_tuning_job(): Exception occurred - {e}") + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.cancel_fine_tuning_job(): Exception occurred - %s", e + ) raise handle_exception_on_proxy(e) diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index 12373b7fb97..baf44e5a291 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -313,7 +313,7 @@ async def list_guardrails_v2( return ListGuardrailsResponse(guardrails=guardrail_configs) except Exception as e: - verbose_proxy_logger.exception(f"Error getting guardrails from db: {e}") + verbose_proxy_logger.exception("Error getting guardrails from db: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -397,7 +397,7 @@ async def create_guardrail( try: IN_MEMORY_GUARDRAIL_HANDLER.initialize_guardrail(guardrail=cast(Guardrail, result), source="db") verbose_proxy_logger.info( - f"Immediate sync: Successfully initialized guardrail '{guardrail_name}' (ID: {guardrail_id})" + "Immediate sync: Successfully initialized guardrail '%s' (ID: %s)", guardrail_name, guardrail_id ) except (ValueError, TypeError) as init_error: # Configuration error — roll back the DB write so the guardrail isn't orphaned @@ -405,19 +405,22 @@ async def create_guardrail( try: await _delete_guardrail_row(prisma_client, where={"guardrail_id": guardrail_id}) except Exception as rollback_err: - verbose_proxy_logger.warning(f"Rollback failed for guardrail '{guardrail_id}': {rollback_err}") + verbose_proxy_logger.warning("Rollback failed for guardrail '%s': %s", guardrail_id, rollback_err) raise HTTPException( status_code=400, detail=f"Guardrail configuration error: {init_error}", ) except Exception as init_error: verbose_proxy_logger.warning( - f"Immediate sync: Failed to initialize guardrail '{guardrail_name}' (ID: {guardrail_id}) in memory: {init_error}" + "Immediate sync: Failed to initialize guardrail '%s' (ID: %s) in memory: %s", + guardrail_name, + guardrail_id, + init_error, ) return result except Exception as e: - verbose_proxy_logger.exception(f"Error adding guardrail to db: {e}") + verbose_proxy_logger.exception("Error adding guardrail to db: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -515,11 +518,14 @@ async def update_guardrail( guardrail_id=guardrail_id, guardrail=cast(Guardrail, result) ) verbose_proxy_logger.info( - f"Immediate sync: Successfully updated guardrail '{guardrail_name}' (ID: {guardrail_id})" + "Immediate sync: Successfully updated guardrail '%s' (ID: %s)", guardrail_name, guardrail_id ) except Exception as update_error: verbose_proxy_logger.warning( - f"Immediate sync: Failed to update '{guardrail_name}' (ID: {guardrail_id}) in memory: {update_error}" + "Immediate sync: Failed to update '%s' (ID: %s) in memory: %s", + guardrail_name, + guardrail_id, + update_error, ) return result @@ -587,11 +593,14 @@ async def delete_guardrail( guardrail_id=guardrail_id, ) verbose_proxy_logger.info( - f"Immediate sync: Successfully removed guardrail '{guardrail_name}' (ID: {guardrail_id}) from memory" + "Immediate sync: Successfully removed guardrail '%s' (ID: %s) from memory", guardrail_name, guardrail_id ) except Exception as delete_error: verbose_proxy_logger.warning( - f"Immediate sync: Failed to remove guardrail '{guardrail_name}' (ID: {guardrail_id}) from memory: {delete_error}" + "Immediate sync: Failed to remove guardrail '%s' (ID: %s) from memory: %s", + guardrail_name, + guardrail_id, + delete_error, ) return result @@ -1203,18 +1212,21 @@ async def patch_guardrail( guardrail=guardrail, ) verbose_proxy_logger.info( - f"Immediate sync: Successfully updated guardrail '{guardrail_name}' (ID: {guardrail_id})" + "Immediate sync: Successfully updated guardrail '%s' (ID: %s)", guardrail_name, guardrail_id ) except Exception as update_error: verbose_proxy_logger.warning( - f"Immediate sync: Failed to update '{guardrail_name}' (ID: {guardrail_id}) in memory: {update_error}" + "Immediate sync: Failed to update '%s' (ID: %s) in memory: %s", + guardrail_name, + guardrail_id, + update_error, ) return result except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.exception(f"Error updating guardrail: {e}") + verbose_proxy_logger.exception("Error updating guardrail: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -2126,7 +2138,7 @@ async def test_custom_code_guardrail( ) except Exception as e: - verbose_proxy_logger.exception(f"Error testing custom code guardrail: {e}") + verbose_proxy_logger.exception("Error testing custom code guardrail: %s", e) return TestCustomCodeGuardrailResponse( success=False, error=f"Unexpected error: {e}", diff --git a/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py b/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py index 076cdfc8ecd..d9bcef731bb 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py +++ b/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py @@ -123,7 +123,7 @@ class AimGuardrail(CustomGuardrail): elif action_type == "anonymize_action": return self._anonymize_request(res, data) else: - verbose_proxy_logger.error(f"Aim: {action_type} action") + verbose_proxy_logger.error("Aim: %s action", action_type) return data @staticmethod @@ -328,7 +328,7 @@ class AimGuardrail(CustomGuardrail): from litellm.proxy.proxy_server import StreamingCallbackError raise StreamingCallbackError(blocking_message) - verbose_proxy_logger.error(f"Unknown message received from AIM: {result}") + verbose_proxy_logger.error("Unknown message received from AIM: %s", result) return async def forward_the_stream_to_aim( diff --git a/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py b/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py index 3df4f230dbf..95fe649a565 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py +++ b/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py @@ -58,7 +58,7 @@ class AzureContentSafetyPromptShieldGuardrail(AzureGuardrailBase, CustomGuardrai **kwargs, ) - verbose_proxy_logger.debug(f"Initialized Azure Prompt Shield Guardrail: {guardrail_name}") + verbose_proxy_logger.debug("Initialized Azure Prompt Shield Guardrail: %s", guardrail_name) async def async_make_request(self, user_prompt: str) -> "AzurePromptShieldGuardrailResponse": """ @@ -127,7 +127,7 @@ class AzureContentSafetyPromptShieldGuardrail(AzureGuardrailBase, CustomGuardrai user_prompt = self.get_user_prompt(new_messages) if user_prompt: - verbose_proxy_logger.debug(f"Azure Prompt Shield: User prompt: {user_prompt}") + verbose_proxy_logger.debug("Azure Prompt Shield: User prompt: %s", user_prompt) await self.async_make_request( user_prompt=user_prompt, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/azure/text_moderation.py b/litellm/proxy/guardrails/guardrail_hooks/azure/text_moderation.py index 0b1faf99469..5355631def6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/azure/text_moderation.py +++ b/litellm/proxy/guardrails/guardrail_hooks/azure/text_moderation.py @@ -90,7 +90,7 @@ class AzureContentSafetyTextModerationGuardrail(AzureGuardrailBase, CustomGuardr self.severity_threshold = int(severity_threshold) if severity_threshold else None self.severity_threshold_by_category = severity_threshold_by_category - verbose_proxy_logger.info(f"Initialized Azure Text Moderation Guardrail: {guardrail_name}") + verbose_proxy_logger.info("Initialized Azure Text Moderation Guardrail: %s", guardrail_name) @staticmethod def get_config_model() -> type["GuardrailConfigModel"] | None: @@ -223,7 +223,7 @@ class AzureContentSafetyTextModerationGuardrail(AzureGuardrailBase, CustomGuardr user_prompt = self.get_user_prompt(new_messages) if user_prompt: - verbose_proxy_logger.info(f"Azure Text Moderation: User prompt: {user_prompt}") + verbose_proxy_logger.info("Azure Text Moderation: User prompt: %s", user_prompt) await self.async_make_request( text=user_prompt, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 5aae14b83e6..dab9a3d47f5 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -2157,7 +2157,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): # dict.get("texts", []) would return None if the key exists with a None value. texts = inputs.get("texts") or [] try: - verbose_proxy_logger.debug(f"Bedrock Guardrail: Applying guardrail to {len(texts)} text(s)") + verbose_proxy_logger.debug("Bedrock Guardrail: Applying guardrail to %s text(s)", len(texts)) if input_type == "request": incremental_result = await self._apply_incremental_request_scan( diff --git a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py index e0411de2db4..661e1d4c749 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py +++ b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py @@ -264,7 +264,7 @@ class CatoNetworksGuardrail(CustomGuardrail): elif action_type == "anonymize_action": return self._anonymize_request(res, data) else: - verbose_proxy_logger.error(f"Cato: {action_type} action") + verbose_proxy_logger.error("Cato: %s action", action_type) return data def _handle_block_action(self, analysis_result: Any, required_action: Any) -> None: @@ -555,7 +555,7 @@ class CatoNetworksGuardrail(CustomGuardrail): return if blocking_message := result.get("blocking_message"): raise StreamingCallbackError(blocking_message) - verbose_proxy_logger.error(f"Unknown message received from Cato: {result}") + verbose_proxy_logger.error("Unknown message received from Cato: %s", result) return finally: await self._cancel_background_task(sender) diff --git a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py index 600da6ecfc4..b93a1f99a3a 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py @@ -275,7 +275,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): # Pass relevant kwargs to the parent class super().__init__(guardrail_name=guardrail_name, **kwargs) verbose_proxy_logger.debug( - f"Initialized CrowdStrike AIDR Guardrail: name={guardrail_name}, api_base={self.api_base}" + "Initialized CrowdStrike AIDR Guardrail: name=%s, api_base=%s", guardrail_name, self.api_base ) async def _call_crowdstrike_aidr_guard( @@ -306,7 +306,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): } verbose_proxy_logger.debug( - f"CrowdStrike AIDR Guardrail ({hook_name}): Calling endpoint {endpoint} with payload: {payload}" + "CrowdStrike AIDR Guardrail (%s): Calling endpoint %s with payload: %s", hook_name, endpoint, payload ) response = await self.async_handler.post(url=endpoint, json=payload, headers=headers) @@ -317,7 +317,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): if result.blocked: verbose_proxy_logger.warning( - f"CrowdStrike AIDR Guardrail ({hook_name}): Request blocked. Response: {result}" + "CrowdStrike AIDR Guardrail (%s): Request blocked. Response: %s", hook_name, result ) raise HTTPException( status_code=400, # Bad Request, indicating violation @@ -327,7 +327,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): }, ) verbose_proxy_logger.debug( - f"CrowdStrike AIDR Guardrail ({hook_name}): Request passed. Response: {result.detectors}" + "CrowdStrike AIDR Guardrail (%s): Request passed. Response: %s", hook_name, result.detectors ) return result @@ -396,7 +396,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): input_type: Literal["request", "response"], logging_obj: Optional["LiteLLMLoggingObj"] = None, ) -> GenericGuardrailAPIInputs: - verbose_proxy_logger.debug(f"CrowdStrike AIDR Guardrail: Applying guardrail to {input_type}") + verbose_proxy_logger.debug("CrowdStrike AIDR Guardrail: Applying guardrail to %s", input_type) # Extract inputs texts = inputs.get("texts", []) diff --git a/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py index c7a036562f7..8b7c231b690 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py @@ -176,7 +176,7 @@ class CustomCodeGuardrail(CustomGuardrail): try: self._do_compile() - verbose_proxy_logger.debug(f"Custom code guardrail '{self.guardrail_name}' compiled successfully") + verbose_proxy_logger.debug("Custom code guardrail '%s' compiled successfully", self.guardrail_name) except SyntaxError as e: self._compile_error = f"Syntax error in custom code: {e}" @@ -254,7 +254,7 @@ class CustomCodeGuardrail(CustomGuardrail): # Pre-call block uses passthrough; must not wrap as execution error (500) raise except Exception as e: - verbose_proxy_logger.error(f"Custom code guardrail '{self.guardrail_name}' execution error: {e}") + verbose_proxy_logger.error("Custom code guardrail '%s' execution error: %s", self.guardrail_name, e) raise CustomCodeExecutionError( f"Custom code guardrail execution failed: {e}", details={ @@ -308,15 +308,16 @@ class CustomCodeGuardrail(CustomGuardrail): """ if not isinstance(result, dict): verbose_proxy_logger.warning( - f"Custom code guardrail '{self.guardrail_name}': " - f"Expected dict result, got {type(result).__name__}. Treating as allow." + "Custom code guardrail '%s': Expected dict result, got %s. Treating as allow.", + self.guardrail_name, + type(result).__name__, ) return inputs action = result.get("action", "allow") if action == "allow": - verbose_proxy_logger.debug(f"Custom code guardrail '{self.guardrail_name}': Allowing {input_type}") + verbose_proxy_logger.debug("Custom code guardrail '%s': Allowing %s", self.guardrail_name, input_type) return inputs elif action == "block": @@ -324,7 +325,7 @@ class CustomCodeGuardrail(CustomGuardrail): detection_info = result.get("detection_info", {}) verbose_proxy_logger.info( - f"Custom code guardrail '{self.guardrail_name}': Blocking {input_type} - {reason}" + "Custom code guardrail '%s': Blocking %s - %s", self.guardrail_name, input_type, reason ) is_output = input_type == "response" @@ -348,7 +349,7 @@ class CustomCodeGuardrail(CustomGuardrail): ) elif action == "modify": - verbose_proxy_logger.debug(f"Custom code guardrail '{self.guardrail_name}': Modifying {input_type}") + verbose_proxy_logger.debug("Custom code guardrail '%s': Modifying %s", self.guardrail_name, input_type) # Apply modifications modified_inputs = dict(inputs) @@ -366,7 +367,7 @@ class CustomCodeGuardrail(CustomGuardrail): else: verbose_proxy_logger.warning( - f"Custom code guardrail '{self.guardrail_name}': Unknown action '{action}'. Treating as allow." + "Custom code guardrail '%s': Unknown action '%s'. Treating as allow.", self.guardrail_name, action ) return inputs @@ -393,7 +394,7 @@ class CustomCodeGuardrail(CustomGuardrail): try: self.custom_code = new_code self._do_compile() - verbose_proxy_logger.info(f"Custom code guardrail '{self.guardrail_name}': Code updated successfully") + verbose_proxy_logger.info("Custom code guardrail '%s': Code updated successfully", self.guardrail_name) except SyntaxError as e: # Rollback on failure self.custom_code = old_code diff --git a/litellm/proxy/guardrails/guardrail_hooks/custom_code/primitives.py b/litellm/proxy/guardrails/guardrail_hooks/custom_code/primitives.py index 43a3671ad97..f77bd324462 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/custom_code/primitives.py +++ b/litellm/proxy/guardrails/guardrail_hooks/custom_code/primitives.py @@ -94,7 +94,7 @@ def regex_match(text: str, pattern: str, flags: int = 0) -> bool: try: return bool(re.search(pattern, text, flags)) except re.error as e: - verbose_proxy_logger.warning(f"Starlark regex_match error: {e}") + verbose_proxy_logger.warning("Starlark regex_match error: %s", e) return False @@ -113,7 +113,7 @@ def regex_match_all(text: str, pattern: str, flags: int = 0) -> bool: try: return bool(re.fullmatch(pattern, text, flags)) except re.error as e: - verbose_proxy_logger.warning(f"Starlark regex_match_all error: {e}") + verbose_proxy_logger.warning("Starlark regex_match_all error: %s", e) return False @@ -133,7 +133,7 @@ def regex_replace(text: str, pattern: str, replacement: str, flags: int = 0) -> try: return re.sub(pattern, replacement, text, flags=flags) except re.error as e: - verbose_proxy_logger.warning(f"Starlark regex_replace error: {e}") + verbose_proxy_logger.warning("Starlark regex_replace error: %s", e) return text @@ -152,7 +152,7 @@ def regex_find_all(text: str, pattern: str, flags: int = 0) -> list[str]: try: return re.findall(pattern, text, flags) except re.error as e: - verbose_proxy_logger.warning(f"Starlark regex_find_all error: {e}") + verbose_proxy_logger.warning("Starlark regex_find_all error: %s", e) return [] @@ -174,7 +174,7 @@ def json_parse(text: str) -> Any | None: try: return json.loads(text) except (json.JSONDecodeError, TypeError) as e: - verbose_proxy_logger.debug(f"Starlark json_parse error: {e}") + verbose_proxy_logger.debug("Starlark json_parse error: %s", e) return None @@ -191,7 +191,7 @@ def json_stringify(obj: Any) -> str: try: return json.dumps(obj) except (TypeError, ValueError) as e: - verbose_proxy_logger.warning(f"Starlark json_stringify error: {e}") + verbose_proxy_logger.warning("Starlark json_stringify error: %s", e) return "" @@ -222,7 +222,7 @@ def json_schema_valid(obj: Any, schema: dict[str, Any]) -> bool: return False raise except Exception as e: - verbose_proxy_logger.warning(f"Custom code json_schema_valid error: {e}") + verbose_proxy_logger.warning("Custom code json_schema_valid error: %s", e) return False @@ -473,16 +473,16 @@ async def http_request( return _http_success_response(response) except httpx.TimeoutException as e: - verbose_proxy_logger.warning(f"Custom code http_request timeout: {e}") + verbose_proxy_logger.warning("Custom code http_request timeout: %s", e) return _http_error_response(f"Request timeout after {timeout}s") except httpx.HTTPStatusError as e: # Return the response even for non-2xx status codes return _http_success_response(e.response) except httpx.RequestError as e: - verbose_proxy_logger.warning(f"Custom code http_request error: {e}") + verbose_proxy_logger.warning("Custom code http_request error: %s", e) return _http_error_response(f"Request failed: {e}") except Exception as e: - verbose_proxy_logger.warning(f"Custom code http_request unexpected error: {e}") + verbose_proxy_logger.warning("Custom code http_request unexpected error: %s", e) return _http_error_response(f"Unexpected error: {e}") diff --git a/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py b/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py index 73306ce1154..26f61418cae 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py +++ b/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py @@ -237,7 +237,7 @@ class HiddenlayerGuardrail(CustomGuardrail): response.raise_for_status() result = response.json() - verbose_proxy_logger.debug(f"Hiddenlayer reponse: {result}") + verbose_proxy_logger.debug("Hiddenlayer reponse: %s", result) return result except HTTPStatusError as e: @@ -261,7 +261,7 @@ class HiddenlayerGuardrail(CustomGuardrail): response.raise_for_status() result = response.json() - verbose_proxy_logger.debug(f"Hiddenlayer reponse: {result}") + verbose_proxy_logger.debug("Hiddenlayer reponse: %s", result) return result @staticmethod @@ -434,7 +434,7 @@ class HiddenlayerGuardrailV2(CustomGuardrail): ) response.raise_for_status() - verbose_proxy_logger.debug(f"Hiddenlayer reponse: {response}") + verbose_proxy_logger.debug("Hiddenlayer reponse: %s", response) return response except HTTPStatusError as e: @@ -457,7 +457,7 @@ class HiddenlayerGuardrailV2(CustomGuardrail): response.raise_for_status() - verbose_proxy_logger.debug(f"Hiddenlayer reponse: {response}") + verbose_proxy_logger.debug("Hiddenlayer reponse: %s", response) return response @staticmethod diff --git a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py index 90d131893c6..6b713e4d519 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py +++ b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py @@ -116,8 +116,10 @@ class LassoGuardrail(CustomGuardrail): self.api_base = api_base or os.getenv("LASSO_API_BASE") or "https://server.lasso.security/gateway/v3" verbose_proxy_logger.debug( - f"Lasso guardrail initialized: {kwargs.get('guardrail_name', 'unknown')}, " - f"event_hook: {kwargs.get('event_hook', 'unknown')}, mask: {self.mask}" + "Lasso guardrail initialized: %s, event_hook: %s, mask: %s", + kwargs.get("guardrail_name", "unknown"), + kwargs.get("event_hook", "unknown"), + self.mask, ) super().__init__(**kwargs) @@ -299,7 +301,7 @@ class LassoGuardrail(CustomGuardrail): except Exception as e: if isinstance(e, HTTPException): raise e - verbose_proxy_logger.error(f"Error in post-call Lasso masking: {e}") + verbose_proxy_logger.error("Error in post-call Lasso masking: %s", e) raise LassoGuardrailAPIError(f"Failed to apply post-call masking: {e}") else: # Use the same data for conversation_id consistency (no cache access needed) @@ -308,7 +310,7 @@ class LassoGuardrail(CustomGuardrail): else: verbose_proxy_logger.warning("No response messages found to validate") else: - verbose_proxy_logger.warning(f"Unexpected response type for post-call hook: {type(response)}") + verbose_proxy_logger.warning("Unexpected response type for post-call hook: %s", type(response)) return response @@ -353,7 +355,7 @@ class LassoGuardrail(CustomGuardrail): if cached_conversation_id: return cached_conversation_id except Exception as e: - verbose_proxy_logger.warning(f"Cache retrieval failed: {e}") + verbose_proxy_logger.warning("Cache retrieval failed: %s", e) # Generate new conversation_id and store in cache generated_id = self._generate_ulid() @@ -361,7 +363,7 @@ class LassoGuardrail(CustomGuardrail): try: cache.set_cache(cache_key, generated_id, ttl=3600) # Cache for 1 hour except Exception as e: - verbose_proxy_logger.warning(f"Cache storage failed: {e}") + verbose_proxy_logger.warning("Cache storage failed: %s", e) return generated_id @@ -599,7 +601,8 @@ class LassoGuardrail(CustomGuardrail): # Log error with context verbose_proxy_logger.error( - f"Error calling Lasso API: {error}", + "Error calling Lasso API: %s", + error, extra={ "guardrail_name": getattr(self, "guardrail_name", "unknown"), "message_type": message_type, @@ -810,7 +813,7 @@ class LassoGuardrail(CustomGuardrail): ) -> LassoResponse: """Call the Lasso API and return the response.""" url = api_url or f"{self.api_base}/classify" - verbose_proxy_logger.debug(f"Calling Lasso API with messageType: {payload.get('messageType')}") + verbose_proxy_logger.debug("Calling Lasso API with messageType: %s", payload.get("messageType")) response = await self.async_handler.post( url=url, headers=headers, @@ -848,7 +851,7 @@ class LassoGuardrail(CustomGuardrail): """ if response and response.get("violations_detected") is True: violated_deputies = self._parse_violated_deputies(response) - verbose_proxy_logger.warning(f"Lasso guardrail detected violations: {violated_deputies}") + verbose_proxy_logger.warning("Lasso guardrail detected violations: %s", violated_deputies) # Check if any findings have "BLOCK" action blocking_violations = self._check_for_blocking_actions(response) @@ -866,7 +869,7 @@ class LassoGuardrail(CustomGuardrail): else: # Continue with warning for non-blocking violations (e.g., AUTO_MASKING) verbose_proxy_logger.info( - f"Non-blocking Lasso violations detected, continuing with warning: {violated_deputies}" + "Non-blocking Lasso violations detected, continuing with warning: %s", violated_deputies ) def _check_for_blocking_actions(self, response: LassoResponse) -> list[str]: @@ -955,7 +958,7 @@ class LassoGuardrail(CustomGuardrail): if msg.content and apply_text and text_cursor < len(masked_text): msg.content = masked_text[text_cursor] text_cursor += 1 - verbose_proxy_logger.debug(f"Applied masked text content to choice {text_cursor}") + verbose_proxy_logger.debug("Applied masked text content to choice %s", text_cursor) for call in getattr(msg, "tool_calls", None) or []: call_id = self._get_field(call, "id") @@ -970,7 +973,7 @@ class LassoGuardrail(CustomGuardrail): func = getattr(call, "function", None) if func: func.arguments = json.dumps(masked_input) - verbose_proxy_logger.debug(f"Applied masked tool_call arguments for call_id={call_id}") + verbose_proxy_logger.debug("Applied masked tool_call arguments for call_id=%s", call_id) @staticmethod def get_config_model() -> type["GuardrailConfigModel"] | None: diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py index c6900c38cbf..c36b5849fb6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py @@ -121,7 +121,7 @@ class CategoryConfig: try: self.phrase_patterns.append((p, re.compile(p, re.IGNORECASE))) except re.error: - verbose_proxy_logger.warning(f"Invalid phrase pattern in {category_name}: {p}") + verbose_proxy_logger.warning("Invalid phrase pattern in %s: %s", category_name, p) class ContentFilterGuardrail(CustomGuardrail): @@ -226,8 +226,8 @@ class ContentFilterGuardrail(CustomGuardrail): p["action"] == ContentFilterAction.MASK for p in self.compiled_patterns ): verbose_proxy_logger.warning( - f"ContentFilterGuardrail '{self.guardrail_name}': 'during_call' mode with 'MASK' action is unstable due to race conditions. " - "Use 'pre_call' mode for reliable request masking." + "ContentFilterGuardrail '%s': 'during_call' mode with 'MASK' action is unstable due to race conditions. Use 'pre_call' mode for reliable request masking.", + self.guardrail_name, ) # Load blocked words - always initialize as dict @@ -238,7 +238,7 @@ class ContentFilterGuardrail(CustomGuardrail): # Defensive check: ensure blocked_words is a dict (not a list) if not isinstance(self.blocked_words, dict): verbose_proxy_logger.error( - f"blocked_words is not a dict, got {type(self.blocked_words)}. Resetting to empty dict." + "blocked_words is not a dict, got %s. Resetting to empty dict.", type(self.blocked_words) ) self.blocked_words = {} @@ -247,11 +247,12 @@ class ContentFilterGuardrail(CustomGuardrail): self._load_blocked_words_file(blocked_words_file) verbose_proxy_logger.debug( - f"ContentFilterGuardrail initialized with {len(self.compiled_patterns)} patterns " - f"and {len(self.blocked_words)} blocked words" + "ContentFilterGuardrail initialized with %s patterns and %s blocked words", + len(self.compiled_patterns), + len(self.blocked_words), ) verbose_proxy_logger.debug( - f"Loaded {len(self.loaded_categories)} categories with {len(self.category_keywords)} keywords" + "Loaded %s categories with %s keywords", len(self.loaded_categories), len(self.category_keywords) ) def _init_competitor_intent_checker(self, competitor_intent_config: dict[str, Any]) -> None: @@ -406,7 +407,7 @@ class ContentFilterGuardrail(CustomGuardrail): # Prevent path traversal via category_name (e.g. "../../etc/passwd") if not re.match(r"^[a-zA-Z0-9_\-]+$", category_name): - verbose_proxy_logger.warning(f"Category name '{category_name}' contains invalid characters, skipping") + verbose_proxy_logger.warning("Category name '%s' contains invalid characters, skipping", category_name) continue enabled = cat_config.get("enabled", True) @@ -417,7 +418,7 @@ class ContentFilterGuardrail(CustomGuardrail): custom_file = cat_config.get("category_file") if not enabled: - verbose_proxy_logger.debug(f"Category {category_name} is disabled, skipping") + verbose_proxy_logger.debug("Category %s is disabled, skipping", category_name) continue # Load category file (custom or default) @@ -425,7 +426,9 @@ class ContentFilterGuardrail(CustomGuardrail): try: category_file_path = self._resolve_category_file_path(custom_file) except ValueError as e: - verbose_proxy_logger.warning(f"Category {category_name}: invalid category_file path, skipping. {e}") + verbose_proxy_logger.warning( + "Category %s: invalid category_file path, skipping. %s", category_name, e + ) continue else: # Try .yaml first, then .json (e.g. harm_toxic_abuse.json) @@ -439,7 +442,7 @@ class ContentFilterGuardrail(CustomGuardrail): category_file_path = yaml_path # will trigger "not found" below if not os.path.exists(category_file_path): - verbose_proxy_logger.warning(f"Category file not found: {category_file_path}, skipping") + verbose_proxy_logger.warning("Category file not found: %s, skipping", category_file_path) continue try: @@ -487,13 +490,14 @@ class ContentFilterGuardrail(CustomGuardrail): ) verbose_proxy_logger.info( - f"Loaded category {category_name}: " - f"{len(category_config_obj.keywords)} keywords, " - f"{len(category_config_obj.always_block_keywords)} always-block keywords, " - f"conditional: {bool(category_config_obj.identifier_words)}" + "Loaded category %s: %s keywords, %s always-block keywords, conditional: %s", + category_name, + len(category_config_obj.keywords), + len(category_config_obj.always_block_keywords), + bool(category_config_obj.identifier_words), ) except Exception as e: - verbose_proxy_logger.error(f"Error loading category {category_name}: {e}") + verbose_proxy_logger.error("Error loading category %s: %s", category_name, e) def _load_conditional_category( self, @@ -534,9 +538,12 @@ class ContentFilterGuardrail(CustomGuardrail): inherit_file_path = inherit_json_path else: verbose_proxy_logger.warning( - f"Category {category_name}: inherit_from '{inherit_from}' file not found at {categories_dir}" + "Category %s: inherit_from '%s' file not found at %s", + category_name, + inherit_from, + categories_dir, ) - verbose_proxy_logger.debug(f"Tried paths: {inherit_yaml_path}, {inherit_json_path}") + verbose_proxy_logger.debug("Tried paths: %s, %s", inherit_yaml_path, inherit_json_path) if inherit_file_path: # Load the inherited category @@ -583,7 +590,7 @@ class ContentFilterGuardrail(CustomGuardrail): verbose_proxy_logger.info(log_msg) except Exception as e: - verbose_proxy_logger.error(f"Error loading conditional category for {category_name}: {e}") + verbose_proxy_logger.error("Error loading conditional category for %s: %s", category_name, e) def _load_category_file(self, file_path: str) -> CategoryConfig: """ @@ -708,9 +715,9 @@ class ContentFilterGuardrail(CustomGuardrail): "allow_word_numbers": bool(extra_config.get("allow_word_numbers")), } ) - verbose_proxy_logger.debug(f"Added pattern: {pattern_name} with action {pattern_config.action}") + verbose_proxy_logger.debug("Added pattern: %s with action %s", pattern_name, pattern_config.action) except Exception as e: - verbose_proxy_logger.error(f"Error adding pattern {pattern_config}: {e}") + verbose_proxy_logger.error("Error adding pattern %s: %s", pattern_config, e) raise def _load_blocked_words_file(self, file_path: str) -> None: @@ -737,7 +744,7 @@ class ContentFilterGuardrail(CustomGuardrail): for word_data in data["blocked_words"]: if not isinstance(word_data, dict) or "keyword" not in word_data or "action" not in word_data: - verbose_proxy_logger.warning(f"Skipping invalid word entry: {word_data}") + verbose_proxy_logger.warning("Skipping invalid word entry: %s", word_data) continue keyword = word_data["keyword"].lower() @@ -746,7 +753,7 @@ class ContentFilterGuardrail(CustomGuardrail): self.blocked_words[keyword] = (action, description) - verbose_proxy_logger.info(f"Loaded {len(data['blocked_words'])} blocked words from {file_path}") + verbose_proxy_logger.info("Loaded %s blocked words from %s", len(data["blocked_words"]), file_path) except FileNotFoundError: raise FileNotFoundError(f"Blocked words file not found: {file_path}") except Exception as e: @@ -889,7 +896,7 @@ class ContentFilterGuardrail(CustomGuardrail): matched_text = text[start:end] pattern_name = pattern_entry["pattern_name"] action = pattern_entry["action"] - verbose_proxy_logger.debug(f"Pattern '{pattern_name}' matched: {matched_text[:20]}...") + verbose_proxy_logger.debug("Pattern '%s' matched: %s...", pattern_name, matched_text[:20]) return (matched_text, pattern_name, action) return None @@ -933,7 +940,7 @@ class ContentFilterGuardrail(CustomGuardrail): for exception in category_obj.exceptions: if exception in text_lower: verbose_proxy_logger.debug( - f"Category exception '{exception}' found for {category_name}, skipping" + "Category exception '%s' found for %s, skipping", exception, category_name ) exception_found = True break @@ -975,7 +982,7 @@ class ContentFilterGuardrail(CustomGuardrail): if block_word_found: matched_phrase = f"{identifier_found} + {block_word_found}" verbose_proxy_logger.warning( - f"Conditional match in {category_name}: '{matched_phrase}' in sentence" + "Conditional match in %s: '%s' in sentence", category_name, matched_phrase ) return (matched_phrase, category_name, severity, action) @@ -1020,7 +1027,7 @@ class ContentFilterGuardrail(CustomGuardrail): for pattern_str, pattern in config.phrase_patterns: if pattern.search(text): - verbose_proxy_logger.warning(f"Phrase pattern match in {category_name}: '{pattern_str}'") + verbose_proxy_logger.warning("Phrase pattern match in %s: '%s'", category_name, pattern_str) return ( f"phrase: {pattern_str}", category_name, @@ -1048,7 +1055,7 @@ class ContentFilterGuardrail(CustomGuardrail): # Check exceptions first — they take precedence over always-block keywords too. for exception in exceptions: if exception in text_lower: - verbose_proxy_logger.debug(f"Exception phrase '{exception}' found, skipping category keyword check") + verbose_proxy_logger.debug("Exception phrase '%s' found, skipping category keyword check", exception) return None # Always-block keywords are checked after exceptions. @@ -1064,7 +1071,7 @@ class ContentFilterGuardrail(CustomGuardrail): keyword_pattern = r"\b" + keyword_pattern_str + r"\b" keyword_found = bool(re.search(keyword_pattern, text_lower)) if keyword_found: - verbose_proxy_logger.debug(f"Always-block keyword '{keyword}' found in category '{category}'") + verbose_proxy_logger.debug("Always-block keyword '%s' found in category '%s'", keyword, category) return (keyword, category, severity, action) # Check category keywords @@ -1095,7 +1102,7 @@ class ContentFilterGuardrail(CustomGuardrail): for exception in category_obj.exceptions: if exception in text_lower: verbose_proxy_logger.debug( - f"Category exception '{exception}' found for keyword '{keyword}', skipping" + "Category exception '%s' found for keyword '%s', skipping", exception, keyword ) exception_found = True break @@ -1103,7 +1110,7 @@ class ContentFilterGuardrail(CustomGuardrail): continue verbose_proxy_logger.debug( - f"Category keyword '{keyword}' found in category '{category}' with severity {severity}" + "Category keyword '%s' found in category '%s' with severity %s", keyword, category, severity ) return (keyword, category, severity, action) return None @@ -1140,7 +1147,7 @@ class ContentFilterGuardrail(CustomGuardrail): text_lower = text.lower() for keyword, (action, description) in self.blocked_words.items(): if keyword in text_lower: - verbose_proxy_logger.debug(f"Blocked word '{keyword}' found with action {action}") + verbose_proxy_logger.debug("Blocked word '%s' found with action %s", keyword, action) return (keyword, action, description) return None @@ -1179,7 +1186,9 @@ class ContentFilterGuardrail(CustomGuardrail): ) elif action == ContentFilterAction.MASK: verbose_proxy_logger.warning( - f"Conditional match '{matched_phrase}' from {category_name} detected but MASK action not supported for conditional categories" + "Conditional match '%s' from %s detected but MASK action not supported for conditional categories", + matched_phrase, + category_name, ) def _handle_category_keyword_match( @@ -1223,7 +1232,7 @@ class ContentFilterGuardrail(CustomGuardrail): flags=re.IGNORECASE, ) verbose_proxy_logger.info( - f"Masked category keyword '{keyword}' from {category_name} (severity: {severity})" + "Masked category keyword '%s' from %s (severity: %s)", keyword, category_name, severity ) return text @@ -1255,7 +1264,7 @@ class ContentFilterGuardrail(CustomGuardrail): elif action == ContentFilterAction.MASK: redaction_tag = self.pattern_redaction_format.format(pattern_name=pattern_name.upper()) text = self._mask_spans(text, spans, redaction_tag) - verbose_proxy_logger.info(f"Masked all {pattern_name} matches in content") + verbose_proxy_logger.info("Masked all %s matches in content", pattern_name) return text @@ -1268,7 +1277,7 @@ class ContentFilterGuardrail(CustomGuardrail): detections: list[ContentFilterDetection] | None, ) -> str: """Handle blocked word match detection and action.""" - verbose_proxy_logger.debug(f"Blocked word '{keyword}' found with action {action}") + verbose_proxy_logger.debug("Blocked word '%s' found with action %s", keyword, action) if detections is not None: blocked_word_detection: BlockedWordDetection = { @@ -1300,7 +1309,7 @@ class ContentFilterGuardrail(CustomGuardrail): text, flags=re.IGNORECASE, ) - verbose_proxy_logger.info(f"Masked keyword '{keyword}' in content") + verbose_proxy_logger.info("Masked keyword '%s' in content", keyword) return text @@ -1427,14 +1436,14 @@ class ContentFilterGuardrail(CustomGuardrail): message = getattr(choice, "message", None) if message and getattr(message, "content", None): image_description = message.content - verbose_proxy_logger.debug(f"Image description: {image_description}") + verbose_proxy_logger.debug("Image description: %s", image_description) descriptions.append(image_description) else: verbose_proxy_logger.warning("No image description found") # Apply content filtering to image descriptions verbose_proxy_logger.debug( - f"ContentFilterGuardrail: Applying guardrail to {len(descriptions)} image description(s)" + "ContentFilterGuardrail: Applying guardrail to %s image description(s)", len(descriptions) ) for description in descriptions: # This will raise HTTPException if BLOCK action is triggered @@ -1780,7 +1789,7 @@ class ContentFilterGuardrail(CustomGuardrail): await self._process_images(images, detections) # Process texts - verbose_proxy_logger.debug(f"ContentFilterGuardrail: Applying guardrail to {len(texts)} text(s)") + verbose_proxy_logger.debug("ContentFilterGuardrail: Applying guardrail to %s text(s)", len(texts)) processed_texts = [] for text in texts: @@ -1852,7 +1861,7 @@ class ContentFilterGuardrail(CustomGuardrail): exception_str: str = "" verbose_proxy_logger.info( - f"ContentFilterGuardrail: Starting robust streaming masking for model {request_data.get('model')}" + "ContentFilterGuardrail: Starting robust streaming masking for model %s", request_data.get("model") ) try: @@ -1897,7 +1906,7 @@ class ContentFilterGuardrail(CustomGuardrail): latest_detections_by_choice[choice_index] = choice_detections raise except Exception as e: - verbose_proxy_logger.error(f"ContentFilterGuardrail: Error in masking: {e}") + verbose_proxy_logger.error("ContentFilterGuardrail: Error in masking: %s", e) masked_text = text_to_scan # Fallback to current text # Determine how much can be safely yielded diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py index 8292f575c74..6b1d7f6a93e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py @@ -168,7 +168,7 @@ def get_available_content_categories() -> list[dict[str, str]]: # Skip files that can't be loaded but log the error for debugging from litellm._logging import verbose_proxy_logger - verbose_proxy_logger.warning(f"Failed to load category file {filename}: {e}") + verbose_proxy_logger.warning("Failed to load category file %s: %s", filename, e) continue elif filename.endswith(".json"): # JSON category files (e.g. harm_toxic_abuse.json) - no YAML header, use filename diff --git a/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py index afbc67f2abb..725719048d7 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py @@ -216,7 +216,9 @@ class LLMAsAJudgeGuardrail(CustomGuardrail): try: judge_result = await self._run_judge(messages, response_text) except Exception as judge_err: - verbose_logger.warning(f"llm_as_a_judge guardrail: judge call failed, failing open. Error: {judge_err}") + verbose_logger.warning( + "llm_as_a_judge guardrail: judge call failed, failing open. Error: %s", judge_err + ) status = "guardrail_failed_to_respond" return inputs @@ -263,7 +265,7 @@ class LLMAsAJudgeGuardrail(CustomGuardrail): except HTTPException: raise except Exception as e: - verbose_logger.warning(f"llm_as_a_judge guardrail unexpected error: {e}") + verbose_logger.warning("llm_as_a_judge guardrail unexpected error: %s", e) return inputs finally: self.add_standard_logging_guardrail_information_to_request_data( diff --git a/litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission/mcp_end_user_permission.py b/litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission/mcp_end_user_permission.py index 1760d01e247..08bd79a4a0c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission/mcp_end_user_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission/mcp_end_user_permission.py @@ -80,7 +80,7 @@ class MCPEndUserPermissionGuardrail(CustomGuardrail): if allowed_mcp_servers is None: return inputs # No restrictions → pass through unchanged - verbose_proxy_logger.debug(f"MCP guardrail: end user restricted to MCP servers: {allowed_mcp_servers}") + verbose_proxy_logger.debug("MCP guardrail: end user restricted to MCP servers: %s", allowed_mcp_servers) filtered_tools = [] removed_tools = [] @@ -97,13 +97,14 @@ class MCPEndUserPermissionGuardrail(CustomGuardrail): else: removed_tools.append(tool_name) verbose_proxy_logger.warning( - f"MCP guardrail: removing tool '{tool_name}' " - f"(server: '{server_name}') — not in end user's allowed servers" + "MCP guardrail: removing tool '%s' (server: '%s') — not in end user's allowed servers", + tool_name, + server_name, ) if removed_tools: verbose_proxy_logger.debug( - f"MCP guardrail: removed {len(removed_tools)} unauthorized MCP tool(s): {removed_tools}" + "MCP guardrail: removed %s unauthorized MCP tool(s): %s", len(removed_tools), removed_tools ) inputs["tools"] = filtered_tools @@ -162,7 +163,7 @@ class MCPEndUserPermissionGuardrail(CustomGuardrail): route="/mcp", ) except Exception as e: - verbose_proxy_logger.warning(f"MCP guardrail: failed to fetch end_user_object for '{end_user_id}': {e}") + verbose_proxy_logger.warning("MCP guardrail: failed to fetch end_user_object for '%s': %s", end_user_id, e) return None # ------------------------------------------------------------------ diff --git a/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py b/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py index b2f91083cc0..a1980134166 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py +++ b/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py @@ -163,7 +163,7 @@ class NomaGuardrail(CustomGuardrail): try: asyncio.create_task(coro) except Exception as e: - verbose_proxy_logger.error(f"Failed to create background Noma task: {e}") + verbose_proxy_logger.error("Failed to create background Noma task: %s", e) async def _process_user_message_check( self, @@ -233,7 +233,7 @@ class NomaGuardrail(CustomGuardrail): if anonymized_content: # Replace the user message content with anonymized version self._replace_user_message_content(request_data, anonymized_content) - verbose_proxy_logger.debug(f"Noma guardrail anonymized user message: {anonymized_content}") + verbose_proxy_logger.debug("Noma guardrail anonymized user message: %s", anonymized_content) return anonymized_content await self._check_verdict(USER_ROLE, json.dumps(input_items), response_json) @@ -309,7 +309,7 @@ class NomaGuardrail(CustomGuardrail): if anonymized_content: # Replace the LLM response content with anonymized version self._replace_llm_response_content(response, anonymized_content) - verbose_proxy_logger.debug(f"Noma guardrail anonymized LLM response: {anonymized_content}") + verbose_proxy_logger.debug("Noma guardrail anonymized LLM response: %s", anonymized_content) return anonymized_content await self._check_verdict(ASSISTANT_ROLE, content, response_json) @@ -348,7 +348,7 @@ class NomaGuardrail(CustomGuardrail): return "guardrail_failed_to_respond" except Exception as e: - verbose_proxy_logger.error(f"Error determining NOMA guardrail status: {e}") + verbose_proxy_logger.error("Error determining NOMA guardrail status: %s", e) return "guardrail_failed_to_respond" def _should_only_sensitive_data_failed(self, classification_obj: dict) -> bool: @@ -513,7 +513,7 @@ class NomaGuardrail(CustomGuardrail): try: await self._process_user_message_check(request_data, user_auth) except Exception as e: - verbose_proxy_logger.error(f"Noma background user message check failed: {e}") + verbose_proxy_logger.error("Noma background user message check failed: %s", e) async def _check_llm_response_background( self, @@ -525,7 +525,7 @@ class NomaGuardrail(CustomGuardrail): try: await self._process_llm_response_check(request_data, response, user_auth) except Exception as e: - verbose_proxy_logger.error(f"Noma background response check failed: {e}") + verbose_proxy_logger.error("Noma background response check failed: %s", e) async def _handle_verdict_background( self, @@ -547,7 +547,7 @@ class NomaGuardrail(CustomGuardrail): msg = f"Noma guardrail allowed {type} message: {message}" verbose_proxy_logger.info(msg) except Exception as e: - verbose_proxy_logger.error(f"Noma background verdict handling failed: {e}") + verbose_proxy_logger.error("Noma background verdict handling failed: %s", e) async def async_pre_call_hook( self, @@ -570,7 +570,7 @@ class NomaGuardrail(CustomGuardrail): try: self._create_background_noma_check(self._check_user_message_background(data, user_api_key_dict)) except Exception as e: - verbose_proxy_logger.error(f"Failed to start background Noma pre-call check: {e}") + verbose_proxy_logger.error("Failed to start background Noma pre-call check: %s", e) return data try: @@ -594,7 +594,7 @@ class NomaGuardrail(CustomGuardrail): event_type=GuardrailEventHooks.pre_call, ) - verbose_proxy_logger.error(f"Noma pre-call hook failed: {e}") + verbose_proxy_logger.error("Noma pre-call hook failed: %s", e) if self.block_failures: raise @@ -618,7 +618,7 @@ class NomaGuardrail(CustomGuardrail): try: self._create_background_noma_check(self._check_user_message_background(data, user_api_key_dict)) except Exception as e: - verbose_proxy_logger.error(f"Failed to start background Noma moderation check: {e}") + verbose_proxy_logger.error("Failed to start background Noma moderation check: %s", e) return data try: @@ -642,7 +642,7 @@ class NomaGuardrail(CustomGuardrail): event_type=GuardrailEventHooks.during_call, ) - verbose_proxy_logger.error(f"Noma moderation hook failed: {e}") + verbose_proxy_logger.error("Noma moderation hook failed: %s", e) if self.block_failures: raise @@ -665,7 +665,7 @@ class NomaGuardrail(CustomGuardrail): self._check_llm_response_background(data, response, user_api_key_dict) ) except Exception as e: - verbose_proxy_logger.error(f"Failed to start background Noma post-call check: {e}") + verbose_proxy_logger.error("Failed to start background Noma post-call check: %s", e) return response try: @@ -689,7 +689,7 @@ class NomaGuardrail(CustomGuardrail): event_type=GuardrailEventHooks.post_call, ) - verbose_proxy_logger.error(f"Noma post-call hook failed: {e}") + verbose_proxy_logger.error("Noma post-call hook failed: %s", e) if self.block_failures: raise return response @@ -828,7 +828,7 @@ class NomaGuardrail(CustomGuardrail): except Exception as e: if self.block_failures: raise - verbose_proxy_logger.error(f"Noma streaming post-call hook failed: {e}") + verbose_proxy_logger.error("Noma streaming post-call hook failed: %s", e) for chunk in all_chunks: yield chunk return diff --git a/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py b/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py index 37ce84b8e6e..7f6787015f6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py +++ b/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py @@ -59,7 +59,7 @@ class OnyxGuardrail(CustomGuardrail): raise ValueError("ONYX_API_KEY environment variable is not set") self.optional_params = kwargs super().__init__(**kwargs) - verbose_proxy_logger.info(f"OnyxGuard initialized with server: {self.api_base}") + verbose_proxy_logger.info("OnyxGuard initialized with server: %s", self.api_base) async def _validate_with_guard_server( self, @@ -87,7 +87,7 @@ class OnyxGuardrail(CustomGuardrail): detection_message = "Unknown violation" if "violated_rules" in result: detection_message = ", ".join(result["violated_rules"]) - verbose_proxy_logger.warning(f"Request blocked by Onyx Guard. Violations: {detection_message}.") + verbose_proxy_logger.warning("Request blocked by Onyx Guard. Violations: %s.", detection_message) raise HTTPException( status_code=400, detail=f"Request blocked by Onyx Guard. Violations: {detection_message}.", @@ -118,7 +118,8 @@ class OnyxGuardrail(CustomGuardrail): payload = parsed.get("response", {}) except Exception as e: verbose_proxy_logger.error( - f"Error in converting request_data to ModelResponse: {e}", + "Error in converting request_data to ModelResponse: %s", + e, extra={ "conversation_id": conversation_id, "input_type": input_type, @@ -133,7 +134,8 @@ class OnyxGuardrail(CustomGuardrail): raise e except Exception as e: verbose_proxy_logger.error( - f"Error in apply_guardrail guard: {e}", + "Error in apply_guardrail guard: %s", + e, extra={"conversation_id": conversation_id, "input_type": input_type}, ) return inputs diff --git a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py index 5ff04864e75..31d777ff089 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py +++ b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py @@ -87,7 +87,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): ) verbose_proxy_logger.debug( - f"Initialized OpenAI Moderation Guardrail: {guardrail_name} with model: {self.model}" + "Initialized OpenAI Moderation Guardrail: %s with model: %s", guardrail_name, self.model ) def _get_api_key(self) -> str | None: diff --git a/litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py b/litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py index 5c707153873..09e24cca9d8 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py +++ b/litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py @@ -94,7 +94,10 @@ class PangeaHandler(CustomGuardrail): **kwargs, ) verbose_proxy_logger.debug( - f"Initialized Pangea Guardrail: name={guardrail_name}, recipe={pangea_input_recipe}, api_base={self.api_base}" + "Initialized Pangea Guardrail: name=%s, recipe=%s, api_base=%s", + guardrail_name, + pangea_input_recipe, + self.api_base, ) async def _call_pangea_ai_guard(self, api: str, payload: dict, hook_name: str) -> dict: @@ -125,7 +128,7 @@ class PangeaHandler(CustomGuardrail): } verbose_proxy_logger.debug( - f"Pangea Guardrail ({hook_name}): Calling endpoint {endpoint} with payload: {payload}" + "Pangea Guardrail (%s): Calling endpoint %s with payload: %s", hook_name, endpoint, payload ) response = await self.async_handler.post(url=endpoint, json=payload, headers=headers) @@ -134,7 +137,7 @@ class PangeaHandler(CustomGuardrail): result = response.json() if result.get("result", {}).get("blocked"): - verbose_proxy_logger.warning(f"Pangea Guardrail ({hook_name}): Request blocked. Response: {result}") + verbose_proxy_logger.warning("Pangea Guardrail (%s): Request blocked. Response: %s", hook_name, result) raise HTTPException( status_code=400, # Bad Request, indicating violation detail={ @@ -143,7 +146,7 @@ class PangeaHandler(CustomGuardrail): }, ) verbose_proxy_logger.debug( - f"Pangea Guardrail ({hook_name}): Request passed. Response: {result.get('result', {}).get('detectors')}" + "Pangea Guardrail (%s): Request passed. Response: %s", hook_name, result.get("result", {}).get("detectors") ) return result @@ -195,7 +198,7 @@ class PangeaHandler(CustomGuardrail): event_type = GuardrailEventHooks.pre_call if self.should_run_guardrail(data=data, event_type=event_type) is not True: verbose_proxy_logger.debug( - f"Pangea Guardrail (async_pre_call_hook): Guardrail is disabled {self.guardrail_name}." + "Pangea Guardrail (async_pre_call_hook): Guardrail is disabled %s.", self.guardrail_name ) return data @@ -286,7 +289,7 @@ class PangeaHandler(CustomGuardrail): event_type = GuardrailEventHooks.post_call if self.should_run_guardrail(data=data, event_type=event_type) is not True: verbose_proxy_logger.debug( - f"Pangea Guardrail (async_pre_call_hook): Guardrail is disabled {self.guardrail_name}." + "Pangea Guardrail (async_pre_call_hook): Guardrail is disabled %s.", self.guardrail_name ) return data try: diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index 782ffef61cf..64831c8161f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -122,10 +122,8 @@ class PanwPrismaAirsHandler(CustomGuardrail): # Warn if no profile is configured (user must have API key with linked profile) if not self.profile_name: verbose_proxy_logger.warning( - f"PANW Prisma AIRS Guardrail '{guardrail_name}': No profile_name configured. " - f"Ensure your API key has a linked profile in Strata Cloud Manager, " - f"or provide 'profile_name'/'profile_id' via config or per-request metadata. " - f"Requests will fail if the API key is not linked to a profile." + "PANW Prisma AIRS Guardrail '%s': No profile_name configured. Ensure your API key has a linked profile in Strata Cloud Manager, or provide 'profile_name'/'profile_id' via config or per-request metadata. Requests will fail if the API key is not linked to a profile.", + guardrail_name, ) self.fallback_on_error = fallback_on_error @@ -143,15 +141,18 @@ class PanwPrismaAirsHandler(CustomGuardrail): if self.fallback_on_error == "allow": verbose_proxy_logger.warning( - f"PANW Prisma AIRS Guardrail '{guardrail_name}': fallback_on_error='allow' - " - f"requests will proceed without scanning when API is unavailable." + "PANW Prisma AIRS Guardrail '%s': fallback_on_error='allow' - requests will proceed without scanning when API is unavailable.", + guardrail_name, ) verbose_proxy_logger.info( - f"Initialized PANW Prisma AIRS Guardrail: {guardrail_name} " - f"(profile={self.profile_name or 'API-key-linked'}, " - f"mask_request={self.mask_request_content}, mask_response={self.mask_response_content}, " - f"fallback_on_error={self.fallback_on_error}, timeout={self.timeout})" + "Initialized PANW Prisma AIRS Guardrail: %s (profile=%s, mask_request=%s, mask_response=%s, fallback_on_error=%s, timeout=%s)", + guardrail_name, + self.profile_name or "API-key-linked", + self.mask_request_content, + self.mask_response_content, + self.fallback_on_error, + self.timeout, ) # MCP event → base-call compatibility map. @@ -231,7 +232,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): return " ".join(text_parts) if text_parts else "" except (AttributeError, IndexError) as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS: Error extracting response text: {e}") + verbose_proxy_logger.error("PANW Prisma AIRS: Error extracting response text: %s", e) return "" async def _call_panw_api( @@ -355,7 +356,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): # Validate response format if "action" not in result: - verbose_proxy_logger.error(f"PANW Prisma AIRS: Invalid API response format: {result}") + verbose_proxy_logger.error("PANW Prisma AIRS: Invalid API response format: %s", result) return {"action": "block", "category": "api_error"} # Check for profile-related errors from PANW API @@ -365,14 +366,14 @@ class PanwPrismaAirsHandler(CustomGuardrail): "not found" in error_msg or "required" in error_msg or "invalid" in error_msg ): verbose_proxy_logger.error( - f"PANW Prisma AIRS: Profile configuration error. " - f"Ensure your API key has a linked profile in Strata Cloud Manager, " - f"or provide 'profile_name' or 'profile_id' in config/metadata. " - f"PANW API response: {result}" + "PANW Prisma AIRS: Profile configuration error. Ensure your API key has a linked profile in Strata Cloud Manager, or provide 'profile_name' or 'profile_id' in config/metadata. PANW API response: %s", + result, ) verbose_proxy_logger.debug( - f"PANW Prisma AIRS: Scan result - Action: {result.get('action')}, Category: {result.get('category', 'unknown')}" + "PANW Prisma AIRS: Scan result - Action: %s, Category: %s", + result.get("action"), + result.get("category", "unknown"), ) return result @@ -406,8 +407,8 @@ class PanwPrismaAirsHandler(CustomGuardrail): if status in (401, 403) or is_profile_error: verbose_proxy_logger.error( - f"PANW Prisma AIRS: Authentication/config error (HTTP {status}). " - f"Check API key and profile configuration." + "PANW Prisma AIRS: Authentication/config error (HTTP %s). Check API key and profile configuration.", + status, ) return { "action": "block", @@ -416,7 +417,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): } elif status == 429 or status >= 500: # Transient: rate-limit and server errors — safe to fail-open - verbose_proxy_logger.error(f"PANW Prisma AIRS: API error (HTTP {status}): {error_body[:500]}") + verbose_proxy_logger.error("PANW Prisma AIRS: API error (HTTP %s): %s", status, error_body[:500]) return { "action": "block", "category": f"http_{status}_error", @@ -425,7 +426,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): else: # Permanent 4xx client errors (400, 404, etc.) — must not bypass scanning if status != 400: # 400 already logged with diagnostics above - verbose_proxy_logger.error(f"PANW Prisma AIRS: API error (HTTP {status}): {error_body[:500]}") + verbose_proxy_logger.error("PANW Prisma AIRS: API error (HTTP %s): %s", status, error_body[:500]) return { "action": "block", "category": f"http_{status}_error", @@ -433,7 +434,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): } except httpx.TimeoutException as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS: Timeout error: {e}") + verbose_proxy_logger.error("PANW Prisma AIRS: Timeout error: %s", e) return { "action": "block", "category": "timeout_error", @@ -441,7 +442,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): } except httpx.RequestError as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS: Network/request error: {e}") + verbose_proxy_logger.error("PANW Prisma AIRS: Network/request error: %s", e) return { "action": "block", "category": "network_error", @@ -449,7 +450,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): } except Exception as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS: Unexpected error: {e}") + verbose_proxy_logger.error("PANW Prisma AIRS: Unexpected error: %s", e) return {"action": "block", "category": "api_error", "_is_transient": True} @staticmethod @@ -713,8 +714,9 @@ class PanwPrismaAirsHandler(CustomGuardrail): if scan_result.get("_is_transient") and self.fallback_on_error == "allow": verbose_proxy_logger.warning( - f"PANW Prisma AIRS: Allowing {'response' if is_response else 'request'} " - f"without scanning (fallback_on_error='allow', error: {category})" + "PANW Prisma AIRS: Allowing %s without scanning (fallback_on_error='allow', error: %s)", + "response" if is_response else "request", + category, ) add_guardrail_to_applied_guardrails_header( request_data=data, guardrail_name=f"{self.guardrail_name}:unscanned" @@ -915,7 +917,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): litellm_metadata = data.setdefault("litellm_metadata", {}) if litellm_metadata.get(scan_key): - verbose_proxy_logger.debug(f"PANW Prisma AIRS: Skipping duplicate {scan_type}-call scan") + verbose_proxy_logger.debug("PANW Prisma AIRS: Skipping duplicate %s-call scan", scan_type) return True # Already scanned litellm_metadata[scan_key] = True @@ -1030,9 +1032,9 @@ class PanwPrismaAirsHandler(CustomGuardrail): data["messages"] = self._apply_masking_to_messages(messages, masked_text) elif "prompt" in data: data["prompt"] = masked_text - verbose_proxy_logger.info(f"PANW Prisma AIRS: Prompt allowed with masking (Category: {category})") + verbose_proxy_logger.info("PANW Prisma AIRS: Prompt allowed with masking (Category: %s)", category) else: - verbose_proxy_logger.info(f"PANW Prisma AIRS: Prompt allowed (Category: {category})") + verbose_proxy_logger.info("PANW Prisma AIRS: Prompt allowed (Category: %s)", category) add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=self.guardrail_name) return None @@ -1050,13 +1052,13 @@ class PanwPrismaAirsHandler(CustomGuardrail): # Block the request error_detail = self._build_error_detail(scan_result, is_response=False) - verbose_proxy_logger.warning(f"PANW Prisma AIRS: {error_detail['error']['message']}") + verbose_proxy_logger.warning("PANW Prisma AIRS: %s", error_detail["error"]["message"]) raise HTTPException(status_code=400, detail=error_detail) except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS scan failed: {e}") + verbose_proxy_logger.error("PANW Prisma AIRS scan failed: %s", e) raise HTTPException( status_code=500, detail={ @@ -1147,9 +1149,11 @@ class PanwPrismaAirsHandler(CustomGuardrail): if action == "allow": if masked_text: self._apply_masking_to_response(response, masked_text) - verbose_proxy_logger.info(f"PANW Prisma AIRS: Response allowed with masking (Category: {category})") + verbose_proxy_logger.info( + "PANW Prisma AIRS: Response allowed with masking (Category: %s)", category + ) else: - verbose_proxy_logger.info(f"PANW Prisma AIRS: Response allowed (Category: {category})") + verbose_proxy_logger.info("PANW Prisma AIRS: Response allowed (Category: %s)", category) add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=self.guardrail_name) return response @@ -1164,13 +1168,13 @@ class PanwPrismaAirsHandler(CustomGuardrail): # Block the response error_detail = self._build_error_detail(scan_result, is_response=True) - verbose_proxy_logger.warning(f"PANW Prisma AIRS: {error_detail['error']['message']}") + verbose_proxy_logger.warning("PANW Prisma AIRS: %s", error_detail["error"]["message"]) raise HTTPException(status_code=400, detail=error_detail) except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS scan failed: {e}") + verbose_proxy_logger.error("PANW Prisma AIRS scan failed: %s", e) raise HTTPException( status_code=500, detail={ @@ -1229,10 +1233,10 @@ class PanwPrismaAirsHandler(CustomGuardrail): self._apply_masking_to_response(assembled_model_response, masked_text) content_was_modified = True verbose_proxy_logger.info( - f"PANW Prisma AIRS: Streaming response allowed with masking (Category: {category})" + "PANW Prisma AIRS: Streaming response allowed with masking (Category: %s)", category ) else: - verbose_proxy_logger.info(f"PANW Prisma AIRS: Streaming response allowed (Category: {category})") + verbose_proxy_logger.info("PANW Prisma AIRS: Streaming response allowed (Category: %s)", category) elif masked_text and self.mask_response_content: self._apply_masking_to_response(assembled_model_response, masked_text) content_was_modified = True @@ -1241,7 +1245,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): ) else: error_detail = self._build_error_detail(scan_result, is_response=True) - verbose_proxy_logger.warning(f"PANW Prisma AIRS: {error_detail['error']['message']}") + verbose_proxy_logger.warning("PANW Prisma AIRS: %s", error_detail["error"]["message"]) raise HTTPException(status_code=400, detail=error_detail) return content_was_modified, assembled_model_response, scan_result @@ -1366,7 +1370,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): error_obj["code"] = e.status_code yield f"data: {json.dumps({'error': error_obj})}\n\n" except Exception as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS streaming error: {e}") + verbose_proxy_logger.error("PANW Prisma AIRS streaming error: %s", e) yield f"data: {json.dumps({'error': {'message': 'Security scan failed - streaming response blocked for safety', 'type': 'guardrail_scan_error', 'code': 500, 'guardrail': self.guardrail_name}})}\n\n" async def _scan_tool_calls_for_guardrail( diff --git a/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py b/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py index 77767c8c61b..058b0a2f23c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py +++ b/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py @@ -209,10 +209,10 @@ class PillarGuardrail(CustomGuardrail): self.on_flagged_action = action else: if action: - verbose_proxy_logger.warning(f"Invalid action '{action}', using default") + verbose_proxy_logger.warning("Invalid action '%s', using default", action) self.on_flagged_action = self.DEFAULT_ON_FLAGGED_ACTION - verbose_proxy_logger.debug(f"Pillar Guardrail: Initialized with on_flagged_action: {self.on_flagged_action}") + verbose_proxy_logger.debug("Pillar Guardrail: Initialized with on_flagged_action: %s", self.on_flagged_action) self.async_mode = self._resolve_bool_config( provided_value=async_mode, @@ -246,11 +246,11 @@ class PillarGuardrail(CustomGuardrail): else: if action: verbose_proxy_logger.warning( - f"Invalid fallback action '{action}', using default '{self.DEFAULT_FALLBACK_ACTION}'" + "Invalid fallback action '%s', using default '%s'", action, self.DEFAULT_FALLBACK_ACTION ) self.fallback_on_error = self.DEFAULT_FALLBACK_ACTION - verbose_proxy_logger.debug(f"Pillar Guardrail: Initialized with fallback_on_error: {self.fallback_on_error}") + verbose_proxy_logger.debug("Pillar Guardrail: Initialized with fallback_on_error: %s", self.fallback_on_error) # Set timeout with graceful fallback on invalid configuration if timeout is not None: @@ -260,8 +260,9 @@ class PillarGuardrail(CustomGuardrail): self.timeout = float(os.environ.get("PILLAR_TIMEOUT", str(self.DEFAULT_TIMEOUT))) except (ValueError, TypeError): verbose_proxy_logger.warning( - f"Pillar Guardrail: Invalid PILLAR_TIMEOUT value '{os.environ.get('PILLAR_TIMEOUT')}', " - f"falling back to default {self.DEFAULT_TIMEOUT}s" + "Pillar Guardrail: Invalid PILLAR_TIMEOUT value '%s', falling back to default %ss", + os.environ.get("PILLAR_TIMEOUT"), + self.DEFAULT_TIMEOUT, ) self.timeout = self.DEFAULT_TIMEOUT @@ -311,7 +312,7 @@ class PillarGuardrail(CustomGuardrail): """ event_type = GuardrailEventHooks.pre_call if self.should_run_guardrail(data=data, event_type=event_type) is not True: - verbose_proxy_logger.debug(f"Pillar Guardrail: Pre-call scanning disabled for {self.guardrail_name}") + verbose_proxy_logger.debug("Pillar Guardrail: Pre-call scanning disabled for %s", self.guardrail_name) return data verbose_proxy_logger.debug("Pillar Guardrail: Pre-call hook") @@ -354,7 +355,7 @@ class PillarGuardrail(CustomGuardrail): """ event_type = GuardrailEventHooks.during_call if self.should_run_guardrail(data=data, event_type=event_type) is not True: - verbose_proxy_logger.debug(f"Pillar Guardrail: During-call scanning disabled for {self.guardrail_name}") + verbose_proxy_logger.debug("Pillar Guardrail: During-call scanning disabled for %s", self.guardrail_name) return data verbose_proxy_logger.debug("Pillar Guardrail: During-call moderation hook") @@ -388,7 +389,7 @@ class PillarGuardrail(CustomGuardrail): """ event_type = GuardrailEventHooks.post_call if self.should_run_guardrail(data=data, event_type=event_type) is not True: - verbose_proxy_logger.debug(f"Pillar Guardrail: Post-call scanning disabled for {self.guardrail_name}") + verbose_proxy_logger.debug("Pillar Guardrail: Post-call scanning disabled for %s", self.guardrail_name) return response verbose_proxy_logger.debug("Pillar Guardrail: Post-call hook") @@ -457,7 +458,7 @@ class PillarGuardrail(CustomGuardrail): raise e # Handle API communication errors based on fallback_on_error setting - verbose_proxy_logger.error(f"Pillar Guardrail: API communication failed - {e}") + verbose_proxy_logger.error("Pillar Guardrail: API communication failed - %s", e) return self._handle_api_error(e, data) @@ -677,8 +678,11 @@ class PillarGuardrail(CustomGuardrail): payload["provider"] = provider verbose_proxy_logger.debug( - f"Pillar Guardrail: Request context - user={user_id}, session={session_id}, " - f"model={model}, provider={provider}" + "Pillar Guardrail: Request context - user=%s, session=%s, model=%s, provider=%s", + user_id, + session_id, + model, + provider, ) return payload @@ -694,7 +698,7 @@ class PillarGuardrail(CustomGuardrail): Pillar API response as dictionary """ verbose_proxy_logger.debug( - f"Pillar Guardrail: Scanning {len(payload.get('messages', []))} messages for security threats" + "Pillar Guardrail: Scanning %s messages for security threats", len(payload.get("messages", [])) ) response = await self.async_handler.post( url=f"{self.api_base}/api/v1/protect", @@ -707,7 +711,7 @@ class PillarGuardrail(CustomGuardrail): flagged = res.get("flagged") session_id = res.get("session_id") - verbose_proxy_logger.debug(f"Pillar Guardrail: Analysis complete - flagged={flagged}, session={session_id}") + verbose_proxy_logger.debug("Pillar Guardrail: Analysis complete - flagged=%s, session=%s", flagged, session_id) return res def _process_pillar_response(self, pillar_response: dict[str, Any], original_data: dict) -> None: @@ -739,7 +743,7 @@ class PillarGuardrail(CustomGuardrail): # Store session_id from Pillar response for potential reuse pillar_session_id = pillar_response.get("session_id") if pillar_session_id: - verbose_proxy_logger.debug(f"Pillar Guardrail: Received session_id from server: {pillar_session_id}") + verbose_proxy_logger.debug("Pillar Guardrail: Received session_id from server: %s", pillar_session_id) # Store in request metadata for use in subsequent hooks if "pillar_session_id" not in metadata_store: metadata_store["pillar_session_id"] = pillar_session_id diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 7a38c4087c6..56f08fe44ca 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -740,7 +740,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): elif isinstance(content, list) and content_idx_optional is not None: messages[msg_idx]["content"][content_idx_optional]["text"] = r - verbose_proxy_logger.debug(f"Presidio PII Masking: Redacted pii message: {data['messages']}") + verbose_proxy_logger.debug("Presidio PII Masking: Redacted pii message: %s", data["messages"]) data["messages"] = messages return data except Exception as e: @@ -832,7 +832,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): elif isinstance(content, list) and content_idx_optional is not None: messages[msg_idx]["content"][content_idx_optional]["text"] = r - verbose_proxy_logger.debug(f"Presidio PII Masking: Redacted pii message: {messages}") + verbose_proxy_logger.debug("Presidio PII Masking: Redacted pii message: %s", messages) kwargs["messages"] = messages return kwargs, result @@ -847,7 +847,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): Output parse the response object to replace the masked tokens with user sent values """ verbose_proxy_logger.debug( - f"PII Masking Args: self.output_parse_pii={self.output_parse_pii}; type of response={type(response)}" + "PII Masking Args: self.output_parse_pii=%s; type of response=%s", self.output_parse_pii, type(response) ) if self.apply_to_output is True: @@ -1124,7 +1124,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): yield mock_response_stream except Exception as e: - verbose_proxy_logger.error(f"Error masking streaming PII output: {e}") + verbose_proxy_logger.error("Error masking streaming PII output: %s", e) for chunk in all_chunks: yield chunk @@ -1253,7 +1253,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): yield mock_response_stream except Exception as e: - verbose_proxy_logger.error(f"Error in PII streaming processing: {e}") + verbose_proxy_logger.error("Error in PII streaming processing: %s", e) for chunk in remaining_chunks: yield chunk diff --git a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py index 0f3a817b12c..a816ef3846e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py +++ b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py @@ -326,7 +326,7 @@ class PromptSecurityGuardrail(CustomGuardrail): except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error processing image: {e}") + verbose_proxy_logger.error("Error processing image: %s", e) @staticmethod def _resolve_key_alias_from_request_data(request_data: dict) -> str | None: @@ -481,7 +481,7 @@ class PromptSecurityGuardrail(CustomGuardrail): except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error sanitizing image file: {e}") + verbose_proxy_logger.error("Error sanitizing image file: %s", e) raise HTTPException(status_code=500, detail=f"File sanitization failed: {e}") async def _process_document_item(self, item: dict, user_api_key_alias: str | None) -> dict: @@ -520,7 +520,7 @@ class PromptSecurityGuardrail(CustomGuardrail): extension = mime_type.split("/")[-1] filename = f"document.{extension}" - verbose_proxy_logger.info(f"Sanitizing document: {filename}") + verbose_proxy_logger.info("Sanitizing document: %s", filename) sanitization_result = await self.sanitize_file_content( file_data, filename, user_api_key_alias=user_api_key_alias @@ -554,7 +554,7 @@ class PromptSecurityGuardrail(CustomGuardrail): except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error sanitizing document: {e}") + verbose_proxy_logger.error("Error sanitizing document: %s", e) raise HTTPException(status_code=500, detail=f"Document sanitization failed: {e}") async def process_message_files(self, messages: list, user_api_key_alias: str | None = None) -> list: diff --git a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py index fe2cc40074f..c93cc3f36f3 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py +++ b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py @@ -344,7 +344,7 @@ class QualifireGuardrail(CustomGuardrail): ) url = f"{self.qualifire_api_base}/api/evaluation/evaluate" - verbose_proxy_logger.debug(f"Qualifire Guardrail: Making request to {url}") + verbose_proxy_logger.debug("Qualifire Guardrail: Making request to %s", url) # Make the API request response = await self.async_handler.post( @@ -373,8 +373,8 @@ class QualifireGuardrail(CustomGuardrail): if is_flagged: if on_flagged == "monitor": verbose_proxy_logger.warning( - "Qualifire Guardrail: Monitoring mode - violation detected but allowing request. " - f"Response: {qualifire_response}" + "Qualifire Guardrail: Monitoring mode - violation detected but allowing request. Response: %s", + qualifire_response, ) else: # Block the request @@ -389,7 +389,7 @@ class QualifireGuardrail(CustomGuardrail): except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Qualifire Guardrail error: {e}") + verbose_proxy_logger.exception("Qualifire Guardrail error: %s", e) raise @log_guardrail_information diff --git a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py index c30b2b0910c..056489de70f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py +++ b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py @@ -112,7 +112,7 @@ class SemanticGuardRouteLoader: ) ) - verbose_logger.info(f"SemanticGuard: built {len(routes)} routes") + verbose_logger.info("SemanticGuard: built %s routes", len(routes)) return routes @classmethod diff --git a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/semantic_guard.py b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/semantic_guard.py index f57827d03c9..1657485ed78 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/semantic_guard.py +++ b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/semantic_guard.py @@ -89,8 +89,11 @@ class SemanticGuardrail(CustomGuardrail): self.route_count = len(routes) verbose_logger.info( - f"SemanticGuardrail '{guardrail_name}' initialized with {self.route_count} routes, " - f"embedding_model={embedding_model}, threshold={similarity_threshold}" + "SemanticGuardrail '%s' initialized with %s routes, embedding_model=%s, threshold=%s", + guardrail_name, + self.route_count, + embedding_model, + similarity_threshold, ) @classmethod @@ -219,7 +222,7 @@ def _handle_match( } verbose_logger.warning( - f"SemanticGuard match: route={route_name}, score={similarity_score}, action={guardrail.on_flagged_action}" + "SemanticGuard match: route=%s, score=%s, action=%s", route_name, similarity_score, guardrail.on_flagged_action ) if guardrail.on_flagged_action == "passthrough": diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py index b15e5b61243..b3cfa0ab4d9 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -227,7 +227,7 @@ class ToolPermissionGuardrail(CustomGuardrail): Returns: Tuple of (is_allowed, rule_id, message) """ - verbose_proxy_logger.debug(f"Checking permission for tool: {tool_name or tool_type}") + verbose_proxy_logger.debug("Checking permission for tool: %s", tool_name or tool_type) # Check each rule in order for rule in self.rules: @@ -539,7 +539,7 @@ class ToolPermissionGuardrail(CustomGuardrail): if not denied_tool_names: return data - verbose_proxy_logger.info(f"Blocking {len(denied_tool_names)} unauthorized tool uses") + verbose_proxy_logger.info("Blocking %s unauthorized tool uses", len(denied_tool_names)) # Create a mapping of tool_use_id to error result error_tool_names = set() @@ -606,7 +606,7 @@ class ToolPermissionGuardrail(CustomGuardrail): if not denied_tools: return - verbose_proxy_logger.info(f"Blocking {len(denied_tools)} unauthorized tool uses") + verbose_proxy_logger.info("Blocking %s unauthorized tool uses", len(denied_tools)) # Create a mapping of tool_use_id to error result error_results = {} @@ -680,7 +680,7 @@ class ToolPermissionGuardrail(CustomGuardrail): is_allowed, _, message = self._check_tool_permission(tool_name, tool_type) if not is_allowed and message is not None: - verbose_proxy_logger.warning(f"Tool Permission Guardrail: {message}") + verbose_proxy_logger.warning("Tool Permission Guardrail: %s", message) if self.on_disallowed_action == "block": raise HTTPException( status_code=400, @@ -730,7 +730,7 @@ class ToolPermissionGuardrail(CustomGuardrail): verbose_proxy_logger.debug("Tool Permission Guardrail: No tool uses found") return response - verbose_proxy_logger.debug(f"Tool Permission Guardrail: Found {len(tool_calls)} tool calls") + verbose_proxy_logger.debug("Tool Permission Guardrail: Found %s tool calls", len(tool_calls)) # Check permissions for each tool use denied_tools = [] @@ -738,7 +738,7 @@ class ToolPermissionGuardrail(CustomGuardrail): is_allowed, rule_id, message = self._get_permission_for_tool_call(tool_call) if not is_allowed and message is not None: - verbose_proxy_logger.warning(f"Tool Permission Guardrail: {message}") + verbose_proxy_logger.warning("Tool Permission Guardrail: %s", message) if self.on_disallowed_action == "block": raise GuardrailRaisedException( @@ -809,7 +809,7 @@ class ToolPermissionGuardrail(CustomGuardrail): yield chunk return - verbose_proxy_logger.debug(f"Tool Permission Guardrail: Found {len(tool_calls)} tool calls") + verbose_proxy_logger.debug("Tool Permission Guardrail: Found %s tool calls", len(tool_calls)) # Check permissions for each tool use denied_tools = [] @@ -817,7 +817,7 @@ class ToolPermissionGuardrail(CustomGuardrail): is_allowed, rule_id, message = self._get_permission_for_tool_call(tool_call) if not is_allowed and message is not None: - verbose_proxy_logger.warning(f"Tool Permission Guardrail: {message}") + verbose_proxy_logger.warning("Tool Permission Guardrail: %s", message) if self.on_disallowed_action == "block": raise GuardrailRaisedException( diff --git a/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py b/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py index 9e16e9d5786..f28fd7975f2 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py +++ b/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py @@ -70,9 +70,10 @@ class ZscalerAIGuard(CustomGuardrail): ) verbose_proxy_logger.debug( - f"""send_user_api_key_alias: {self.send_user_api_key_alias}, - send_user_api_key_user_id:{self.send_user_api_key_user_id}, - send_user_api_key_team_id:{self.send_user_api_key_team_id}""" + "send_user_api_key_alias: %s, \n send_user_api_key_user_id:%s, \n send_user_api_key_team_id:%s", + self.send_user_api_key_alias, + self.send_user_api_key_user_id, + self.send_user_api_key_team_id, ) super().__init__(**kwargs) @@ -144,7 +145,7 @@ class ZscalerAIGuard(CustomGuardrail): texts = inputs.get("texts", []) try: - verbose_proxy_logger.debug(f"ZscalerAIGuard: Checking {len(texts)} text(s)") + verbose_proxy_logger.debug("ZscalerAIGuard: Checking %s text(s)", len(texts)) metadata = request_data.get("metadata", {}) user_api_key_metadata = metadata.get("user_api_key_metadata", {}) or {} @@ -166,7 +167,7 @@ class ZscalerAIGuard(CustomGuardrail): ) ) ) - verbose_proxy_logger.info(f"policy_id applied: {policy_id}") + verbose_proxy_logger.info("policy_id applied: %s", policy_id) kwargs = {} if self.send_user_api_key_alias: @@ -179,11 +180,11 @@ class ZscalerAIGuard(CustomGuardrail): kwargs["user_api_key_user_id"] = ( self._resolve_metadata_value(request_data, "user_api_key_user_id") or "N/A" ) - verbose_proxy_logger.debug(f"inside apply_guardrail kwargs: {kwargs}") + verbose_proxy_logger.debug("inside apply_guardrail kwargs: %s", kwargs) zscaler_ai_guard_result = None direction = "OUT" if input_type == "response" else "IN" - verbose_proxy_logger.debug(f"direction: {direction}") + verbose_proxy_logger.debug("direction: %s", direction) # Concatenate all texts and send to Zscaler AI Guard if texts: concatenated_text = " ".join(texts) @@ -195,7 +196,7 @@ class ZscalerAIGuard(CustomGuardrail): content=concatenated_text, **kwargs, ) - verbose_proxy_logger.debug(f"response from zscaler ai guards: {zscaler_ai_guard_result}") + verbose_proxy_logger.debug("response from zscaler ai guards: %s", zscaler_ai_guard_result) if zscaler_ai_guard_result and zscaler_ai_guard_result.get("action") == "BLOCK": blocking_info = zscaler_ai_guard_result.get("zscaler_ai_guard_response") error_message = f"Content blocked by Zscaler AI Guard: {self.extract_blocking_info(blocking_info)}" @@ -241,9 +242,9 @@ class ZscalerAIGuard(CustomGuardrail): } extra_headers = headers.copy() if self.send_user_api_key_alias: - verbose_proxy_logger.debug(f"kwargs: {kwargs}") + verbose_proxy_logger.debug("kwargs: %s", kwargs) user_api_key_alias = kwargs.get("user_api_key_alias", "N/A") - verbose_proxy_logger.debug(f"kwargs user_api_key_alias: {user_api_key_alias}") + verbose_proxy_logger.debug("kwargs user_api_key_alias: %s", user_api_key_alias) extra_headers.update({"user-api-key-alias": user_api_key_alias}) if self.send_user_api_key_team_id: @@ -254,7 +255,7 @@ class ZscalerAIGuard(CustomGuardrail): user_api_key_user_id = kwargs.get("user_api_key_user_id", "N/A") extra_headers.update({"user-api-key-user-id": user_api_key_user_id}) - verbose_proxy_logger.debug(f"extra_headers: {extra_headers}") + verbose_proxy_logger.debug("extra_headers: %s", extra_headers) return extra_headers async def _send_request(self, url, headers, data): @@ -279,7 +280,7 @@ class ZscalerAIGuard(CustomGuardrail): if response.status_code >= 500: # Server error verbose_proxy_logger.error( - f"Zscaler AI Guard service is unavailable (Status: {response.status_code}). Blocking request." + "Zscaler AI Guard service is unavailable (Status: %s). Blocking request.", response.status_code ) user_facing_error = self._create_user_facing_error(f"Service is unavailable (HTTP {response.status_code})") raise HTTPException(status_code=500, detail=user_facing_error) @@ -289,11 +290,11 @@ class ZscalerAIGuard(CustomGuardrail): statusCode_in_response = json_response.get("statusCode", None) if statusCode_in_response == 200: guardrail_result = json_response.get("action", None) - verbose_proxy_logger.info(f"Zscaler AI Guard response: {json_response}") + verbose_proxy_logger.info("Zscaler AI Guard response: %s", json_response) if guardrail_result == "BLOCK": verbose_proxy_logger.info( - f"Violated Zscaler AI Guard guardrail policy. zscaler_ai_guard_response: {json_response}" + "Violated Zscaler AI Guard guardrail policy. zscaler_ai_guard_response: %s", json_response ) return { "action": "BLOCK", @@ -301,7 +302,7 @@ class ZscalerAIGuard(CustomGuardrail): } elif guardrail_result == "ALLOW" or guardrail_result == "DETECT": verbose_proxy_logger.debug( - f"{direction} is allowed by Zscaler AI Guard. guardrail_result: {guardrail_result}" + "%s is allowed by Zscaler AI Guard. guardrail_result: %s", direction, guardrail_result ) return { "action": "ALLOW", @@ -310,7 +311,7 @@ class ZscalerAIGuard(CustomGuardrail): } else: verbose_proxy_logger.error( - f"Action field in response is {guardrail_result}, expecting 'ALLOW', 'BLOCK' or 'DETECT'" + "Action field in response is %s, expecting 'ALLOW', 'BLOCK' or 'DETECT'", guardrail_result ) user_facing_error = self._create_user_facing_error( f"Action field in response is {guardrail_result}, expecting 'ALLOW', 'BLOCK' or 'DETECT'" @@ -318,13 +319,13 @@ class ZscalerAIGuard(CustomGuardrail): raise HTTPException(status_code=500, detail=user_facing_error) else: errorMsg = json_response.get("errorMsg", None) - verbose_proxy_logger.error(f"statusCode in response: {statusCode_in_response}, errorMsg: {errorMsg}") + verbose_proxy_logger.error("statusCode in response: %s, errorMsg: %s", statusCode_in_response, errorMsg) user_facing_error = self._create_user_facing_error( f"statusCode in response: {statusCode_in_response}, errorMsg: {errorMsg}" ) raise HTTPException(status_code=500, detail=user_facing_error) else: - verbose_proxy_logger.error(f"Zscaler AI Guard status_code - {response.status_code}") + verbose_proxy_logger.error("Zscaler AI Guard status_code - %s", response.status_code) user_facing_error = self._create_user_facing_error(f"Response status code: {response.status_code}") raise HTTPException(status_code=response.status_code, detail=user_facing_error) @@ -350,7 +351,7 @@ class ZscalerAIGuard(CustomGuardrail): response = await self._send_request(zscaler_ai_guard_url, extra_headers, data) return self._handle_response(response, direction) except Exception as e: - verbose_proxy_logger.error(f"{e}. Blocking request.") + verbose_proxy_logger.error("%s. Blocking request.", e) user_facing_error = self._create_user_facing_error(f"{e}") raise HTTPException(status_code=500, detail=user_facing_error) diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index b0e16c0ed2e..bb8787ce72d 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -115,7 +115,7 @@ def get_guardrail_initializer_from_hooks(): module_path = f"litellm.proxy.guardrails.guardrail_hooks.{item}" try: # Import the module - verbose_proxy_logger.debug(f"Discovering guardrails in: {module_path}") + verbose_proxy_logger.debug("Discovering guardrails in: %s", module_path) module = importlib.import_module(module_path) @@ -125,7 +125,7 @@ def get_guardrail_initializer_from_hooks(): if isinstance(registry, dict): discovered_initializers.update(registry) verbose_proxy_logger.debug( - f"Found guardrail_initializer_registry in {module_path}: {list(registry.keys())}" + "Found guardrail_initializer_registry in %s: %s", module_path, list(registry.keys()) ) # Check for standalone initialize_guardrail function (fallback for directory-based guardrails) @@ -133,21 +133,23 @@ def get_guardrail_initializer_from_hooks(): # For directories with just initialize_guardrail, use the directory name as the key initialize_fn = getattr(module, "initialize_guardrail") discovered_initializers[item] = initialize_fn - verbose_proxy_logger.debug(f"Found initialize_guardrail function in {module_path}") + verbose_proxy_logger.debug("Found initialize_guardrail function in %s", module_path) except ImportError as e: - verbose_proxy_logger.error(f"Could not import {module_path}: {e}") + verbose_proxy_logger.error("Could not import %s: %s", module_path, e) continue except Exception as e: - verbose_proxy_logger.error(f"Error processing {module_path}: {e}") + verbose_proxy_logger.error("Error processing %s: %s", module_path, e) continue verbose_proxy_logger.debug( - f"Discovered {len(discovered_initializers)} guardrail initializers: {list(discovered_initializers.keys())}" + "Discovered %s guardrail initializers: %s", + len(discovered_initializers), + list(discovered_initializers.keys()), ) except Exception as e: - verbose_proxy_logger.error(f"Error discovering guardrail initializers: {e}") + verbose_proxy_logger.error("Error discovering guardrail initializers: %s", e) return discovered_initializers @@ -194,7 +196,7 @@ def get_guardrail_class_from_hooks(): try: # Import the module - verbose_proxy_logger.debug(f"Discovering guardrails in: {module_path}") + verbose_proxy_logger.debug("Discovering guardrails in: %s", module_path) module = importlib.import_module(module_path) @@ -205,14 +207,14 @@ def get_guardrail_class_from_hooks(): discovered_classes.update(registry) except ImportError as e: - verbose_proxy_logger.debug(f"Could not import {module_path}: {e}") + verbose_proxy_logger.debug("Could not import %s: %s", module_path, e) continue except Exception as e: - verbose_proxy_logger.exception(f"Error processing {module_path}: {e}") + verbose_proxy_logger.exception("Error processing %s: %s", module_path, e) continue except Exception as e: - verbose_proxy_logger.error(f"Error discovering guardrail initializers: {e}") + verbose_proxy_logger.error("Error discovering guardrail initializers: %s", e) return discovered_classes @@ -686,8 +688,8 @@ class InMemoryGuardrailHandler: return LitellmParams(**params).model_dump() except ValidationError as e: verbose_proxy_logger.warning( - f"Could not normalize guardrail litellm_params for comparison; " - f"treating the guardrail as changed. Error: {e}" + "Could not normalize guardrail litellm_params for comparison; treating the guardrail as changed. Error: %s", + e, ) return params return params @@ -723,7 +725,7 @@ class InMemoryGuardrailHandler: # Log differences if any found if changed_fields: - verbose_proxy_logger.debug(f"Guardrail params changed. Differences: {changed_fields}") + verbose_proxy_logger.debug("Guardrail params changed. Differences: %s", changed_fields) # Return True if any fields changed return len(changed_fields) > 0 @@ -763,7 +765,7 @@ class InMemoryGuardrailHandler: if self._has_guardrail_params_changed(guardrail_id, guardrail): guardrail_name = guardrail.get("guardrail_name", "Unknown") verbose_proxy_logger.info( - f"Guardrail '{guardrail_name}' (ID: {guardrail_id}) params changed, re-initializing..." + "Guardrail '%s' (ID: %s) params changed, re-initializing...", guardrail_name, guardrail_id ) return self.reinitialize_guardrail( guardrail=guardrail, diff --git a/litellm/proxy/guardrails/init_guardrails.py b/litellm/proxy/guardrails/init_guardrails.py index 036ee5dca78..183afb941cb 100644 --- a/litellm/proxy/guardrails/init_guardrails.py +++ b/litellm/proxy/guardrails/init_guardrails.py @@ -86,7 +86,7 @@ def _populate_router_guardrail_list(guardrail_list: list[Guardrail]) -> None: router_guardrail_list.append(router_guardrail) llm_router.guardrail_list = router_guardrail_list - verbose_proxy_logger.debug(f"Populated router guardrail_list with {len(router_guardrail_list)} guardrails") + verbose_proxy_logger.debug("Populated router guardrail_list with %s guardrails", len(router_guardrail_list)) ### LEGACY IMPLEMENTATION ### @@ -97,7 +97,7 @@ def initialize_guardrails( litellm_settings: dict, ) -> dict[str, GuardrailItem]: try: - verbose_proxy_logger.debug(f"validating guardrails passed {guardrails_config}") + verbose_proxy_logger.debug("validating guardrails passed %s", guardrails_config) global all_guardrails for item in guardrails_config: """ @@ -141,5 +141,5 @@ def initialize_guardrails( return litellm.guardrail_name_config_map except Exception as e: - verbose_proxy_logger.exception(f"error initializing guardrails {e}") + verbose_proxy_logger.exception("error initializing guardrails %s", e) raise e diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index c3bce5e8370..40b986f39e6 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -425,7 +425,7 @@ async def health_services_endpoint( } except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.health_services_endpoint(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.health_services_endpoint(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -504,7 +504,7 @@ async def _save_health_check_to_db( checked_by=user_id, ) except Exception as db_error: - verbose_proxy_logger.warning(f"Failed to save health check to database for model {model_name}: {db_error}") + verbose_proxy_logger.warning("Failed to save health check to database for model %s: %s", model_name, db_error) # Continue execution - don't let database save failure break health checks @@ -708,7 +708,7 @@ async def _save_background_health_checks_to_db( checked_by, ) except Exception as db_error: - verbose_proxy_logger.warning(f"Failed to save background health checks to database: {db_error}") + verbose_proxy_logger.warning("Failed to save background health checks to database: %s", db_error) # Continue execution - don't let database save failure break health checks @@ -882,7 +882,7 @@ def _health_endpoint_resolve_target_model_name( try: deployment = llm_router.get_deployment(model_id=model_id) except Exception as e: - verbose_proxy_logger.error(f"Error getting deployment for model_id {model_id}: {e}") + verbose_proxy_logger.error("Error getting deployment for model_id %s: %s", model_id, e) raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail={"error": f"Model with ID {model_id} not found"}, @@ -1069,7 +1069,7 @@ async def health_endpoint( ) return _post_process(router_result) except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.py::health_endpoint(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.py::health_endpoint(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) raise e @@ -1107,7 +1107,7 @@ async def health_check_history_endpoint( "offset": offset, } except Exception as e: - verbose_proxy_logger.error(f"Error getting health check history: {e}") + verbose_proxy_logger.error("Error getting health check history: %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to retrieve health check history: {e}"}, @@ -1139,7 +1139,7 @@ async def latest_health_checks_endpoint( "total_models": len(checks_data), } except Exception as e: - verbose_proxy_logger.error(f"Error getting latest health checks: {e}") + verbose_proxy_logger.error("Error getting latest health checks: %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to retrieve latest health checks: {e}"}, @@ -1182,7 +1182,7 @@ async def shared_health_check_status_endpoint( health_status = await shared_health_manager.get_health_check_status() return {"shared_health_check_enabled": True, "status": health_status} except Exception as e: - verbose_proxy_logger.error(f"Error getting shared health check status: {e}") + verbose_proxy_logger.error("Error getting shared health check status: %s", e) raise HTTPException( status_code=fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to retrieve shared health check status: {e}"}, @@ -1850,7 +1850,7 @@ async def test_model_connection( loaded_model_info = dict(deployments[0].get("model_info") or {}) except Exception as e: verbose_proxy_logger.debug( - f"Could not find model {model_name} in router: {e}. Proceeding with request params only." + "Could not find model %s in router: %s. Proceeding with request params only.", model_name, e ) # Merge: config params (from proxy config) as base, request params override @@ -1897,7 +1897,7 @@ async def test_model_connection( except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.debug(f"litellm.proxy.health_endpoints.test_model_connection(): Exception occurred - {e}") + verbose_proxy_logger.debug("litellm.proxy.health_endpoints.test_model_connection(): Exception occurred - %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to test connection: {e}"}, diff --git a/litellm/proxy/hooks/azure_content_safety.py b/litellm/proxy/hooks/azure_content_safety.py index 3c7713b2819..8195cc87a1f 100644 --- a/litellm/proxy/hooks/azure_content_safety.py +++ b/litellm/proxy/hooks/azure_content_safety.py @@ -123,7 +123,7 @@ class _PROXY_AzureContentSafety( raise e except Exception as e: verbose_proxy_logger.error( - f"litellm.proxy.hooks.azure_content_safety.py::async_pre_call_hook(): Exception occured - {e}" + "litellm.proxy.hooks.azure_content_safety.py::async_pre_call_hook(): Exception occured - %s", e ) verbose_proxy_logger.debug(traceback.format_exc()) diff --git a/litellm/proxy/hooks/batch_rate_limiter.py b/litellm/proxy/hooks/batch_rate_limiter.py index ce4ff2cb370..afaf0ebf392 100644 --- a/litellm/proxy/hooks/batch_rate_limiter.py +++ b/litellm/proxy/hooks/batch_rate_limiter.py @@ -256,7 +256,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): if skip_providers: batch_provider = self._resolve_batch_provider(self._get_batch_routing_model(data)) if batch_provider and batch_provider in skip_providers: - verbose_proxy_logger.debug(f"Skipping batch input file processing for provider={batch_provider}") + verbose_proxy_logger.debug("Skipping batch input file processing for provider=%s", batch_provider) return True, None descriptors = self._create_batch_rate_limit_descriptors( @@ -592,15 +592,15 @@ class _PROXY_BatchRateLimiter(CustomLogger): # in the access log instead of getting buried in error noise. if e.status_code == 403: verbose_proxy_logger.warning( - f"Batch rejected: caller not authorized for a model named in {file_id}: {e.detail}" + "Batch rejected: caller not authorized for a model named in %s: %s", file_id, e.detail ) else: verbose_proxy_logger.error( - f"Batch input file rejected for {file_id}: status={e.status_code} detail={e.detail}" + "Batch input file rejected for %s: status=%s detail=%s", file_id, e.status_code, e.detail ) raise except Exception as e: - verbose_proxy_logger.error(f"Error counting input file usage for {file_id}: {e}") + verbose_proxy_logger.error("Error counting input file usage for %s: %s", file_id, e) raise async def _enforce_batch_file_model_access( @@ -791,7 +791,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): # Only handle batch creation if call_type != "acreate_batch": verbose_proxy_logger.debug( - f"Batch rate limiter: Not handling batch creation rate limiting for call type: {call_type}" + "Batch rate limiter: Not handling batch creation rate limiting for call type: %s", call_type ) return data @@ -814,7 +814,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): custom_llm_provider = data.get("custom_llm_provider", "openai") # Count tokens and requests from input file - verbose_proxy_logger.debug(f"Counting tokens from batch input file: {input_file_id}") + verbose_proxy_logger.debug("Counting tokens from batch input file: %s", input_file_id) batch_usage = await self.count_input_file_usage( file_id=input_file_id, custom_llm_provider=custom_llm_provider, @@ -823,7 +823,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): ) verbose_proxy_logger.debug( - f"Batch input file usage - Tokens: {batch_usage.total_tokens}, Requests: {batch_usage.request_count}" + "Batch input file usage - Tokens: %s, Requests: %s", batch_usage.total_tokens, batch_usage.request_count ) # Store batch usage in data for later reference @@ -846,6 +846,6 @@ class _PROXY_BatchRateLimiter(CustomLogger): # Re-raise HTTP exceptions (rate limit exceeded) raise except Exception as e: - verbose_proxy_logger.error(f"Error in batch rate limiting: {e}", exc_info=True) + verbose_proxy_logger.error("Error in batch rate limiting: %s", e, exc_info=True) # Don't block the request if rate limiting fails return data diff --git a/litellm/proxy/hooks/batch_redis_get.py b/litellm/proxy/hooks/batch_redis_get.py index 377cd8d3d45..e2e7c1a3d27 100644 --- a/litellm/proxy/hooks/batch_redis_get.py +++ b/litellm/proxy/hooks/batch_redis_get.py @@ -84,7 +84,7 @@ class _PROXY_BatchRedisRequests(CustomLogger): raise e except Exception as e: verbose_proxy_logger.error( - f"litellm.proxy.hooks.batch_redis_get.py::async_pre_call_hook(): Exception occured - {e}" + "litellm.proxy.hooks.batch_redis_get.py::async_pre_call_hook(): Exception occured - %s", e ) verbose_proxy_logger.debug(traceback.format_exc()) diff --git a/litellm/proxy/hooks/cache_control_check.py b/litellm/proxy/hooks/cache_control_check.py index f2a0f06b95b..7f486200c2d 100644 --- a/litellm/proxy/hooks/cache_control_check.py +++ b/litellm/proxy/hooks/cache_control_check.py @@ -52,5 +52,5 @@ class _PROXY_CacheControlCheck(CustomLogger): raise e except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.cache_control_check.py::async_pre_call_hook(): Exception occured - {e}" + "litellm.proxy.hooks.cache_control_check.py::async_pre_call_hook(): Exception occured - %s", e ) diff --git a/litellm/proxy/hooks/dynamic_rate_limiter.py b/litellm/proxy/hooks/dynamic_rate_limiter.py index 5d890b6787c..c0adcfd1b75 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter.py @@ -68,7 +68,7 @@ class DynamicRateLimiterCache: await self.cache.async_set_cache_sadd(key=key_name, value=value, ttl=self.ttl) except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.hooks.dynamic_rate_limiter.py::async_set_cache_sadd(): Exception occured - {e}" + "litellm.proxy.hooks.dynamic_rate_limiter.py::async_set_cache_sadd(): Exception occured - %s", e ) raise e @@ -106,7 +106,9 @@ class _PROXY_DynamicRateLimitHandler(CustomLogger): weight: float = 1 if litellm.priority_reservation is None or priority not in litellm.priority_reservation: verbose_proxy_logger.error( - f"Priority Reservation not set. priority={priority}, but litellm.priority_reservation is {litellm.priority_reservation}." + "Priority Reservation not set. priority=%s, but litellm.priority_reservation is %s.", + priority, + litellm.priority_reservation, ) elif priority is not None and litellm.priority_reservation is not None: if os.getenv("LITELLM_LICENSE", None) is None: @@ -172,7 +174,7 @@ class _PROXY_DynamicRateLimitHandler(CustomLogger): ) except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.hooks.dynamic_rate_limiter.py::check_available_usage: Exception occurred - {e}" + "litellm.proxy.hooks.dynamic_rate_limiter.py::check_available_usage: Exception occurred - %s", e ) return None, None, None, None, None @@ -263,6 +265,6 @@ class _PROXY_DynamicRateLimitHandler(CustomLogger): ) except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.hooks.dynamic_rate_limiter.py::async_post_call_success_hook(): Exception occured - {e}" + "litellm.proxy.hooks.dynamic_rate_limiter.py::async_post_call_success_hook(): Exception occured - %s", e ) return response diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index 773abed1785..326a8e01407 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -173,7 +173,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): if total_weight > 1.0: normalized = {k: v / total_weight for k, v in weights.items()} - verbose_proxy_logger.debug(f"Normalized over-allocated priorities: {weights} -> {normalized}") + verbose_proxy_logger.debug("Normalized over-allocated priorities: %s -> %s", weights, normalized) return normalized return weights @@ -282,7 +282,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): return max_saturation except Exception as e: - verbose_proxy_logger.error(f"Error checking saturation for {model}: {e}") + verbose_proxy_logger.error("Error checking saturation for %s: %s", model, e) # Fail open: assume not saturated on error return 0.0 @@ -454,7 +454,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): parent_otel_span=user_api_key_dict.parent_otel_span, ) - verbose_proxy_logger.debug(f"Atomic check+increment response: {json.dumps(atomic_response, indent=2)}") + verbose_proxy_logger.debug("Atomic check+increment response: %s", json.dumps(atomic_response, indent=2)) if atomic_response["overall_code"] == "OVER_LIMIT": resolved_model, llm_provider = resolve_llm_provider_for_rate_limit(model) @@ -518,8 +518,8 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): None, ) verbose_proxy_logger.error( - f"Dynamic rate limiter: OVER_LIMIT response with unknown " - f"descriptor_key(s) — refusing request. response={atomic_response}" + "Dynamic rate limiter: OVER_LIMIT response with unknown descriptor_key(s) — refusing request. response=%s", + atomic_response, ) raise ProxyRateLimitError( detail={ @@ -610,7 +610,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): # Get model configuration model_group_info: ModelGroupInfo | None = self.llm_router.get_model_group_info(model_group=model) if model_group_info is None: - verbose_proxy_logger.debug(f"No model group info for {model}, allowing request") + verbose_proxy_logger.debug("No model group info for %s, allowing request", model) return None try: @@ -640,7 +640,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error in dynamic rate limiter: {e}, allowing request") + verbose_proxy_logger.error("Error in dynamic rate limiter: %s, allowing request", e) # Fail open on unexpected errors return None @@ -676,7 +676,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): return response except Exception as e: - verbose_proxy_logger.exception(f"Error in dynamic rate limiter v3 post-call hook: {e}") + verbose_proxy_logger.exception("Error in dynamic rate limiter v3 post-call hook: %s", e) return response async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -786,9 +786,11 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): SAFE_PRIORITIES = {"low", "medium", "high", "default"} logged_priority = key_priority if key_priority in SAFE_PRIORITIES else "REDACTED" verbose_proxy_logger.debug( - f"[Dynamic Rate Limiter] Incremented tokens by {total_tokens} for " - f"model={model_group}, priority={logged_priority}" + "[Dynamic Rate Limiter] Incremented tokens by %s for model=%s, priority=%s", + total_tokens, + model_group, + logged_priority, ) except Exception as e: - verbose_proxy_logger.exception(f"Error in dynamic rate limiter success event: {e}") + verbose_proxy_logger.exception("Error in dynamic rate limiter success event: %s", e) diff --git a/litellm/proxy/hooks/key_management_event_hooks.py b/litellm/proxy/hooks/key_management_event_hooks.py index 7ef1341a168..800d4874a98 100644 --- a/litellm/proxy/hooks/key_management_event_hooks.py +++ b/litellm/proxy/hooks/key_management_event_hooks.py @@ -51,7 +51,7 @@ class KeyManagementEventHooks: try: await KeyManagementEventHooks._send_key_created_email(response.model_dump(exclude_none=True)) except Exception as e: - verbose_proxy_logger.warning(f"Failed to send key created email: {e}") + verbose_proxy_logger.warning("Failed to send key created email: %s", e) # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True if litellm.store_audit_logs is True: @@ -84,7 +84,7 @@ class KeyManagementEventHooks: team_id=data.team_id, ) except Exception as e: - verbose_proxy_logger.warning(f"Failed to store virtual key in secret manager: {e}") + verbose_proxy_logger.warning("Failed to store virtual key in secret manager: %s", e) @staticmethod async def async_key_updated_hook( @@ -168,7 +168,7 @@ class KeyManagementEventHooks: new_secret_name, ) except Exception as e: - verbose_proxy_logger.warning(f"Failed to rotate virtual key in secret manager: {e}") + verbose_proxy_logger.warning("Failed to rotate virtual key in secret manager: %s", e) # Send key rotated email if configured - non-blocking, independent operation try: @@ -177,7 +177,7 @@ class KeyManagementEventHooks: existing_key_alias=existing_key_row.key_alias, ) except Exception as e: - verbose_proxy_logger.warning(f"Failed to send key rotated email: {e}") + verbose_proxy_logger.warning("Failed to send key rotated email: %s", e) # store the audit log if litellm.store_audit_logs is True and existing_key_row.token is not None: @@ -273,7 +273,7 @@ class KeyManagementEventHooks: description = getattr(litellm._key_management_settings, "description", None) optional_params = await KeyManagementEventHooks._get_secret_manager_optional_params(team_id) verbose_proxy_logger.debug( - f"Creating secret with {secret_name} and tags={tags} and description={description}" + "Creating secret with %s and tags=%s and description=%s", secret_name, tags, description ) await litellm.secret_manager_client.async_write_secret( @@ -355,7 +355,8 @@ class KeyManagementEventHooks: ) else: verbose_proxy_logger.warning( - f"KeyManagementEventHooks._delete_virtual_key_from_secret_manager: Key alias not found for key {key.token}. Skipping deletion from secret manager." + "KeyManagementEventHooks._delete_virtual_key_from_secret_manager: Key alias not found for key %s. Skipping deletion from secret manager.", + key.token, ) @staticmethod @@ -385,7 +386,7 @@ class KeyManagementEventHooks: user_api_key_cache=user_api_key_cache, ) except Exception as exc: # pragma: no cover - defensive logging - verbose_proxy_logger.debug(f"Unable to load team metadata for team_id={team_id}: {exc}") + verbose_proxy_logger.debug("Unable to load team metadata for team_id=%s: %s", team_id, exc) return None metadata = getattr(team_obj, "metadata", None) diff --git a/litellm/proxy/hooks/litellm_skills/main.py b/litellm/proxy/hooks/litellm_skills/main.py index 983a59657ce..770578d988b 100644 --- a/litellm/proxy/hooks/litellm_skills/main.py +++ b/litellm/proxy/hooks/litellm_skills/main.py @@ -96,7 +96,7 @@ class SkillsInjectionHook(CustomLogger): if not skills or not isinstance(skills, list): return data - verbose_proxy_logger.debug(f"SkillsInjectionHook: Processing {len(skills)} skills") + verbose_proxy_logger.debug("SkillsInjectionHook: Processing %s skills", len(skills)) litellm_skills: list[LiteLLM_SkillsTable] = [] anthropic_skills: list[dict[str, Any]] = [] @@ -116,7 +116,7 @@ class SkillsInjectionHook(CustomLogger): if db_skill: litellm_skills.append(db_skill) else: - verbose_proxy_logger.warning(f"SkillsInjectionHook: Skill '{skill_id}' not found in LiteLLM DB") + verbose_proxy_logger.warning("SkillsInjectionHook: Skill '%s' not found in LiteLLM DB", skill_id) else: # Native Anthropic skill - pass through anthropic_skills.append(skill) @@ -198,9 +198,10 @@ class SkillsInjectionHook(CustomLogger): data.pop("container", None) verbose_proxy_logger.debug( - f"SkillsInjectionHook: Messages API - converted {len(litellm_skills)} skills to Anthropic tools, " - f"injected {len(skill_contents)} skill contents, " - f"added litellm_code_execution tool with {len(all_module_paths)} modules" + "SkillsInjectionHook: Messages API - converted %s skills to Anthropic tools, injected %s skill contents, added litellm_code_execution tool with %s modules", + len(litellm_skills), + len(skill_contents), + len(all_module_paths), ) return data @@ -266,9 +267,10 @@ class SkillsInjectionHook(CustomLogger): data.pop("container", None) verbose_proxy_logger.debug( - f"SkillsInjectionHook: Non-Anthropic model - converted {len(litellm_skills)} skills to tools, " - f"injected {len(skill_contents)} skill contents, " - f"added execute_code tool with {len(all_module_paths)} modules" + "SkillsInjectionHook: Non-Anthropic model - converted %s skills to tools, injected %s skill contents, added execute_code tool with %s modules", + len(litellm_skills), + len(skill_contents), + len(all_module_paths), ) return data @@ -295,7 +297,7 @@ class SkillsInjectionHook(CustomLogger): user_api_key_dict=user_api_key_dict, ) except Exception as e: - verbose_proxy_logger.warning(f"SkillsInjectionHook: Error fetching skill {skill_id}: {e}") + verbose_proxy_logger.warning("SkillsInjectionHook: Error fetching skill %s: %s", skill_id, e) return None def _is_anthropic_model(self, model: str) -> bool: @@ -500,8 +502,9 @@ class SkillsInjectionHook(CustomLogger): # Check if we're done (no tool calls) if stop_reason != "tool_use" or not tool_calls: verbose_proxy_logger.debug( - f"SkillsInjectionHook: Loop completed after {iteration + 1} iterations, " - f"{len(generated_files)} files generated" + "SkillsInjectionHook: Loop completed after %s iterations, %s files generated", + iteration + 1, + len(generated_files), ) return self._attach_files_to_response(current_response, generated_files) @@ -536,7 +539,7 @@ class SkillsInjectionHook(CustomLogger): messages.append({"role": "user", "content": tool_results}) # Make next LLM call - verbose_proxy_logger.debug(f"SkillsInjectionHook: Making LLM call iteration {iteration + 2}") + verbose_proxy_logger.debug("SkillsInjectionHook: Making LLM call iteration %s", iteration + 2) try: current_response = await litellm.anthropic.acreate( model=model, @@ -548,10 +551,10 @@ class SkillsInjectionHook(CustomLogger): verbose_proxy_logger.error("SkillsInjectionHook: LLM call returned None") return self._attach_files_to_response(response, generated_files) except Exception as e: - verbose_proxy_logger.error(f"SkillsInjectionHook: LLM call failed: {e}") + verbose_proxy_logger.error("SkillsInjectionHook: LLM call failed: %s", e) return self._attach_files_to_response(response, generated_files) - verbose_proxy_logger.warning(f"SkillsInjectionHook: Max iterations ({self.max_iterations}) reached") + verbose_proxy_logger.warning("SkillsInjectionHook: Max iterations (%s) reached", self.max_iterations) return self._attach_files_to_response(current_response, generated_files) async def _execute_code( @@ -563,7 +566,7 @@ class SkillsInjectionHook(CustomLogger): ) -> str: """Execute code in sandbox and return result string.""" try: - verbose_proxy_logger.debug(f"SkillsInjectionHook: Executing code ({len(code)} chars)") + verbose_proxy_logger.debug("SkillsInjectionHook: Executing code (%s chars)", len(code)) exec_result = executor.execute(code=code, skill_files=skill_files) @@ -731,8 +734,9 @@ print('No executable skill module found') # Check if we're done (no tool calls) if stop_reason != "tool_calls" or not assistant_message.tool_calls: verbose_proxy_logger.debug( - f"SkillsInjectionHook: Code execution loop completed after " - f"{iteration + 1} iterations, {len(generated_files)} files generated" + "SkillsInjectionHook: Code execution loop completed after %s iterations, %s files generated", + iteration + 1, + len(generated_files), ) # Attach generated files to response return self._attach_files_to_response(current_response, generated_files) @@ -761,7 +765,7 @@ print('No executable skill module found') ) # Make next LLM call using the messages API - verbose_proxy_logger.debug(f"SkillsInjectionHook: Making LLM call iteration {iteration + 2}") + verbose_proxy_logger.debug("SkillsInjectionHook: Making LLM call iteration %s", iteration + 2) current_response = await litellm.anthropic.acreate( model=model, messages=messages, @@ -770,7 +774,7 @@ print('No executable skill module found') ) # Max iterations reached - verbose_proxy_logger.warning(f"SkillsInjectionHook: Max iterations ({self.max_iterations}) reached") + verbose_proxy_logger.warning("SkillsInjectionHook: Max iterations (%s) reached", self.max_iterations) return self._attach_files_to_response(current_response, generated_files) async def _execute_code_tool( @@ -785,7 +789,7 @@ print('No executable skill module found') args = json.loads(tool_call.function.arguments) code = args.get("code", "") - verbose_proxy_logger.debug(f"SkillsInjectionHook: Executing code ({len(code)} chars)") + verbose_proxy_logger.debug("SkillsInjectionHook: Executing code (%s chars)", len(code)) exec_result = executor.execute( code=code, @@ -811,7 +815,7 @@ print('No executable skill module found') tool_result += f"\n- {f['name']} ({len(file_content)} bytes)" verbose_proxy_logger.debug( - f"SkillsInjectionHook: Generated file {f['name']} ({len(file_content)} bytes)" + "SkillsInjectionHook: Generated file %s (%s bytes)", f["name"], len(file_content) ) if exec_result.get("error"): @@ -820,7 +824,7 @@ print('No executable skill module found') return tool_result except Exception as e: - verbose_proxy_logger.error(f"SkillsInjectionHook: Code execution failed: {e}") + verbose_proxy_logger.error("SkillsInjectionHook: Code execution failed: %s", e) return f"Code execution failed: {e}" def _attach_files_to_response( @@ -840,7 +844,7 @@ print('No executable skill module found') # Handle dict response (Anthropic/messages API format) if isinstance(response, dict): response["_litellm_generated_files"] = generated_files - verbose_proxy_logger.debug(f"SkillsInjectionHook: Attached {len(generated_files)} files to dict response") + verbose_proxy_logger.debug("SkillsInjectionHook: Attached %s files to dict response", len(generated_files)) return response # Handle object response (OpenAI format) @@ -855,7 +859,7 @@ print('No executable skill module found') response.model_extra = {} response.model_extra["_litellm_generated_files"] = generated_files - verbose_proxy_logger.debug(f"SkillsInjectionHook: Attached {len(generated_files)} files to response") + verbose_proxy_logger.debug("SkillsInjectionHook: Attached %s files to response", len(generated_files)) return response diff --git a/litellm/proxy/hooks/max_budget_limiter.py b/litellm/proxy/hooks/max_budget_limiter.py index 4a768b4e7de..7790dd8e175 100644 --- a/litellm/proxy/hooks/max_budget_limiter.py +++ b/litellm/proxy/hooks/max_budget_limiter.py @@ -75,5 +75,5 @@ class _PROXY_MaxBudgetLimiter(CustomLogger): raise e except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.max_budget_limiter.py::async_pre_call_hook(): Exception occured - {e}" + "litellm.proxy.hooks.max_budget_limiter.py::async_pre_call_hook(): Exception occured - %s", e ) diff --git a/litellm/proxy/hooks/mcp_semantic_filter/hook.py b/litellm/proxy/hooks/mcp_semantic_filter/hook.py index 10293bc5e5f..0356cbfa702 100644 --- a/litellm/proxy/hooks/mcp_semantic_filter/hook.py +++ b/litellm/proxy/hooks/mcp_semantic_filter/hook.py @@ -62,8 +62,9 @@ class SemanticToolFilterHook(CustomLogger): self.filter = semantic_filter verbose_proxy_logger.debug( - f"Initialized SemanticToolFilterHook with filter: " - f"enabled={semantic_filter.enabled}, top_k={semantic_filter.top_k}" + "Initialized SemanticToolFilterHook with filter: enabled=%s, top_k=%s", + semantic_filter.enabled, + semantic_filter.top_k, ) def _should_expand_mcp_tools(self, tools: list[Any]) -> bool: @@ -114,22 +115,24 @@ class SemanticToolFilterHook(CustomLogger): if hasattr(tool, "model_dump"): tool_dict = tool.model_dump(exclude_none=True) verbose_proxy_logger.debug( - f"Converted Pydantic tool to dict: {type(tool).__name__} -> dict with keys: {list(tool_dict.keys())}" + "Converted Pydantic tool to dict: %s -> dict with keys: %s", + type(tool).__name__, + list(tool_dict.keys()), ) openai_tools_as_dicts.append(tool_dict) elif hasattr(tool, "dict"): tool_dict = tool.dict(exclude_none=True) - verbose_proxy_logger.debug(f"Converted Pydantic tool (v1) to dict: {type(tool).__name__} -> dict") + verbose_proxy_logger.debug("Converted Pydantic tool (v1) to dict: %s -> dict", type(tool).__name__) openai_tools_as_dicts.append(tool_dict) elif isinstance(tool, dict): - verbose_proxy_logger.debug(f"Tool is already a dict with keys: {list(tool.keys())}") + verbose_proxy_logger.debug("Tool is already a dict with keys: %s", list(tool.keys())) openai_tools_as_dicts.append(tool) else: - verbose_proxy_logger.warning(f"Tool is unknown type: {type(tool)}, passing as-is") + verbose_proxy_logger.warning("Tool is unknown type: %s, passing as-is", type(tool)) openai_tools_as_dicts.append(tool) verbose_proxy_logger.debug( - f"Expanded {len(mcp_tools)} MCP reference(s) to {len(openai_tools_as_dicts)} tools (all as dicts)" + "Expanded %s MCP reference(s) to %s tools (all as dicts)", len(mcp_tools), len(openai_tools_as_dicts) ) return openai_tools_as_dicts @@ -235,13 +238,14 @@ class SemanticToolFilterHook(CustomLogger): metadata["litellm_semantic_filter_tools"] = tool_names_csv verbose_proxy_logger.info( - f"Semantic tool filter: {filter_stats} MCP tools " - f"({len(native_tools)} native preserved, " - f"{len(filtered_tools)} total)" + "Semantic tool filter: %s MCP tools (%s native preserved, %s total)", + filter_stats, + len(native_tools), + len(filtered_tools), ) else: verbose_proxy_logger.info( - f"Semantic tool filter: all {len(native_tools)} tools are native, no MCP filtering applied" + "Semantic tool filter: all %s tools are native, no MCP filtering applied", len(native_tools) ) def _emit_filter_metadata_safe( @@ -266,7 +270,8 @@ class SemanticToolFilterHook(CustomLogger): ) except Exception as e: verbose_proxy_logger.warning( - f"Failed to emit semantic filter metadata: {e}", + "Failed to emit semantic filter metadata: %s", + e, exc_info=True, ) @@ -284,7 +289,7 @@ class SemanticToolFilterHook(CustomLogger): tools list to only include semantically relevant tools. """ if call_type not in ("completion", "acompletion", "aresponses"): - verbose_proxy_logger.debug(f"Skipping semantic filter for call_type={call_type}") + verbose_proxy_logger.debug("Skipping semantic filter for call_type=%s", call_type) return None tools = data.get("tools") @@ -321,16 +326,17 @@ class SemanticToolFilterHook(CustomLogger): filtered_tools=narrowed_tools, ) verbose_proxy_logger.info( - f"Expanded MCP references to {len(expanded_tools)} tools " - f"({len(native_tools_before_expand)} native preserved), " - f"semantic filter selected {len(filtered_expanded_tools)}" + "Expanded MCP references to %s tools (%s native preserved), semantic filter selected %s", + len(expanded_tools), + len(native_tools_before_expand), + len(filtered_expanded_tools), ) return data except SemanticToolFilterContextWindowError as e: raise HTTPException(status_code=400, detail={"error": str(e)}) from e except Exception as e: - verbose_proxy_logger.error(f"Failed to expand MCP references: {e}", exc_info=True) + verbose_proxy_logger.error("Failed to expand MCP references: %s", e, exc_info=True) return None messages = data.get("messages", []) @@ -361,9 +367,10 @@ class SemanticToolFilterHook(CustomLogger): native_tools.append(t) verbose_proxy_logger.debug( - f"Applying semantic filter: {len(mcp_tools)} MCP tools, " - f"{len(native_tools)} native tools, " - f"query: '{user_query[:50]}...'" + "Applying semantic filter: %s MCP tools, %s native tools, query: '%s...'", + len(mcp_tools), + len(native_tools), + user_query[:50], ) if mcp_tools: @@ -404,7 +411,7 @@ class SemanticToolFilterHook(CustomLogger): except SemanticToolFilterContextWindowError as e: raise HTTPException(status_code=400, detail={"error": str(e)}) from e except Exception as e: - verbose_proxy_logger.warning(f"Semantic tool filter hook failed: {e}. Proceeding with all tools.") + verbose_proxy_logger.warning("Semantic tool filter hook failed: %s. Proceeding with all tools.", e) return None async def async_post_call_response_headers_hook( @@ -497,18 +504,19 @@ class SemanticToolFilterHook(CustomLogger): hook = SemanticToolFilterHook(semantic_filter) verbose_proxy_logger.info( - f"✅ MCP Semantic Tool Filter enabled: " - f"embedding_model={embedding_model}, top_k={top_k}, " - f"similarity_threshold={similarity_threshold}" + "✅ MCP Semantic Tool Filter enabled: embedding_model=%s, top_k=%s, similarity_threshold=%s", + embedding_model, + top_k, + similarity_threshold, ) return hook except ImportError as e: verbose_proxy_logger.warning( - f"semantic-router not installed. Install with: pip install 'litellm[semantic-router]'. Error: {e}" + "semantic-router not installed. Install with: pip install 'litellm[semantic-router]'. Error: %s", e ) return None except Exception as e: - verbose_proxy_logger.exception(f"Failed to initialize MCP semantic tool filter: {e}") + verbose_proxy_logger.exception("Failed to initialize MCP semantic tool filter: %s", e) return None diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py index 2aeb505bd97..630fcdb8fe7 100644 --- a/litellm/proxy/hooks/model_max_budget_limiter.py +++ b/litellm/proxy/hooks/model_max_budget_limiter.py @@ -56,7 +56,7 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): model=model, internal_model_max_budget=internal_model_max_budget ) if _current_model_budget_info is None: - verbose_proxy_logger.debug(f"Model {model} not found in internal_model_max_budget") + verbose_proxy_logger.debug("Model %s not found in internal_model_max_budget", model) return True # check if current model is within budget @@ -122,7 +122,7 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): model=model, internal_model_max_budget=internal_model_max_budget ) if _current_model_budget_info is None: - verbose_proxy_logger.debug(f"Model {model} not found in end_user_model_max_budget") + verbose_proxy_logger.debug("Model %s not found in end_user_model_max_budget", model) return True # check if current model is within budget diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index 04f34d0e9cf..081200366d6 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -69,7 +69,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): rate_limit_type: Literal["key", "model_per_key", "user", "customer", "team"], values_to_update_in_cache: list[tuple[Any, Any]], ) -> dict: - verbose_proxy_logger.info(f"Current Usage of {rate_limit_type} in this minute: {current}") + verbose_proxy_logger.info("Current Usage of %s in this minute: %s", rate_limit_type, current) if current is None: if max_parallel_requests == 0 or tpm_limit == 0 or rpm_limit == 0: # base case — at least one dimension is set to 0 (effectively @@ -776,7 +776,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): litellm_parent_otel_span=litellm_parent_otel_span, ) # save in cache for up to 1 min. except Exception as e: - verbose_proxy_logger.exception(f"Inside Parallel Request Limiter: An exception occurred - {e}") + verbose_proxy_logger.exception("Inside Parallel Request Limiter: An exception occurred - %s", e) async def get_internal_user_object( self, diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 98f1e650845..af4818dec02 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -490,7 +490,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): parallel_request_limiter=self, ) except Exception as e: - verbose_proxy_logger.debug(f"Could not load batch rate limiter: {e}") + verbose_proxy_logger.debug("Could not load batch rate limiter: %s", e) return self._batch_rate_limiter def _get_current_time(self) -> datetime: @@ -579,9 +579,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): total_estimated = estimated_input_tokens + max_tokens_estimate verbose_proxy_logger.debug( - f"TPM reservation estimate: input={estimated_input_tokens}, " - f"max_tokens={max_tokens_estimate} (explicit={explicit_max_tokens is not None}), " - f"total={total_estimated}" + "TPM reservation estimate: input=%s, max_tokens=%s (explicit=%s), total=%s", + estimated_input_tokens, + max_tokens_estimate, + explicit_max_tokens is not None, + total_estimated, ) return total_estimated @@ -808,7 +810,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) all_cache_values.extend(group_cache_values) except Exception as e: - verbose_proxy_logger.warning(f"Redis Lua script failed for hash tag {hash_tag}: {e}") + verbose_proxy_logger.warning("Redis Lua script failed for hash tag %s: %s", hash_tag, e) # Fallback to in-memory cache for this group group_cache_values = await self.in_memory_cache_sliding_window( keys=group_keys, @@ -1055,7 +1057,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) counts = [max(0, int(value)) for value in raw_counts] except Exception as e: # noqa: BLE001 - any Redis/Lua failure degrades to the local mirror, never a 500 - verbose_proxy_logger.warning(f"parallel_count_script failed, using local mirror: {e}") + verbose_proxy_logger.warning("parallel_count_script failed, using local mirror: %s", e) counts = await self._read_local_gauge_counts(gauge_keys, parent_otel_span) else: counts = await self._read_local_gauge_counts(gauge_keys, parent_otel_span) @@ -1085,7 +1087,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ], ) except Exception as e: # noqa: BLE001 - any Redis/Lua failure degrades to in-memory enforcement, never a 500 - verbose_proxy_logger.warning(f"parallel_acquire_script failed, falling back to in-memory gauge: {e}") + verbose_proxy_logger.warning("parallel_acquire_script failed, falling back to in-memory gauge: %s", e) async with self._check_and_increment_lock: return await self._acquire_parallel_slots_in_memory(gauges, slot_id, parent_otel_span) if int(raw[0]) == 1: @@ -1212,7 +1214,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) return except Exception as e: # noqa: BLE001 - any Redis/Lua failure degrades to the in-memory release, never a 500 - verbose_proxy_logger.warning(f"parallel_release_script failed, falling back to in-memory release: {e}") + verbose_proxy_logger.warning("parallel_release_script failed, falling back to in-memory release: %s", e) async with self._check_and_increment_lock: for counter_key in counter_keys: @@ -1387,12 +1389,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # to its pre-call state, then fall back to in-memory for the # whole call (counters there are independent of Redis). verbose_proxy_logger.error( - f"atomic_check_and_increment_by_n: Redis Lua execution " - f"failed ({type(e).__name__}: {e}). Refunding " - f"{len(applied)} prior descriptors and falling back to " - f"in-memory enforcement — counters will diverge from " - f"Redis until window expires (window_size=" - f"{self.window_size}s)." + "atomic_check_and_increment_by_n: Redis Lua execution failed (%s: %s). Refunding %s prior descriptors and falling back to in-memory enforcement — counters will diverge from Redis until window expires (window_size=%ss).", + type(e).__name__, + e, + len(applied), + self.window_size, ) await self._refund_applied_descriptor_groups(applied) flat_meta: list[dict[str, Any]] = [m for _k, _a, group_meta in descriptor_groups for m in group_meta] @@ -1434,7 +1435,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) except Exception as e: verbose_proxy_logger.warning( - f"Failed to refund {entry['counter_key']} on cross-descriptor rollback: {e}" + "Failed to refund %s on cross-descriptor rollback: %s", entry["counter_key"], e ) def _build_atomic_response( @@ -2227,18 +2228,20 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if failure_count > DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE: verbose_proxy_logger.debug( - f"[Dynamic Rate Limit] Deployment {deployment_id} has {failure_count} failures " - f"in current minute - enforcing rate limits for model {model}" + "[Dynamic Rate Limit] Deployment %s has %s failures in current minute - enforcing rate limits for model %s", + deployment_id, + failure_count, + model, ) return True verbose_proxy_logger.debug( - f"[Dynamic Rate Limit] No failures detected for model {model} - allowing dynamic exceeding" + "[Dynamic Rate Limit] No failures detected for model %s - allowing dynamic exceeding", model ) return False except Exception as e: - verbose_proxy_logger.debug(f"Error checking model failure status: {e}, defaulting to enforce limits") + verbose_proxy_logger.debug("Error checking model failure status: %s, defaulting to enforce limits", e) # Fail safe: enforce limits if we can't check return True @@ -2573,7 +2576,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if stored_response is not None: stored_response["statuses"].extend(tpm_response["statuses"]) - verbose_proxy_logger.debug(f"TPM tokens reserved: {estimated_tokens} for model {requested_model}") + verbose_proxy_logger.debug( + "TPM tokens reserved: %s for model %s", estimated_tokens, requested_model + ) def _create_pipeline_operations( self, @@ -2704,8 +2709,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ttl_value = op["ttl"] if op["ttl"] is not None else 0 verbose_proxy_logger.debug( - f"Executing TTL-preserving increment for key={op['key']}, " - f"increment={op['increment_value']}, ttl={ttl_value}" + "Executing TTL-preserving increment for key=%s, increment=%s, ttl=%s", + op["key"], + op["increment_value"], + ttl_value, ) keys.append(op["key"]) args.extend([op["increment_value"], ttl_value]) @@ -2740,11 +2747,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): await self._execute_token_increment_script(pipeline_operations) verbose_proxy_logger.debug( - f"Successfully executed TTL-preserving increment for {len(pipeline_operations)} keys" + "Successfully executed TTL-preserving increment for %s keys", len(pipeline_operations) ) except Exception as e: - verbose_proxy_logger.warning(f"TTL preservation failed, falling back to regular pipeline: {e}") + verbose_proxy_logger.warning("TTL preservation failed, falling back to regular pipeline: %s", e) # Fallback to regular pipeline on error await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline( increment_list=pipeline_operations, @@ -2950,9 +2957,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) if reserved_tokens > 0 and total_tokens < reserved_tokens: verbose_proxy_logger.debug( - f"Releasing unused TPM budget on success: " - f"reserved={reserved_tokens}, actual={total_tokens}, " - f"release={reserved_tokens - total_tokens}" + "Releasing unused TPM budget on success: reserved=%s, actual=%s, release=%s", + reserved_tokens, + total_tokens, + reserved_tokens - total_tokens, ) pipeline_operations.extend( self._build_reservation_aware_tpm_ops( @@ -3001,7 +3009,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) except Exception as e: - verbose_proxy_logger.exception(f"Error in rate limit success event: {e}") + verbose_proxy_logger.exception("Error in rate limit success event: %s", e) async def async_logging_hook( self, @@ -3095,7 +3103,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if stash is not None and not stash.reservation_released: reserved_tokens = stash.reserved_tokens if stash is not None and reserved_tokens > 0: - verbose_proxy_logger.debug(f"Releasing reserved TPM tokens on failure: {reserved_tokens}") + verbose_proxy_logger.debug("Releasing reserved TPM tokens on failure: %s", reserved_tokens) # Refund only against the scopes the reservation actually # charged. _build_reservation_aware_tpm_ops with # actual_tokens=0 emits -reserved on reserved scopes and 0 @@ -3118,7 +3126,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if stash is not None and reserved_tokens > 0: stash.reservation_released = True except Exception as e: - verbose_proxy_logger.exception(f"Error in rate limit failure event: {e}") + verbose_proxy_logger.exception("Error in rate limit failure event: %s", e) async def async_release_max_parallel_requests_on_disconnect( self, @@ -3183,7 +3191,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) except Exception as e: - verbose_proxy_logger.exception(f"Error in rate limit post-call hook: {e}") + verbose_proxy_logger.exception("Error in rate limit post-call hook: %s", e) async def async_post_call_failure_hook( self, @@ -3231,12 +3239,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): reserved_tokens=reserved_tokens, ) if ops: - verbose_proxy_logger.debug(f"Releasing reserved TPM tokens on proxy-level rejection: {reserved_tokens}") + verbose_proxy_logger.debug( + "Releasing reserved TPM tokens on proxy-level rejection: %s", reserved_tokens + ) await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline( increment_list=ops, litellm_parent_otel_span=user_api_key_dict.parent_otel_span, ) stash.reservation_released = True except Exception as e: - verbose_proxy_logger.exception(f"Error releasing TPM reservation on post-call failure: {e}") + verbose_proxy_logger.exception("Error releasing TPM reservation on post-call failure: %s", e) return diff --git a/litellm/proxy/hooks/prompt_injection_detection.py b/litellm/proxy/hooks/prompt_injection_detection.py index e7192b9b063..1cb80bc37f6 100644 --- a/litellm/proxy/hooks/prompt_injection_detection.py +++ b/litellm/proxy/hooks/prompt_injection_detection.py @@ -197,7 +197,7 @@ class _OPTIONAL_PromptInjectionDetection(CustomLogger): raise e except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {e}" + "litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - %s", e ) async def async_moderation_hook( # type: ignore diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 0319a680714..e58923fa416 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -195,7 +195,9 @@ class _ProxyDBLogger(CustomLogger): verbose_proxy_logger.debug("INSIDE _PROXY_track_cost_callback") try: verbose_proxy_logger.debug( - f"kwargs stream: {kwargs.get('stream', None)} + complete streaming response: {kwargs.get('complete_streaming_response', None)}" + "kwargs stream: %s + complete streaming response: %s", + kwargs.get("stream", None), + kwargs.get("complete_streaming_response", None), ) parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs=kwargs) litellm_params = kwargs.get("litellm_params", {}) or {} @@ -225,10 +227,14 @@ class _ProxyDBLogger(CustomLogger): user_api_key = metadata.get("user_api_key", None) if kwargs.get("cache_hit", False) is True: response_cost = 0.0 - verbose_proxy_logger.debug(f"Cache Hit: response_cost {response_cost}, for user_id {user_id}") + verbose_proxy_logger.debug("Cache Hit: response_cost %s, for user_id %s", response_cost, user_id) verbose_proxy_logger.debug( - f"user_api_key {user_api_key}, user_id {user_id}, team_id {team_id}, end_user_id {end_user_id}" + "user_api_key %s, user_id %s, team_id %s, end_user_id %s", + user_api_key, + user_id, + team_id, + end_user_id, ) call_type: str | None = kwargs.get("call_type") if _should_track_cost_callback( diff --git a/litellm/proxy/hooks/responses_id_security.py b/litellm/proxy/hooks/responses_id_security.py index 8b6bfd95892..5a8abcc9324 100644 --- a/litellm/proxy/hooks/responses_id_security.py +++ b/litellm/proxy/hooks/responses_id_security.py @@ -83,7 +83,9 @@ class ResponsesIDSecurity(CustomLogger): if response_id_user_id and response_id_user_id != user_api_key_dict.user_id: if general_settings.get("disable_responses_id_security", False): verbose_proxy_logger.debug( - f"Responses ID Security is disabled. User {user_api_key_dict.user_id} is accessing response id {response_id_user_id} which is not associated with them." + "Responses ID Security is disabled. User %s is accessing response id %s which is not associated with them.", + user_api_key_dict.user_id, + response_id_user_id, ) return True raise HTTPException( @@ -94,7 +96,10 @@ class ResponsesIDSecurity(CustomLogger): if response_id_team_id and response_id_team_id != user_api_key_dict.team_id: if general_settings.get("disable_responses_id_security", False): verbose_proxy_logger.debug( - f"Responses ID Security is disabled. Response belongs to team {response_id_team_id} but user {user_api_key_dict.user_id} is accessing it with team id {user_api_key_dict.team_id}." + "Responses ID Security is disabled. Response belongs to team %s but user %s is accessing it with team id %s.", + response_id_team_id, + user_api_key_dict.user_id, + user_api_key_dict.team_id, ) return True raise HTTPException( diff --git a/litellm/proxy/hooks/user_management_event_hooks.py b/litellm/proxy/hooks/user_management_event_hooks.py index b242f763fcb..b434bc001b9 100644 --- a/litellm/proxy/hooks/user_management_event_hooks.py +++ b/litellm/proxy/hooks/user_management_event_hooks.py @@ -71,7 +71,7 @@ class UserManagementEventHooks: ) ) except Exception as e: - verbose_proxy_logger.warning(f"Unable to create audit log for user on `/user/new` - {e}") + verbose_proxy_logger.warning("Unable to create audit log for user on `/user/new` - %s", e) @staticmethod async def async_send_user_invitation_email( diff --git a/litellm/proxy/image_endpoints/endpoints.py b/litellm/proxy/image_endpoints/endpoints.py index 36f702e1b4b..04e3982b412 100644 --- a/litellm/proxy/image_endpoints/endpoints.py +++ b/litellm/proxy/image_endpoints/endpoints.py @@ -185,7 +185,7 @@ async def image_generation( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.image_generation(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.image_generation(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 6864caccea2..5b9a9c9a202 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -905,7 +905,7 @@ class LiteLLMProxyRequestSetup: user = LiteLLMProxyRequestSetup._get_case_insensitive_header(headers, header_name) if user is not None: - verbose_logger.info(f'found user "{user}" in header "{header_name}"') + verbose_logger.info('found user "%s" in header "%s"', user, header_name) return user @@ -918,7 +918,7 @@ class LiteLLMProxyRequestSetup: return None for header, value in headers.items(): if header.lower() == "openai-organization": - verbose_logger.info(f"found openai org id: {value}, sending to llm") + verbose_logger.info("found openai org id: %s, sending to llm", value) return value return None @@ -1059,14 +1059,14 @@ class LiteLLMProxyRequestSetup: if agent_id_from_header: metadata_from_headers["agent_id"] = agent_id_from_header - verbose_proxy_logger.debug(f"Extracted agent_id from header: {agent_id_from_header}") + verbose_proxy_logger.debug("Extracted agent_id from header: %s", agent_id_from_header) if chain_id: metadata_from_headers["trace_id"] = chain_id metadata_from_headers["session_id"] = chain_id data["litellm_session_id"] = chain_id data["litellm_trace_id"] = chain_id - verbose_proxy_logger.debug(f"Extracted chain_id from header (trace-id/session-id): {chain_id}") + verbose_proxy_logger.debug("Extracted chain_id from header (trace-id/session-id): %s", chain_id) else: body_metadata = data.get("metadata") session_id = _get_anthropic_session_id_from_metadata(body_metadata) @@ -1475,8 +1475,8 @@ async def add_litellm_data_to_request( allow_client_message_redaction_opt_out=_allow_client_message_redaction_opt_out, ) _logging_safe_headers = redact_credential_headers(_headers) - verbose_proxy_logger.debug(f"Request Headers: {_logging_safe_headers}") - verbose_proxy_logger.debug(f"Raw Headers: {_raw_headers}") + verbose_proxy_logger.debug("Request Headers: %s", _logging_safe_headers) + verbose_proxy_logger.debug("Raw Headers: %s", _raw_headers) if forward_llm_auth and "x-api-key" in _headers: data["api_key"] = _headers["x-api-key"] @@ -1579,7 +1579,7 @@ async def add_litellm_data_to_request( data["metadata"] = safe_json_loads(data["metadata"]) if not isinstance(data["metadata"], dict): verbose_proxy_logger.warning( - f"Failed to parse 'metadata' as JSON dict. Received value: {data['metadata']}" + "Failed to parse 'metadata' as JSON dict. Received value: %s", data["metadata"] ) # requester_metadata is snapshotted AFTER the strip below so # downstream consumers (e.g. PANW guardrail reading user_ip / @@ -1592,7 +1592,7 @@ async def add_litellm_data_to_request( parsed_litellm_metadata = safe_json_loads(data["litellm_metadata"]) if not isinstance(parsed_litellm_metadata, dict): verbose_proxy_logger.warning( - f"Failed to parse 'litellm_metadata' as JSON dict. Received value: {data['litellm_metadata']}" + "Failed to parse 'litellm_metadata' as JSON dict. Received value: %s", data["litellm_metadata"] ) else: data["litellm_metadata"] = parsed_litellm_metadata @@ -2409,7 +2409,7 @@ def _add_guardrails_from_policies_in_metadata( if not policy_names: return - verbose_proxy_logger.debug(f"Policy engine: resolving guardrails from key/team policies: {policy_names}") + verbose_proxy_logger.debug("Policy engine: resolving guardrails from key/team policies: %s", policy_names) # Check if policy registry is initialized registry = get_policy_registry() @@ -2434,10 +2434,10 @@ def _add_guardrails_from_policies_in_metadata( ) resolved_guardrails.update(resolved_policy.guardrails) verbose_proxy_logger.debug( - f"Policy engine: resolved guardrails from policy '{policy_name}': {resolved_policy.guardrails}" + "Policy engine: resolved guardrails from policy '%s': %s", policy_name, resolved_policy.guardrails ) else: - verbose_proxy_logger.warning(f"Policy engine: policy '{policy_name}' not found in registry") + verbose_proxy_logger.warning("Policy engine: policy '%s' not found in registry", policy_name) if not resolved_guardrails: return @@ -2461,7 +2461,7 @@ def _add_guardrails_from_policies_in_metadata( data[metadata_variable_name]["applied_policies"].extend(list(policy_names)) verbose_proxy_logger.debug( - f"Policy engine: added guardrails from key/team policies to request metadata: {list(resolved_guardrails)}" + "Policy engine: added guardrails from key/team policies to request metadata: %s", list(resolved_guardrails) ) @@ -2592,13 +2592,13 @@ def _match_and_track_policies( matching_policy_names = [m["policy_name"] for m in matches_with_reasons] policy_reasons = {m["policy_name"]: m["matched_via"] for m in matches_with_reasons} - verbose_proxy_logger.debug(f"Policy engine: matched policies via attachments: {matching_policy_names}") + verbose_proxy_logger.debug("Policy engine: matched policies via attachments: %s", matching_policy_names) # Combine attachment-based policies with dynamic request body policies all_policy_names = set(matching_policy_names) if request_body_policies and isinstance(request_body_policies, list): all_policy_names.update(request_body_policies) - verbose_proxy_logger.debug(f"Policy engine: added dynamic policies from request body: {request_body_policies}") + verbose_proxy_logger.debug("Policy engine: added dynamic policies from request body: %s", request_body_policies) if not all_policy_names: return [], {} @@ -2610,7 +2610,7 @@ def _match_and_track_policies( policies=policies_override, ) - verbose_proxy_logger.debug(f"Policy engine: applied policies (conditions matched): {applied_policy_names}") + verbose_proxy_logger.debug("Policy engine: applied policies (conditions matched): %s", applied_policy_names) # Track applied policies in metadata for response headers for policy_name in applied_policy_names: @@ -2641,7 +2641,7 @@ def _apply_resolved_guardrails_to_metadata( policy_names=policy_names, ) - verbose_proxy_logger.debug(f"Policy engine: resolved guardrails: {resolved_guardrails}") + verbose_proxy_logger.debug("Policy engine: resolved guardrails: %s", resolved_guardrails) # Resolve pipelines from matching policies pipelines = PolicyResolver.resolve_pipelines_for_context( @@ -2661,7 +2661,9 @@ def _apply_resolved_guardrails_to_metadata( data[metadata_variable_name]["_guardrail_pipelines"] = pipelines data[metadata_variable_name]["_pipeline_managed_guardrails"] = pipeline_managed_guardrails verbose_proxy_logger.debug( - f"Policy engine: resolved {len(pipelines)} pipeline(s), managed guardrails: {pipeline_managed_guardrails}" + "Policy engine: resolved %s pipeline(s), managed guardrails: %s", + len(pipelines), + pipeline_managed_guardrails, ) if not resolved_guardrails and not pipelines: @@ -2678,7 +2680,7 @@ def _apply_resolved_guardrails_to_metadata( combined -= pipeline_managed_guardrails data[metadata_variable_name]["guardrails"] = list(combined) - verbose_proxy_logger.debug(f"Policy engine: added guardrails to request metadata: {list(combined)}") + verbose_proxy_logger.debug("Policy engine: added guardrails to request metadata: %s", list(combined)) async def add_guardrails_from_policy_engine( @@ -2714,8 +2716,9 @@ async def add_guardrails_from_policy_engine( registry = get_policy_registry() verbose_proxy_logger.debug( - f"Policy engine: registry initialized={registry.is_initialized()}, " - f"policy_count={len(registry.get_all_policies())}" + "Policy engine: registry initialized=%s, policy_count=%s", + registry.is_initialized(), + len(registry.get_all_policies()), ) if not registry.is_initialized(): verbose_proxy_logger.debug("Policy engine not initialized, skipping policy matching") @@ -2733,8 +2736,11 @@ async def add_guardrails_from_policy_engine( ) verbose_proxy_logger.debug( - f"Policy engine: matching policies for context team_alias={context.team_alias}, " - f"key_alias={context.key_alias}, model={context.model}, tags={context.tags}" + "Policy engine: matching policies for context team_alias=%s, key_alias=%s, model=%s, tags=%s", + context.team_alias, + context.key_alias, + context.model, + context.tags, ) # Separate policy names from policy version IDs (policy_) @@ -2760,9 +2766,9 @@ async def add_guardrails_from_policy_engine( pname, policy = result merged_policies[pname] = policy fetched_policy_names.append(pname) - verbose_proxy_logger.debug(f"Policy engine: loaded version by ID policy_{policy_id} -> {pname}") + verbose_proxy_logger.debug("Policy engine: loaded version by ID policy_%s -> %s", policy_id, pname) else: - verbose_proxy_logger.debug(f"Policy engine: policy version {policy_id} not found in cache, skipping") + verbose_proxy_logger.debug("Policy engine: policy version %s not found in cache, skipping", policy_id) # Build request body list: names + policy names from fetched versions request_body_policies = request_body_names + fetched_policy_names diff --git a/litellm/proxy/management_endpoints/cache_settings_endpoints.py b/litellm/proxy/management_endpoints/cache_settings_endpoints.py index 22438e6f336..9b803d94d3c 100644 --- a/litellm/proxy/management_endpoints/cache_settings_endpoints.py +++ b/litellm/proxy/management_endpoints/cache_settings_endpoints.py @@ -386,7 +386,8 @@ class CacheSettingsManager: verbose_proxy_logger.info("Cache settings initialized from database") except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.management_endpoints.cache_settings_endpoints.py::CacheSettingsManager::init_cache_settings_in_db - {e}" + "litellm.proxy.management_endpoints.cache_settings_endpoints.py::CacheSettingsManager::init_cache_settings_in_db - %s", + e, ) @staticmethod @@ -480,7 +481,7 @@ async def get_cache_settings( redis_type_descriptions=REDIS_TYPE_DESCRIPTIONS, ) except Exception as e: - verbose_proxy_logger.error(f"Error fetching cache settings: {e}") + verbose_proxy_logger.error("Error fetching cache settings: %s", e) raise HTTPException(status_code=500, detail=f"Error fetching cache settings: {e}") @@ -539,7 +540,7 @@ async def test_cache_connection( return CacheTestResponse(**result) except Exception as e: - verbose_proxy_logger.error(f"Error testing cache connection: {e}") + verbose_proxy_logger.error("Error testing cache connection: %s", e) return CacheTestResponse( status="failed", message=f"Cache connection test failed: {e}", @@ -652,5 +653,5 @@ async def update_cache_settings( "settings": _redact_credentials(cache_settings), } except Exception as e: - verbose_proxy_logger.error(f"Error updating cache settings: {e}") + verbose_proxy_logger.error("Error updating cache settings: %s", e) raise HTTPException(status_code=500, detail=f"Error updating cache settings: {e}") diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 132a8409e06..1f83fa6a65c 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -1001,7 +1001,7 @@ async def get_daily_activity( ) except Exception as e: - verbose_proxy_logger.exception(f"Error fetching daily activity: {e}") + verbose_proxy_logger.exception("Error fetching daily activity: %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to fetch analytics: {e}"}, @@ -1091,7 +1091,7 @@ async def get_daily_activity_aggregated( ) except Exception as e: - verbose_proxy_logger.exception(f"Error fetching aggregated daily activity: {e}") + verbose_proxy_logger.exception("Error fetching aggregated daily activity: %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to fetch analytics: {e}"}, diff --git a/litellm/proxy/management_endpoints/common_utils.py b/litellm/proxy/management_endpoints/common_utils.py index abb1b686d1d..387d4421188 100644 --- a/litellm/proxy/management_endpoints/common_utils.py +++ b/litellm/proxy/management_endpoints/common_utils.py @@ -222,7 +222,7 @@ async def _user_has_admin_privileges( except Exception as e: # If there's an error checking, default to False for security - verbose_proxy_logger.debug(f"Error checking admin privileges for user {user_api_key_dict.user_id}: {e}") + verbose_proxy_logger.debug("Error checking admin privileges for user %s: %s", user_api_key_dict.user_id, e) return False return False @@ -366,7 +366,7 @@ async def admin_can_invite_user( return False except Exception as e: - verbose_proxy_logger.debug(f"Error checking invite permission for user {user_api_key_dict.user_id}: {e}") + verbose_proxy_logger.debug("Error checking invite permission for user %s: %s", user_api_key_dict.user_id, e) return False diff --git a/litellm/proxy/management_endpoints/cost_tracking_settings.py b/litellm/proxy/management_endpoints/cost_tracking_settings.py index f2295452f4d..d199cea5fd3 100644 --- a/litellm/proxy/management_endpoints/cost_tracking_settings.py +++ b/litellm/proxy/management_endpoints/cost_tracking_settings.py @@ -59,7 +59,7 @@ def _resolve_model_for_cost_lookup(model: str) -> tuple[str, str | None]: # Check base_model first (needed for Azure custom deployment names) base_model = model_info.get("base_model") or litellm_params.get("base_model") if base_model: - verbose_proxy_logger.debug(f"Resolved model '{model}' to base_model '{base_model}' from router") + verbose_proxy_logger.debug("Resolved model '%s' to base_model '%s' from router", model, base_model) custom_llm_provider = litellm_params.get("custom_llm_provider") return ( str(base_model), @@ -69,14 +69,14 @@ def _resolve_model_for_cost_lookup(model: str) -> tuple[str, str | None]: resolved_model = litellm_params.get("model") if resolved_model: - verbose_proxy_logger.debug(f"Resolved model '{model}' to '{resolved_model}' from router") + verbose_proxy_logger.debug("Resolved model '%s' to '%s' from router", model, resolved_model) custom_llm_provider = litellm_params.get("custom_llm_provider") return ( str(resolved_model), (str(custom_llm_provider) if custom_llm_provider is not None else None), ) except Exception as e: - verbose_proxy_logger.debug(f"Could not resolve model '{model}' from router: {e}") + verbose_proxy_logger.debug("Could not resolve model '%s' from router: %s", model, e) # Return original model if not resolved return model, custom_llm_provider @@ -129,7 +129,7 @@ async def get_cost_discount_config( return {"values": cost_discount_config} except Exception as e: - verbose_proxy_logger.error(f"Error fetching cost discount config: {e}") + verbose_proxy_logger.error("Error fetching cost discount config: %s", e) return {"values": {}} @@ -216,7 +216,7 @@ async def update_cost_discount_config( # Update in-memory litellm.cost_discount_config litellm.cost_discount_config = cost_discount_config - verbose_proxy_logger.info(f"Updated cost_discount_config: {cost_discount_config}") + verbose_proxy_logger.info("Updated cost_discount_config: %s", cost_discount_config) return { "message": "Cost discount configuration updated successfully", @@ -224,7 +224,7 @@ async def update_cost_discount_config( "values": cost_discount_config, } except Exception as e: - verbose_proxy_logger.error(f"Error updating cost discount config: {e}") + verbose_proxy_logger.error("Error updating cost discount config: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to update cost discount config: {e}"}, @@ -262,7 +262,7 @@ async def get_cost_margin_config( return {"values": cost_margin_config} except Exception as e: - verbose_proxy_logger.error(f"Error fetching cost margin config: {e}") + verbose_proxy_logger.error("Error fetching cost margin config: %s", e) return {"values": {}} @@ -390,7 +390,7 @@ async def update_cost_margin_config( # Update in-memory litellm.cost_margin_config litellm.cost_margin_config = cost_margin_config - verbose_proxy_logger.info(f"Updated cost_margin_config: {cost_margin_config}") + verbose_proxy_logger.info("Updated cost_margin_config: %s", cost_margin_config) return { "message": "Cost margin configuration updated successfully", @@ -398,7 +398,7 @@ async def update_cost_margin_config( "values": cost_margin_config, } except Exception as e: - verbose_proxy_logger.error(f"Error updating cost margin config: {e}") + verbose_proxy_logger.error("Error updating cost margin config: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to update cost margin config: {e}"}, @@ -450,7 +450,7 @@ async def estimate_cost( # Resolve model name (handles router aliases like 'e-model-router' -> 'azure_ai/gpt-4') resolved_model, resolved_provider = _resolve_model_for_cost_lookup(request.model) - verbose_proxy_logger.debug(f"Cost estimate: request.model='{request.model}' resolved to '{resolved_model}'") + verbose_proxy_logger.debug("Cost estimate: request.model='%s' resolved to '%s'", request.model, resolved_model) # Create a mock response with usage for completion_cost mock_response = ModelResponse( diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index ff384190d31..ab59ee65a0a 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -103,7 +103,7 @@ async def block_user(data: BlockUsers): return {"blocked_users": records} except Exception as e: - verbose_proxy_logger.error(f"An error occurred - {e}") + verbose_proxy_logger.error("An error occurred - %s", e) raise HTTPException(status_code=500, detail={"error": str(e)}) @@ -377,7 +377,8 @@ async def new_end_user( # It should have been converted to object_permission_id by _set_object_permission if "object_permission" in new_end_user_obj: verbose_proxy_logger.warning( - f"object_permission still in new_end_user_obj after _set_object_permission: {new_end_user_obj.get('object_permission')}" + "object_permission still in new_end_user_obj after _set_object_permission: %s", + new_end_user_obj.get("object_permission"), ) new_end_user_obj.pop("object_permission", None) @@ -390,7 +391,7 @@ async def new_end_user( return _to_customer_response(end_user_record) except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.management_endpoints.customer_endpoints.new_end_user(): Exception occured - {e}" + "litellm.proxy.management_endpoints.customer_endpoints.new_end_user(): Exception occured - %s", e ) if "Unique constraint failed on the fields: (`user_id`)" in str(e): raise ProxyException( @@ -455,7 +456,7 @@ async def end_user_info( except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.management_endpoints.customer_endpoints.end_user_info(): Exception occured - {e}" + "litellm.proxy.management_endpoints.customer_endpoints.end_user_info(): Exception occured - %s", e ) raise handle_exception_on_proxy(e) @@ -613,7 +614,8 @@ async def update_end_user( # It should have been converted to object_permission_id by handle_update_object_permission_common if "object_permission" in update_end_user_table_data: verbose_proxy_logger.warning( - f"object_permission still in update_end_user_table_data: {update_end_user_table_data.get('object_permission')}" + "object_permission still in update_end_user_table_data: %s", + update_end_user_table_data.get("object_permission"), ) update_end_user_table_data.pop("object_permission", None) @@ -627,7 +629,7 @@ async def update_end_user( ) if response is None: raise ValueError(f"Failed updating customer data. User ID does not exist passed user_id={data.user_id}") - verbose_proxy_logger.debug(f"received response from updating prisma client. response={response}") + verbose_proxy_logger.debug("received response from updating prisma client. response=%s", response) return _to_customer_response(response) else: @@ -636,7 +638,7 @@ async def update_end_user( # update based on remaining passed in values except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.update_end_user(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.update_end_user(): Exception occured - %s", e) raise handle_exception_on_proxy(e) @@ -701,7 +703,7 @@ async def delete_end_user( response = await EndUserRepository(prisma_client).table.delete_many( where={"user_id": {"in": data.user_ids}} ) - verbose_proxy_logger.debug(f"received response from updating prisma client. response={response}") + verbose_proxy_logger.debug("received response from updating prisma client. response=%s", response) return DeleteCustomersResponse( deleted_customers=response, message="Successfully deleted customers with ids: " + str(data.user_ids), @@ -711,7 +713,7 @@ async def delete_end_user( # update based on remaining passed in values except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.delete_end_user(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.delete_end_user(): Exception occured - %s", e) raise handle_exception_on_proxy(e) @@ -767,7 +769,7 @@ async def list_end_user( except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.management_endpoints.customer_endpoints.list_end_user(): Exception occured - {e}" + "litellm.proxy.management_endpoints.customer_endpoints.list_end_user(): Exception occured - %s", e ) raise handle_exception_on_proxy(e) diff --git a/litellm/proxy/management_endpoints/fallback_management_endpoints.py b/litellm/proxy/management_endpoints/fallback_management_endpoints.py index 3df5384b551..00dc4f0f23b 100644 --- a/litellm/proxy/management_endpoints/fallback_management_endpoints.py +++ b/litellm/proxy/management_endpoints/fallback_management_endpoints.py @@ -169,7 +169,7 @@ async def create_fallback( setattr(llm_router, fallback_key, existing_fallbacks) verbose_proxy_logger.info( - f"Fallback configured: {data.model} -> {data.fallback_models} (type: {data.fallback_type})" + "Fallback configured: %s -> %s (type: %s)", data.model, data.fallback_models, data.fallback_type ) return FallbackResponse( @@ -182,7 +182,7 @@ async def create_fallback( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error creating fallback: {e}", exc_info=True) + verbose_proxy_logger.error("Error creating fallback: %s", e, exc_info=True) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to create fallback: {e}"}, @@ -239,7 +239,7 @@ async def get_fallback( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error getting fallback: {e}", exc_info=True) + verbose_proxy_logger.error("Error getting fallback: %s", e, exc_info=True) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to get fallback: {e}"}, @@ -339,7 +339,7 @@ async def delete_fallback( # Update the in-memory router configuration setattr(llm_router, fallback_key, updated_fallbacks) - verbose_proxy_logger.info(f"Fallback deleted: {model} (type: {fallback_type})") + verbose_proxy_logger.info("Fallback deleted: %s (type: %s)", model, fallback_type) return FallbackDeleteResponse( model=model, @@ -350,7 +350,7 @@ async def delete_fallback( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error deleting fallback: {e}", exc_info=True) + verbose_proxy_logger.error("Error deleting fallback: %s", e, exc_info=True) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to delete fallback: {e}"}, diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 8e31b1f6e62..0f159000fb2 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -330,7 +330,8 @@ async def _add_user_to_team( except HTTPException as e: if e.status_code == 400 and ("already exists" in str(e) or "doesn't exist" in str(e)): verbose_proxy_logger.debug( - f"litellm.proxy.management_endpoints.internal_user_endpoints.new_user(): User already exists in team - {e}" + "litellm.proxy.management_endpoints.internal_user_endpoints.new_user(): User already exists in team - %s", + e, ) else: verbose_proxy_logger.error( @@ -348,7 +349,8 @@ async def _add_user_to_team( and ProxyErrorTypes.team_member_already_in_team in e.type ): verbose_proxy_logger.debug( - f"litellm.proxy.management_endpoints.internal_user_endpoints.new_user(): User already exists in team - {e}" + "litellm.proxy.management_endpoints.internal_user_endpoints.new_user(): User already exists in team - %s", + e, ) else: verbose_proxy_logger.error( @@ -605,7 +607,7 @@ async def new_user( return new_user_response except Exception as e: - verbose_proxy_logger.exception(f"/user/new: Exception occured - {e}") + verbose_proxy_logger.exception("/user/new: Exception occured - %s", e) raise handle_exception_on_proxy(e) @@ -900,7 +902,7 @@ async def user_info( return response_data except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.user_info(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.user_info(): Exception occured - %s", e) raise handle_exception_on_proxy(e) @@ -1050,7 +1052,7 @@ async def user_info_v2( object_permission=user_data.get("object_permission"), ) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.user_info_v2(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.user_info_v2(): Exception occured - %s", e) raise handle_exception_on_proxy(e) @@ -1233,7 +1235,7 @@ async def _schedule_user_update_audit_log( ) ) except Exception as audit_error: - verbose_proxy_logger.warning(f"Failed to create audit log for user {response.get('user_id')}: {audit_error}") + verbose_proxy_logger.warning("Failed to create audit log for user %s: %s", response.get("user_id"), audit_error) def _check_user_update_authz( @@ -1320,7 +1322,7 @@ async def _invalidate_cached_user_entitlement(user_id: str | None, object_permis try: await user_api_key_cache.async_delete_cache(key=key) except Exception as e: # noqa: BLE001 # a cache we cannot clear still expires; never fail the write - verbose_proxy_logger.warning(f"Failed to invalidate cached entitlement key {key!r}: {e}") + verbose_proxy_logger.warning("Failed to invalidate cached entitlement key %r: %s", key, e) async def _update_single_user_helper( @@ -1569,7 +1571,7 @@ async def user_update( ) return response except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.user_update(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.user_update(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -1618,12 +1620,12 @@ async def bulk_update_processed_users( successful_updates += 1 except Exception as e: verbose_proxy_logger.exception( - f"Failed to update user {user_request.user_id or user_request.user_email}: {e}" + "Failed to update user %s: %s", user_request.user_id or user_request.user_email, e ) # Record failure error_message = str(e) verbose_proxy_logger.error( - f"Failed to update user {user_request.user_id or user_request.user_email}: {error_message}" + "Failed to update user %s: %s", user_request.user_id or user_request.user_email, error_message ) results.append( @@ -1643,7 +1645,7 @@ async def bulk_update_processed_users( failed_updates=failed_updates, ) except Exception as e: - verbose_proxy_logger.exception(f"Failed to update users: {e}") + verbose_proxy_logger.exception("Failed to update users: %s", e) raise HTTPException(status_code=500, detail={"error": str(e)}) @@ -1806,10 +1808,10 @@ async def bulk_user_update( ) ) except Exception as audit_error: - verbose_proxy_logger.warning(f"Failed to create bulk audit log: {audit_error}") + verbose_proxy_logger.warning("Failed to create bulk audit log: %s", audit_error) except Exception as e: - verbose_proxy_logger.exception(f"Failed to perform bulk update: {e}") + verbose_proxy_logger.exception("Failed to perform bulk update: %s", e) # Fall back to individual updates if bulk update fails for user in all_users_in_db: user_update_request = data.user_updates.model_copy() @@ -2133,7 +2135,7 @@ async def get_users( else: user_key_counts = {} - verbose_proxy_logger.debug(f"Total count of users: {total_count}") + verbose_proxy_logger.debug("Total count of users: %s", total_count) # Calculate total pages total_pages = -(-total_count // page_size) # Ceiling division @@ -2593,7 +2595,7 @@ async def ui_view_users( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error searching users: {e}") + verbose_proxy_logger.exception("Error searching users: %s", e) raise HTTPException(status_code=500, detail=f"Error searching users: {e}") @@ -2716,7 +2718,7 @@ async def get_user_daily_activity( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"/spend/daily/analytics: Exception occured - {e}") + verbose_proxy_logger.exception("/spend/daily/analytics: Exception occured - %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to fetch analytics: {e}"}, @@ -2808,7 +2810,7 @@ async def get_user_daily_activity_aggregated( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"/user/daily/activity/aggregated: Exception occured - {e}") + verbose_proxy_logger.exception("/user/daily/activity/aggregated: Exception occured - %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to fetch analytics: {e}"}, diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index d4403b3f5db..f37bc6da23e 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -940,7 +940,7 @@ async def _common_key_generation_helper( data = apply_enterprise_key_management_params(data, team_table) except Exception as e: verbose_proxy_logger.debug( - f"litellm.proxy.proxy_server.generate_key_fn(): Enterprise key management params not applied - {e}" + "litellm.proxy.proxy_server.generate_key_fn(): Enterprise key management params not applied - %s", e ) # TODO: @ishaan-jaff: Migrate all budget tracking to use LiteLLM_BudgetTable @@ -1693,7 +1693,7 @@ async def generate_key_fn( check_db_only=True, ) except Exception as e: - verbose_proxy_logger.debug(f"Error getting team object in `/key/generate`: {e}") + verbose_proxy_logger.debug("Error getting team object in `/key/generate`: %s", e) # For non-admin callers, team must exist (LIT-1884) if not _is_proxy_admin: raise HTTPException( @@ -1732,7 +1732,7 @@ async def generate_key_fn( ) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.generate_key_fn(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.generate_key_fn(): Exception occured - %s", e) raise handle_exception_on_proxy(e) @@ -1866,7 +1866,7 @@ async def generate_service_account_key_fn( check_db_only=True, ) except Exception as e: - verbose_proxy_logger.debug(f"Error getting team object in `/key/generate`: {e}") + verbose_proxy_logger.debug("Error getting team object in `/key/generate`: %s", e) team_table = None if team_table is not None: @@ -1934,7 +1934,9 @@ def prepare_metadata_fields(data: BaseModel, non_default_values: dict, existing_ casted_metadata[k] = v except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.prepare_metadata_fields(): Exception occured - {e}") + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.prepare_metadata_fields(): Exception occured - %s", e + ) non_default_values["metadata"] = encrypt_callback_vars(casted_metadata) return non_default_values @@ -2052,7 +2054,7 @@ async def _handle_update_object_permission( # Add the object_permission_id to data_json if one was created/updated if object_permission_id is not None: data_json["object_permission_id"] = object_permission_id - verbose_proxy_logger.debug(f"updated object_permission_id: {object_permission_id}") + verbose_proxy_logger.debug("updated object_permission_id: %s", object_permission_id) return data_json @@ -2797,7 +2799,7 @@ async def update_key_fn( return {"key": key, **response["data"]} # update based on remaining passed in values except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.update_key_fn(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.update_key_fn(): Exception occured - %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"Authentication Error({e})"), @@ -2935,7 +2937,7 @@ async def bulk_update_keys( ) except Exception as e: - verbose_proxy_logger.exception(f"Failed to update key {key_update_item.key}: {e}") + verbose_proxy_logger.exception("Failed to update key %s: %s", key_update_item.key, e) if isinstance(e, HTTPException): error_detail = e.detail @@ -3175,7 +3177,7 @@ async def bulk_update_team_keys( except Exception as e: # Log the hashed prefix — `token` may be a raw sk-... and ERROR logs persist. - verbose_proxy_logger.exception(f"Failed to update key {db_token[:12]}... in team {data.team_id}: {e}") + verbose_proxy_logger.exception("Failed to update key %s... in team %s: %s", db_token[:12], data.team_id, e) failed_updates.append( _build_failed_team_key_update( token=token, @@ -3305,7 +3307,7 @@ async def delete_key_fn( litellm_changed_by = None ## only allow user to delete keys they own - verbose_proxy_logger.debug(f"user_api_key_dict.user_role: {user_api_key_dict.user_role}") + verbose_proxy_logger.debug("user_api_key_dict.user_role: %s", user_api_key_dict.user_role) num_keys_to_be_deleted = 0 deleted_keys = [] @@ -3338,7 +3340,7 @@ async def delete_key_fn( param="keys", code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) - verbose_proxy_logger.debug(f"/key/delete - deleted_keys={number_deleted_keys}") + verbose_proxy_logger.debug("/key/delete - deleted_keys=%s", number_deleted_keys) try: assert num_keys_to_be_deleted == len(deleted_keys) @@ -3351,7 +3353,7 @@ async def delete_key_fn( ) verbose_proxy_logger.debug( - f"/keys/delete - cache after delete: {user_api_key_cache.in_memory_cache.cache_dict}" + "/keys/delete - cache after delete: %s", user_api_key_cache.in_memory_cache.cache_dict ) asyncio.create_task( @@ -3366,7 +3368,7 @@ async def delete_key_fn( return {"deleted_keys": deleted_keys} except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.delete_key_fn(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.delete_key_fn(): Exception occured - %s", e) raise handle_exception_on_proxy(e) @@ -3906,7 +3908,7 @@ async def generate_key_helper_fn( # If it's not valid JSON/YAML, keep as is or set to empty dict key_data["router_settings"] = {} except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.generate_key_helper_fn(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.generate_key_helper_fn(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise e @@ -4113,7 +4115,7 @@ async def delete_verification_tokens( raise Exception("DB not connected. prisma_client is None") except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.delete_verification_tokens(): Exception occured - {e}" + "litellm.proxy.proxy_server.delete_verification_tokens(): Exception occured - %s", e ) verbose_proxy_logger.debug(traceback.format_exc()) raise e @@ -4385,10 +4387,10 @@ async def _rotate_master_key( }, ) except Exception as e: - verbose_proxy_logger.error(f"Failed to re-encrypt credential {cred.credential_name}: {e}") + verbose_proxy_logger.error("Failed to re-encrypt credential %s: %s", cred.credential_name, e) # Continue with next credential instead of failing entire rotation continue - verbose_proxy_logger.debug(f"Successfully re-encrypted {len(credentials)} credentials with new master key") + verbose_proxy_logger.debug("Successfully re-encrypted %s credentials with new master key", len(credentials)) def _require_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None: @@ -5446,7 +5448,7 @@ async def list_keys( return response except Exception as e: - verbose_proxy_logger.exception(f"Error in list_keys: {e}") + verbose_proxy_logger.exception("Error in list_keys: %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"error({e})"), @@ -5585,8 +5587,12 @@ async def key_aliases( total_pages = -(-total_count // size) if total_count > 0 else 0 verbose_proxy_logger.debug( - f"key_aliases: page={page}, size={size}, search={search!r}, " - f"total_count={total_count}, total_pages={total_pages}" + "key_aliases: page=%s, size=%s, search=%r, total_count=%s, total_pages=%s", + page, + size, + search, + total_count, + total_pages, ) return { @@ -5598,7 +5604,7 @@ async def key_aliases( } except Exception as e: - verbose_proxy_logger.exception(f"Error in key_aliases: {e}") + verbose_proxy_logger.exception("Error in key_aliases: %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"error({e})"), @@ -5779,7 +5785,7 @@ def _build_key_filter_conditions( if expires_filter is not None and expires_filter in VALID_EXPIRES_FILTER_VALUES: where = {"AND": [where, _build_expires_where_clause(expires_filter, datetime.now(timezone.utc))]} - verbose_proxy_logger.debug(f"Filter conditions: {where}") + verbose_proxy_logger.debug("Filter conditions: %s", where) return where @@ -5850,7 +5856,7 @@ async def _list_key_helper( # Calculate skip for pagination skip = (page - 1) * size - verbose_proxy_logger.debug(f"Pagination: skip={skip}, take={size}") + verbose_proxy_logger.debug("Pagination: skip=%s, take=%s", skip, size) order_by: dict[str, str] | None = ( _validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None @@ -5890,7 +5896,7 @@ async def _list_key_helper( include={"object_permission": True}, ) - verbose_proxy_logger.debug(f"Fetched {len(keys)} keys") + verbose_proxy_logger.debug("Fetched %s keys", len(keys)) # Get total count of keys if use_deleted_table: @@ -5902,7 +5908,7 @@ async def _list_key_helper( where=where # type: ignore ) - verbose_proxy_logger.debug(f"Total count of keys: {total_count}") + verbose_proxy_logger.debug("Total count of keys: %s", total_count) # Calculate total pages total_pages = -(-total_count // size) # Ceiling division diff --git a/litellm/proxy/management_endpoints/management_v1/budgets.py b/litellm/proxy/management_endpoints/management_v1/budgets.py index 1bd47a940be..a0afcdc56cc 100644 --- a/litellm/proxy/management_endpoints/management_v1/budgets.py +++ b/litellm/proxy/management_endpoints/management_v1/budgets.py @@ -191,7 +191,7 @@ async def list_budgets( raise except Exception as e: # noqa: BLE001 # a driver error answers as a problem document, not the OpenAI error shape verbose_proxy_logger.exception( - f"litellm.proxy.management_endpoints.management_v1.budgets.list_budgets(): Exception occured - {e}" + "litellm.proxy.management_endpoints.management_v1.budgets.list_budgets(): Exception occured - %s", e ) raise ManagementProblem( ProblemDetail( diff --git a/litellm/proxy/management_endpoints/management_v1/spend_logs.py b/litellm/proxy/management_endpoints/management_v1/spend_logs.py index 403e2760fb9..3614a598393 100644 --- a/litellm/proxy/management_endpoints/management_v1/spend_logs.py +++ b/litellm/proxy/management_endpoints/management_v1/spend_logs.py @@ -187,8 +187,8 @@ async def list_spend_log_end_users( raise except Exception as e: verbose_proxy_logger.exception( - "litellm.proxy.management_endpoints.management_v1.spend_logs.list_spend_log_end_users(): " - f"Exception occured - {e}" + "litellm.proxy.management_endpoints.management_v1.spend_logs.list_spend_log_end_users(): Exception occured - %s", + e, ) raise ManagementProblem( ProblemDetail( diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index da86a7f06f2..6a212dd3bf9 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -94,7 +94,7 @@ DEFAULT_MCP_REGISTRY_VERSION = "1.0.0" try: importlib.import_module("mcp") except ImportError as e: - verbose_logger.debug(f"MCP module not found: {e}") + verbose_logger.debug("MCP module not found: %s", e) MCP_AVAILABLE = False if MCP_AVAILABLE: @@ -399,7 +399,7 @@ if MCP_AVAILABLE: try: encrypted_payload = encrypt_value_helper(payload_json) except Exception as e: - verbose_proxy_logger.debug(f"Failed to encrypt temporary MCP server payload for Redis cache: {e}") + verbose_proxy_logger.debug("Failed to encrypt temporary MCP server payload for Redis cache: %s", e) return if not isinstance(encrypted_payload, str): @@ -413,7 +413,7 @@ if MCP_AVAILABLE: ttl=max(1, ttl_seconds), ) except Exception as e: - verbose_proxy_logger.debug(f"Failed to write temporary MCP server to Redis cache: {e}") + verbose_proxy_logger.debug("Failed to write temporary MCP server to Redis cache: %s", e) async def _get_temporary_mcp_server_from_redis( server_id: str, @@ -435,7 +435,7 @@ if MCP_AVAILABLE: key=f"{TEMPORARY_MCP_SERVER_REDIS_KEY_PREFIX}:{server_id}" ) except Exception as e: - verbose_proxy_logger.debug(f"Failed reading temporary MCP server from Redis cache: {e}") + verbose_proxy_logger.debug("Failed reading temporary MCP server from Redis cache: %s", e) return None if not isinstance(cached_server, str): @@ -454,7 +454,7 @@ if MCP_AVAILABLE: try: loaded = json.loads(decrypted_json) except Exception as e: - verbose_proxy_logger.debug(f"Invalid decrypted temporary MCP payload in Redis cache: {e}") + verbose_proxy_logger.debug("Invalid decrypted temporary MCP payload in Redis cache: %s", e) return None if not isinstance(loaded, dict): return None @@ -463,7 +463,7 @@ if MCP_AVAILABLE: try: return MCPServer.model_validate(payload_dict) except Exception as e: - verbose_proxy_logger.debug(f"Invalid temporary MCP server payload in Redis cache: {e}") + verbose_proxy_logger.debug("Invalid temporary MCP server payload in Redis cache: %s", e) return None async def get_cached_temporary_mcp_server( @@ -814,7 +814,7 @@ if MCP_AVAILABLE: if hasattr(server, "mcp_access_groups") and server.mcp_access_groups: access_groups.update(server.mcp_access_groups) except Exception as e: - verbose_proxy_logger.debug(f"Error getting MCP access groups: {e}") + verbose_proxy_logger.debug("Error getting MCP access groups: %s", e) # Convert to sorted list access_groups_list = sorted(list(access_groups)) @@ -864,7 +864,7 @@ if MCP_AVAILABLE: entry = _build_mcp_registry_entry_for_server(server, base_url) except Exception as e: verbose_proxy_logger.debug( - f"Skipping MCP server {getattr(server, 'server_id', 'unknown')} in registry: {e}" + "Skipping MCP server %s in registry: %s", getattr(server, "server_id", "unknown"), e ) continue registry_servers.append({"server": entry}) @@ -1183,7 +1183,7 @@ if MCP_AVAILABLE: touched_by=user_api_key_dict.user_id or user_api_key_dict.team_id, ) except Exception as e: - verbose_proxy_logger.exception(f"Error registering mcp server: {e}") + verbose_proxy_logger.exception("Error registering mcp server: %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Error registering mcp server: {e}"}, @@ -1398,7 +1398,7 @@ if MCP_AVAILABLE: mcp_server.last_health_check = health_result.last_health_check mcp_server.health_check_error = health_result.health_check_error except Exception as e: - verbose_proxy_logger.debug(f"Error performing health check on server {server_id}: {e}") + verbose_proxy_logger.debug("Error performing health check on server %s: %s", server_id, e) mcp_server.status = "unknown" mcp_server.last_health_check = datetime.now() mcp_server.health_check_error = str(e) @@ -1483,7 +1483,7 @@ if MCP_AVAILABLE: touched_by=user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME, ) except Exception as e: - verbose_proxy_logger.exception(f"Error creating mcp server: {e}") + verbose_proxy_logger.exception("Error creating mcp server: %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Error creating mcp server: {e}"}, @@ -1498,7 +1498,7 @@ if MCP_AVAILABLE: await global_mcp_server_manager.reload_servers_from_database() except Exception as e: verbose_proxy_logger.exception( - f"MCP server {new_mcp_server.server_id} created but in-memory registry refresh failed: {e}" + "MCP server %s created but in-memory registry refresh failed: %s", new_mcp_server.server_id, e ) return _redact_mcp_credentials(new_mcp_server) @@ -1559,7 +1559,7 @@ if MCP_AVAILABLE: ttl_seconds=TEMPORARY_MCP_SERVER_TTL_SECONDS, ) except Exception as e: - verbose_proxy_logger.exception(f"Error caching temporary mcp server: {e}") + verbose_proxy_logger.exception("Error caching temporary mcp server: %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Error caching temporary mcp server: {e}"}, @@ -2566,7 +2566,7 @@ if MCP_AVAILABLE: await proxy_config.save_config(new_config=config) verbose_proxy_logger.debug( - f"Updated public mcp servers to: {litellm.public_mcp_servers} by user: {user_api_key_dict.user_id}" + "Updated public mcp servers to: %s by user: %s", litellm.public_mcp_servers, user_api_key_dict.user_id ) return { @@ -2577,7 +2577,7 @@ if MCP_AVAILABLE: except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error making agent public: {e}") + verbose_proxy_logger.exception("Error making agent public: %s", e) raise HTTPException(status_code=500, detail=str(e)) # --- MCP Discovery --- @@ -2598,7 +2598,7 @@ if MCP_AVAILABLE: with open(_MCP_REGISTRY_PATH, "r") as f: data: dict[str, Any] = json.load(f) except Exception as e: - verbose_proxy_logger.warning(f"Failed to load MCP registry from {_MCP_REGISTRY_PATH}: {e}") + verbose_proxy_logger.warning("Failed to load MCP registry from %s: %s", _MCP_REGISTRY_PATH, e) data = {"servers": []} _mcp_registry_cache = data return data @@ -2685,7 +2685,7 @@ if MCP_AVAILABLE: try: return _load_openapi_registry() except Exception as e: - verbose_proxy_logger.warning(f"Failed to load OpenAPI registry from {_OPENAPI_REGISTRY_PATH}: {e}") + verbose_proxy_logger.warning("Failed to load OpenAPI registry from %s: %s", _OPENAPI_REGISTRY_PATH, e) return {"apis": []} # --------------------------------------------------------------------------- diff --git a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py index b294b2674e4..65a6e3639ec 100644 --- a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py @@ -121,7 +121,7 @@ async def _tag_deployment_with_access_group( where={"model_id": model_id}, data={"model_info": json.dumps(updated_model_info)}, ) - verbose_proxy_logger.debug(f"Updated deployment {model_id} with access group: {access_group}") + verbose_proxy_logger.debug("Updated deployment %s with access group: %s", model_id, access_group) return (model_id, updated_model_info) @@ -175,7 +175,7 @@ async def update_deployments_with_access_group( so callers can verify each one survived the post-write reload """ deployments = await ModelRepository(prisma_client).table.find_many(where={"model_name": {"in": model_names}}) - verbose_proxy_logger.debug(f"Found {len(deployments)} deployments for model_names: {model_names}") + verbose_proxy_logger.debug("Found %s deployments for model_names: %s", len(deployments), model_names) found_names = {deployment.model_name for deployment in deployments} for model_name in model_names: @@ -212,7 +212,7 @@ async def update_specific_deployments_with_access_group( their unique model_id. Returns the (model_id, updated model_info) pair of every deployment actually written. """ - verbose_proxy_logger.debug(f"Updating specific deployment model_ids: {model_ids}") + verbose_proxy_logger.debug("Updating specific deployment model_ids: %s", model_ids) tagged = [ await _tag_deployment_with_access_group( model_id=model_id, @@ -344,7 +344,7 @@ async def create_model_group( prisma_client, ) - verbose_proxy_logger.debug(f"Creating access group: {data.access_group} with models: {data.model_names}") + verbose_proxy_logger.debug("Creating access group: %s with models: %s", data.access_group, data.model_names) # Validation: Check if access_group is provided if not data.access_group or not data.access_group.strip(): @@ -426,7 +426,7 @@ async def create_model_group( ) verbose_proxy_logger.info( - f"Successfully created access group '{data.access_group}' with {models_updated} models updated" + "Successfully created access group '%s' with %s models updated", data.access_group, models_updated ) return NewModelGroupResponse( @@ -439,7 +439,7 @@ async def create_model_group( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error creating access group '{data.access_group}': {e}") + verbose_proxy_logger.exception("Error creating access group '%s': %s", data.access_group, e) raise HTTPException( status_code=500, detail={"error": f"Failed to create access group: {e}"}, @@ -489,7 +489,7 @@ async def list_access_groups( return ListAccessGroupsResponse(access_groups=access_groups_list) except Exception as e: - verbose_proxy_logger.exception(f"Error listing access groups: {e}") + verbose_proxy_logger.exception("Error listing access groups: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to list access groups: {e}"}, @@ -546,7 +546,7 @@ async def get_access_group_info( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error getting access group info for '{access_group}': {e}") + verbose_proxy_logger.exception("Error getting access group info for '%s': %s", access_group, e) raise HTTPException( status_code=500, detail={"error": f"Failed to get access group info: {e}"}, @@ -600,7 +600,7 @@ async def update_access_group( detail={"error": "Database not connected."}, ) - verbose_proxy_logger.debug(f"Updating access group: {access_group} with models: {data.model_names}") + verbose_proxy_logger.debug("Updating access group: %s with models: %s", access_group, data.model_names) # Validation: Check that at least one of model_names or model_ids is provided has_model_names = data.model_names and len(data.model_names) > 0 @@ -686,7 +686,7 @@ async def update_access_group( ) verbose_proxy_logger.info( - f"Successfully updated access group '{access_group}' with {models_updated} models updated" + "Successfully updated access group '%s' with %s models updated", access_group, models_updated ) return NewModelGroupResponse( @@ -699,7 +699,7 @@ async def update_access_group( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error updating access group '{access_group}': {e}") + verbose_proxy_logger.exception("Error updating access group '%s': %s", access_group, e) raise HTTPException( status_code=500, detail={"error": f"Failed to update access group: {e}"}, @@ -744,7 +744,7 @@ async def delete_access_group( detail={"error": "Database not connected."}, ) - verbose_proxy_logger.debug(f"Deleting access group: {access_group}") + verbose_proxy_logger.debug("Deleting access group: %s", access_group) # Validation: Check if access group exists try: @@ -788,7 +788,7 @@ async def delete_access_group( ) verbose_proxy_logger.info( - f"Successfully deleted access group '{access_group}' from {models_updated} deployments" + "Successfully deleted access group '%s' from %s deployments", access_group, models_updated ) return DeleteModelGroupResponse( @@ -800,7 +800,7 @@ async def delete_access_group( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error deleting access group '{access_group}': {e}") + verbose_proxy_logger.exception("Error deleting access group '%s': %s", access_group, e) raise HTTPException( status_code=500, detail={"error": f"Failed to delete access group: {e}"}, diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 475c5813cdd..aa5312d58cb 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -365,7 +365,7 @@ async def patch_model( return updated_model except Exception as e: - verbose_proxy_logger.exception(f"Error in patch_model: {e}") + verbose_proxy_logger.exception("Error in patch_model: %s", e) if isinstance(e, (HTTPException, ProxyException)): raise e @@ -472,7 +472,7 @@ async def _set_model_blocked_status( return updated_model except Exception as e: - verbose_proxy_logger.exception(f"Error in model {action}: {e}") + verbose_proxy_logger.exception("Error in model %s: %s", action, e) if isinstance(e, (HTTPException, ProxyException)): raise e @@ -1233,7 +1233,7 @@ async def delete_model( ) except Exception as e: - verbose_proxy_logger.exception(f"Failed to delete model. Due to error - {e}") + verbose_proxy_logger.exception("Failed to delete model. Due to error - %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"Authentication Error({e})"), @@ -1399,7 +1399,7 @@ async def add_new_model( passed_model_info=model_params.model_info, ) except Exception as e: - verbose_proxy_logger.exception(f"Exception in add_new_model: {e}") + verbose_proxy_logger.exception("Exception in add_new_model: %s", e) else: raise HTTPException( @@ -1439,7 +1439,7 @@ async def add_new_model( return model_response except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.add_new_model(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.add_new_model(): Exception occured - %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"Authentication Error({e})"), @@ -1592,7 +1592,7 @@ async def update_model( return model_response except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.update_model(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.update_model(): Exception occured - %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"Authentication Error({e})"), @@ -1675,7 +1675,7 @@ async def update_public_model_groups( litellm.public_model_groups = request.model_groups verbose_proxy_logger.debug( - f"Updated public model groups to: {request.model_groups} by user: {user_api_key_dict.user_id}" + "Updated public model groups to: %s by user: %s", request.model_groups, user_api_key_dict.user_id ) return { @@ -1685,7 +1685,7 @@ async def update_public_model_groups( } except Exception as e: - verbose_proxy_logger.exception(f"Error updating public model groups: {e}") + verbose_proxy_logger.exception("Error updating public model groups: %s", e) if isinstance(e, HTTPException): raise e @@ -1743,7 +1743,7 @@ async def update_useful_links( litellm.public_model_groups_links = request.useful_links verbose_proxy_logger.debug( - f"Updated useful links to: {request.useful_links} by user: {user_api_key_dict.user_id}" + "Updated useful links to: %s by user: %s", request.useful_links, user_api_key_dict.user_id ) return { @@ -1753,7 +1753,7 @@ async def update_useful_links( } except Exception as e: - verbose_proxy_logger.exception(f"Error updating public model groups: {e}") + verbose_proxy_logger.exception("Error updating public model groups: %s", e) if isinstance(e, HTTPException): raise e @@ -1976,9 +1976,9 @@ async def clear_cache() -> frozenset[str] | None: ) verbose_proxy_logger.debug( - f"Cleared {len(db_model_ids)} DB models, preserved {len(config_models)} config models" + "Cleared %s DB models, preserved %s config models", len(db_model_ids), len(config_models) ) return still_desired_ids except Exception as e: - verbose_proxy_logger.exception(f"Failed to clear cache and reload models. Due to error - {e}") + verbose_proxy_logger.exception("Failed to clear cache and reload models. Due to error - %s", e) return None diff --git a/litellm/proxy/management_endpoints/organization_endpoints.py b/litellm/proxy/management_endpoints/organization_endpoints.py index 949c35e4182..66cf111324a 100644 --- a/litellm/proxy/management_endpoints/organization_endpoints.py +++ b/litellm/proxy/management_endpoints/organization_endpoints.py @@ -498,7 +498,7 @@ async def new_organization( new_organization_row = _STR_OBJECT_DICT_ADAPTER.validate_python( prisma_client.jsonify_object(organization_row.json(exclude_none=True)) ) - verbose_proxy_logger.info(f"new_organization_row: {json.dumps(new_organization_row, indent=2)}") + verbose_proxy_logger.info("new_organization_row: %s", json.dumps(new_organization_row, indent=2)) response = await _table(OrganizationRepository(prisma_client)).create( data={ **new_organization_row, @@ -1258,7 +1258,7 @@ async def organization_member_add( updated_organization_memberships=updated_organization_memberships, ) except Exception as e: - verbose_proxy_logger.exception(f"Error adding member to organization: {e}") + verbose_proxy_logger.exception("Error adding member to organization: %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"Authentication Error({e})"), @@ -1444,7 +1444,7 @@ async def organization_member_update( ) return final_organization_membership_pydantic except Exception as e: - verbose_proxy_logger.exception(f"Error updating member in organization: {e}") + verbose_proxy_logger.exception("Error updating member in organization: %s", e) raise e @@ -1493,7 +1493,7 @@ async def organization_member_delete( return member_to_delete except Exception as e: - verbose_proxy_logger.exception(f"Error deleting member from organization: {e}") + verbose_proxy_logger.exception("Error deleting member from organization: %s", e) raise e diff --git a/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py b/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py index fb139902978..a55e32446f5 100644 --- a/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py +++ b/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py @@ -432,7 +432,7 @@ async def validate_policy( from litellm.proxy.policy_engine.policy_validator import PolicyValidator from litellm.proxy.proxy_server import prisma_client - verbose_proxy_logger.debug(f"Validating policy configuration with {len(data.policies)} policies") + verbose_proxy_logger.debug("Validating policy configuration with %s policies", len(data.policies)) validator = PolicyValidator(prisma_client=prisma_client) diff --git a/litellm/proxy/management_endpoints/router_settings_endpoints.py b/litellm/proxy/management_endpoints/router_settings_endpoints.py index 0adc0610c60..a4d4e2fe5cb 100644 --- a/litellm/proxy/management_endpoints/router_settings_endpoints.py +++ b/litellm/proxy/management_endpoints/router_settings_endpoints.py @@ -120,7 +120,7 @@ async def get_router_settings( routing_strategy_descriptions=ROUTING_STRATEGY_DESCRIPTIONS, ) except Exception as e: - verbose_proxy_logger.error(f"Error fetching router settings: {e}") + verbose_proxy_logger.error("Error fetching router settings: %s", e) raise @@ -168,5 +168,5 @@ async def get_router_fields( routing_strategy_descriptions=ROUTING_STRATEGY_DESCRIPTIONS, ) except Exception as e: - verbose_proxy_logger.error(f"Error fetching router fields: {e}") + verbose_proxy_logger.error("Error fetching router fields: %s", e) raise diff --git a/litellm/proxy/management_endpoints/scim/scim_v2.py b/litellm/proxy/management_endpoints/scim/scim_v2.py index b3140ec911e..4437bfcbb1c 100644 --- a/litellm/proxy/management_endpoints/scim/scim_v2.py +++ b/litellm/proxy/management_endpoints/scim/scim_v2.py @@ -365,7 +365,7 @@ async def _get_scim_upsert_user_setting() -> bool: # Default to True if not set (backward compatibility) return bool(scim_upsert_user) except Exception as e: - verbose_proxy_logger.warning(f"Error reading scim_upsert_user setting, defaulting to True: {e}") + verbose_proxy_logger.warning("Error reading scim_upsert_user setting, defaulting to True: %s", e) # Default to True for backward compatibility return True @@ -401,7 +401,7 @@ async def _get_scim_admin_group() -> str | None: litellm_settings = config.get("litellm_settings", {}) or {} return litellm_settings.get("scim_admin_group") or None except Exception as e: - verbose_proxy_logger.warning(f"Error reading scim_admin_group setting, defaulting to None: {e}") + verbose_proxy_logger.warning("Error reading scim_admin_group setting, defaulting to None: %s", e) return None @@ -882,11 +882,11 @@ async def _create_user_if_not_exists(user_id: str, created_via: str = "scim_grou ) created_user = await new_user(data=new_user_request) - verbose_proxy_logger.info(f"Created user {user_id} via {created_via}") + verbose_proxy_logger.info("Created user %s via %s", user_id, created_via) return created_user except Exception as e: - verbose_proxy_logger.exception(f"Failed to create user {user_id}: {e}") + verbose_proxy_logger.exception("Failed to create user %s: %s", user_id, e) return None @@ -1886,15 +1886,15 @@ async def patch_team_membership( except ProxyException as e: # Handle duplicate membership gracefully - this is idempotent if e.type == ProxyErrorTypes.team_member_already_in_team: - verbose_proxy_logger.debug(f"User {user_id} is already in team {_team_id}, skipping add") + verbose_proxy_logger.debug("User %s is already in team %s, skipping add", user_id, _team_id) elif raise_on_error: raise else: - verbose_proxy_logger.exception(f"Error adding user to team {_team_id}: {e}") + verbose_proxy_logger.exception("Error adding user to team %s: %s", _team_id, e) except Exception as e: if raise_on_error: raise - verbose_proxy_logger.exception(f"Error adding user to team {_team_id}: {e}") + verbose_proxy_logger.exception("Error adding user to team %s: %s", _team_id, e) for _team_id in teams_ids_to_remove_user_from: try: @@ -1904,15 +1904,15 @@ async def patch_team_membership( ) except HTTPException as e: if _is_user_not_in_team_error(e): - verbose_proxy_logger.debug(f"User {user_id} is not in team {_team_id}, skipping remove") + verbose_proxy_logger.debug("User %s is not in team %s, skipping remove", user_id, _team_id) elif raise_on_error: raise else: - verbose_proxy_logger.exception(f"Error removing user from team {_team_id}: {e}") + verbose_proxy_logger.exception("Error removing user from team %s: %s", _team_id, e) except Exception as e: if raise_on_error: raise - verbose_proxy_logger.exception(f"Error removing user from team {_team_id}: {e}") + verbose_proxy_logger.exception("Error removing user from team %s: %s", _team_id, e) return True @@ -2045,7 +2045,7 @@ async def get_groups( # team creation, so reading it here would report an empty member # list to the IdP and trigger repeated re-provisioning. members = await _get_team_members_display(await _get_team_member_user_ids_from_team(team)) - verbose_proxy_logger.debug(f"SCIM GET GROUPS members: {members}") + verbose_proxy_logger.debug("SCIM GET GROUPS members: %s", members) team_alias = getattr(team, "team_alias", team.team_id) team_created_at = team.created_at.isoformat() if team.created_at else None team_updated_at = team.updated_at.isoformat() if team.updated_at else None @@ -2063,7 +2063,7 @@ async def get_groups( ) scim_groups.append(scim_group) - verbose_proxy_logger.debug(f"SCIM GET GROUPS response: {scim_groups}") + verbose_proxy_logger.debug("SCIM GET GROUPS response: %s", scim_groups) return SCIMListResponse( totalResults=total_count, startIndex=startIndex, @@ -2092,7 +2092,7 @@ async def get_group( team = await _check_team_exists(group_id) scim_group = await ScimTransformations.transform_litellm_team_to_scim_group(team) - verbose_proxy_logger.debug(f"SCIM GET GROUP response: {scim_group}") + verbose_proxy_logger.debug("SCIM GET GROUP response: %s", scim_group) return scim_group except Exception as e: @@ -2178,8 +2178,8 @@ async def update_group( # Extract and validate group members (all users must exist) member_result = await _extract_group_member_ids(group) - verbose_proxy_logger.debug(f"SCIM PUT GROUP all_member_ids: {member_result.all_member_ids}") - verbose_proxy_logger.debug(f"SCIM PUT GROUP created_users: {len(member_result.created_users)}") + verbose_proxy_logger.debug("SCIM PUT GROUP all_member_ids: %s", member_result.all_member_ids) + verbose_proxy_logger.debug("SCIM PUT GROUP created_users: %s", len(member_result.created_users)) # Prepare update data existing_metadata = existing_team.metadata if existing_team.metadata else {} @@ -2202,9 +2202,9 @@ async def update_group( # Handle user-team relationship changes current_members = set(await _get_team_member_user_ids_from_team(existing_team)) - verbose_proxy_logger.debug(f"SCIM PUT GROUP current_members: {current_members}") + verbose_proxy_logger.debug("SCIM PUT GROUP current_members: %s", current_members) final_members = set(member_result.all_member_ids) - verbose_proxy_logger.debug(f"SCIM PUT GROUP final_members: {final_members}") + verbose_proxy_logger.debug("SCIM PUT GROUP final_members: %s", final_members) await _handle_group_membership_changes( group_id=group_id, @@ -2380,8 +2380,8 @@ async def _handle_group_membership_changes(group_id: str, current_members: set[s members_to_add = final_members - current_members members_to_remove = current_members - final_members - verbose_proxy_logger.debug(f"members_to_add: {members_to_add}") - verbose_proxy_logger.debug(f"members_to_remove: {members_to_remove}") + verbose_proxy_logger.debug("members_to_add: %s", members_to_add) + verbose_proxy_logger.debug("members_to_remove: %s", members_to_remove) # Use existing helper functions for team membership changes for member_id in members_to_add: diff --git a/litellm/proxy/management_endpoints/sso/custom_microsoft_sso.py b/litellm/proxy/management_endpoints/sso/custom_microsoft_sso.py index 2f900f9b6b6..07108712dbb 100644 --- a/litellm/proxy/management_endpoints/sso/custom_microsoft_sso.py +++ b/litellm/proxy/management_endpoints/sso/custom_microsoft_sso.py @@ -68,10 +68,10 @@ class CustomMicrosoftSSO(MicrosoftSSO): if custom_authorization_endpoint or custom_token_endpoint or custom_userinfo_endpoint: verbose_proxy_logger.debug( - f"Using custom Microsoft SSO endpoints - " - f"authorization: {authorization_endpoint}, " - f"token: {token_endpoint}, " - f"userinfo: {userinfo_endpoint}" + "Using custom Microsoft SSO endpoints - authorization: %s, token: %s, userinfo: %s", + authorization_endpoint, + token_endpoint, + userinfo_endpoint, ) return DiscoveryDocument( diff --git a/litellm/proxy/management_endpoints/sso/saml_sso.py b/litellm/proxy/management_endpoints/sso/saml_sso.py index 37b641ca123..c73c57702e6 100644 --- a/litellm/proxy/management_endpoints/sso/saml_sso.py +++ b/litellm/proxy/management_endpoints/sso/saml_sso.py @@ -449,7 +449,9 @@ class SAMLAuthHandler: display_name = " ".join(part for part in (first_name, last_name) if part) or email - verbose_proxy_logger.info(f"SAML login: subject={user_id}, email={email}, attributes={list(attributes.keys())}") + verbose_proxy_logger.info( + "SAML login: subject=%s, email=%s, attributes=%s", user_id, email, list(attributes.keys()) + ) try: return CustomOpenID( diff --git a/litellm/proxy/management_endpoints/tag_management_endpoints.py b/litellm/proxy/management_endpoints/tag_management_endpoints.py index 8e701fa9e20..d6db8dc05cd 100644 --- a/litellm/proxy/management_endpoints/tag_management_endpoints.py +++ b/litellm/proxy/management_endpoints/tag_management_endpoints.py @@ -201,7 +201,7 @@ async def _get_model_names(prisma_client: "PrismaClient", model_ids: Sequence[st models = await _table(ModelRepository(prisma_client)).find_many(where={"model_id": {"in": model_ids}}) return {model.model_id: model.model_name for model in models} except Exception as e: - verbose_proxy_logger.error(f"Error getting model names: {e}") + verbose_proxy_logger.error("Error getting model names: %s", e) return {} @@ -331,7 +331,7 @@ async def new_tag( "tag": tag_config, } except Exception as e: - verbose_proxy_logger.exception(f"Error creating tag: {e}") + verbose_proxy_logger.exception("Error creating tag: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -372,7 +372,7 @@ async def _add_tag_to_deployment(deployment: "Deployment", tag: str): data={"litellm_params": json.dumps(existing_params)}, ) except Exception as e: - verbose_proxy_logger.exception(f"Error adding tag to deployment: {e}") + verbose_proxy_logger.exception("Error adding tag to deployment: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -461,7 +461,7 @@ async def update_tag( "tag": tag_config, } except Exception as e: - verbose_proxy_logger.exception(f"Error updating tag: {e}") + verbose_proxy_logger.exception("Error updating tag: %s", e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 32b22dd6ade..dd20dd97e00 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -354,7 +354,7 @@ async def add_team_callbacks( except ProxyException as e: raise e except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.add_team_callbacks(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.add_team_callbacks(): Exception occured - %s", e) raise ProxyException( message="Internal Server Error, " + str(e), type=ProxyErrorTypes.internal_server_error.value, @@ -492,7 +492,7 @@ async def disable_team_logging( except ProxyException: raise except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.disable_team_logging(): Exception occurred - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.disable_team_logging(): Exception occurred - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) raise ProxyException( message="Internal Server Error, " + str(e), @@ -585,7 +585,7 @@ async def get_team_callbacks( except ProxyException: raise except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.get_team_callbacks(): Exception occurred - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.get_team_callbacks(): Exception occurred - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index bdd7e18a850..61c3fe4505f 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -465,7 +465,11 @@ class TeamMemberBudgetHandler: user_api_key_dict=user_api_key_dict, ) verbose_proxy_logger.info( - f"Updated team member budget table: {budget_row.budget_id}, with team_member_budget={team_member_budget}, team_member_rpm_limit={team_member_rpm_limit}, team_member_tpm_limit={team_member_tpm_limit}" + "Updated team member budget table: %s, with team_member_budget=%s, team_member_rpm_limit=%s, team_member_tpm_limit=%s", + budget_row.budget_id, + team_member_budget, + team_member_rpm_limit, + team_member_tpm_limit, ) if updated_kv.get("metadata") is None: updated_kv["metadata"] = {} @@ -2241,7 +2245,7 @@ async def handle_update_object_permission(data_json: dict, existing_team_row: Li # Add the object_permission_id to data_json if one was created/updated if object_permission_id is not None: data_json["object_permission_id"] = object_permission_id - verbose_proxy_logger.debug(f"updated object_permission_id: {object_permission_id}") + verbose_proxy_logger.debug("updated object_permission_id: %s", object_permission_id) return data_json @@ -2333,7 +2337,9 @@ def team_member_add_duplication_check( ) elif len(invalid_team_members) > 0: verbose_proxy_logger.info( - f"Some users are already in team. Existing members={existing_team_row.members_with_roles}. Duplicate members={invalid_team_members}", + "Some users are already in team. Existing members=%s. Duplicate members=%s", + existing_team_row.members_with_roles, + invalid_team_members, ) @@ -3837,7 +3843,7 @@ async def _add_team_member_budget_table( team_info_response_object.team_member_budget_table = team_budget except Exception: verbose_proxy_logger.info( - f"Team member budget table not found, passed team_member_budget_id={team_member_budget_id}" + "Team member budget table not found, passed team_member_budget_id=%s", team_member_budget_id ) return team_info_response_object @@ -3975,7 +3981,9 @@ async def team_info( except Exception as e: verbose_proxy_logger.error( - f"litellm.proxy.management_endpoints.team_endpoints.py::team_info - Exception occurred - {e}\n{traceback.format_exc()}" + "litellm.proxy.management_endpoints.team_endpoints.py::team_info - Exception occurred - %s\n%s", + e, + traceback.format_exc(), ) if isinstance(e, HTTPException): raise ProxyException( @@ -4915,7 +4923,7 @@ async def get_paginated_teams( ) return teams, total_count except Exception as e: - verbose_proxy_logger.exception(f"[Non-Blocking] Error getting paginated teams: {e}") + verbose_proxy_logger.exception("[Non-Blocking] Error getting paginated teams: %s", e) return [], 0 diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index bc05f72ae14..a2e880eab5f 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -443,7 +443,7 @@ def build_cli_sso_attribution_metadata( metadata: dict[str, Any] = {} for source_claim, dest_key in claim_map: if not _is_safe_cli_sso_metadata_dest_key(dest_key): - verbose_proxy_logger.debug(f"Skipping unsafe CLI SSO metadata destination key: {dest_key}") + verbose_proxy_logger.debug("Skipping unsafe CLI SSO metadata destination key: %s", dest_key) continue raw_value = _extract_sso_claim_value(result=result, claim_path=source_claim) @@ -504,11 +504,12 @@ async def _persist_cli_sso_user_metadata( data={"metadata": merged_metadata}, ) verbose_proxy_logger.info( - f"Persisted CLI SSO attribution metadata for user {user_id}: " - f"{list(_flatten_cli_sso_metadata_for_poll(attribution_metadata).keys())}" + "Persisted CLI SSO attribution metadata for user %s: %s", + user_id, + list(_flatten_cli_sso_metadata_for_poll(attribution_metadata).keys()), ) except Exception as e: - verbose_proxy_logger.error(f"Failed to persist CLI SSO attribution metadata for user {user_id}: {e}") + verbose_proxy_logger.error("Failed to persist CLI SSO attribution metadata for user %s: %s", user_id, e) def _cli_poll_attribution_metadata_from_session( @@ -745,13 +746,15 @@ def determine_role_from_groups( role_groups = role_mappings.roles[role] if isinstance(role_groups, list) and user_groups_set.intersection(set(role_groups)): verbose_proxy_logger.debug( - f"User groups {user_groups} matched role '{role.value}' via groups: {role_groups}" + "User groups %s matched role '%s' via groups: %s", user_groups, role.value, role_groups ) return role # No matching groups found, return default_role verbose_proxy_logger.debug( - f"User groups {user_groups} did not match any role mappings, using default_role: {role_mappings.default_role}" + "User groups %s did not match any role mappings, using default_role: %s", + user_groups, + role_mappings.default_role, ) return role_mappings.default_role @@ -827,7 +830,7 @@ def process_sso_jwt_access_token( if user_groups: user_role = determine_role_from_groups(user_groups, role_mappings) verbose_proxy_logger.debug( - f"Determined role '{user_role}' from access token groups '{user_groups}' using role_mappings" + "Determined role '%s' from access token groups '%s' using role_mappings", user_role, user_groups ) elif role_mappings.default_role: user_role = role_mappings.default_role @@ -839,7 +842,7 @@ def process_sso_jwt_access_token( if user_role_from_token is not None: user_role = get_litellm_user_role(user_role_from_token) verbose_proxy_logger.debug( - f"Extracted role '{user_role}' from access token field '{generic_user_role_attribute_name}'" + "Extracted role '%s' from access token field '%s'", user_role, generic_user_role_attribute_name ) if user_role is not None: @@ -847,7 +850,7 @@ def process_sso_jwt_access_token( result["user_role"] = user_role else: setattr(result, "user_role", user_role) - verbose_proxy_logger.debug(f"Set user_role='{user_role}' from JWT access token") + verbose_proxy_logger.debug("Set user_role='%s' from JWT access token", user_role) return access_token_payload @@ -976,7 +979,7 @@ async def google_login( ) is True ): - verbose_proxy_logger.info(f"Redirecting to SSO login for {redirect_url}") + verbose_proxy_logger.info("Redirecting to SSO login for %s", redirect_url) sso_redirect = await SSOAuthenticationHandler.get_sso_login_redirect( redirect_url=redirect_url, microsoft_client_id=microsoft_client_id, @@ -1031,7 +1034,9 @@ def generic_response_convertor( generic_user_extra_attributes = os.getenv("GENERIC_USER_EXTRA_ATTRIBUTES", None) verbose_proxy_logger.debug( - f" generic_user_id_attribute_name: {generic_user_id_attribute_name}\n generic_user_email_attribute_name: {generic_user_email_attribute_name}" + " generic_user_id_attribute_name: %s\n generic_user_email_attribute_name: %s", + generic_user_id_attribute_name, + generic_user_email_attribute_name, ) all_teams = [] @@ -1048,7 +1053,9 @@ def generic_response_convertor( if team_ids_from_db_mapping: all_teams.extend(team_ids_from_db_mapping) verbose_proxy_logger.debug( - f"Loaded team_ids from DB team_mappings.team_ids_jwt_field='{team_mappings.team_ids_jwt_field}': {team_ids_from_db_mapping}" + "Loaded team_ids from DB team_mappings.team_ids_jwt_field='%s': %s", + team_mappings.team_ids_jwt_field, + team_ids_from_db_mapping, ) else: team_ids = jwt_handler.get_all_jwt_team_ids(cast(dict, response)) @@ -1080,13 +1087,15 @@ def generic_response_convertor( if user_groups: user_role = determine_role_from_groups(user_groups, role_mappings) verbose_proxy_logger.debug( - f"Determined role '{user_role.value if user_role else None}' from groups '{user_groups}' using role_mappings" + "Determined role '%s' from groups '%s' using role_mappings", + user_role.value if user_role else None, + user_groups, ) else: # No groups found, use default_role user_role = role_mappings.default_role verbose_proxy_logger.debug( - f"No groups found in '{group_claim}', using default_role: {role_mappings.default_role}" + "No groups found in '%s', using default_role: %s", group_claim, role_mappings.default_role ) # Fallback to existing logic if role_mappings not used @@ -1097,7 +1106,9 @@ def generic_response_convertor( if role is not None: user_role = role verbose_proxy_logger.debug( - f"Found valid LitellmUserRoles '{role.value}' from SSO attribute '{generic_user_role_attribute_name}'" + "Found valid LitellmUserRoles '%s' from SSO attribute '%s'", + role.value, + generic_user_role_attribute_name, ) # Build extra_fields dict from GENERIC_USER_EXTRA_ATTRIBUTES if specified @@ -1163,9 +1174,12 @@ def _setup_generic_sso_env_vars( ) verbose_proxy_logger.debug( - f"authorization_endpoint: {generic_authorization_endpoint}\ntoken_endpoint: {generic_token_endpoint}\nuserinfo_endpoint: {generic_userinfo_endpoint}" + "authorization_endpoint: %s\ntoken_endpoint: %s\nuserinfo_endpoint: %s", + generic_authorization_endpoint, + generic_token_endpoint, + generic_userinfo_endpoint, ) - verbose_proxy_logger.debug(f"GENERIC_REDIRECT_URI: {redirect_url}\nGENERIC_CLIENT_ID: {generic_client_id}\n") + verbose_proxy_logger.debug("GENERIC_REDIRECT_URI: %s\nGENERIC_CLIENT_ID: %s\n", redirect_url, generic_client_id) return ( generic_client_secret, @@ -1201,11 +1215,11 @@ async def _setup_team_mappings() -> Optional["TeamMappings"]: if team_mappings and team_mappings.team_ids_jwt_field: verbose_proxy_logger.debug( - f"Loaded team_mappings with team_ids_jwt_field: '{team_mappings.team_ids_jwt_field}'" + "Loaded team_mappings with team_ids_jwt_field: '%s'", team_mappings.team_ids_jwt_field ) except Exception as e: verbose_proxy_logger.debug( - f"Could not load team_mappings from database: {e}. Continuing with config-based team mapping." + "Could not load team_mappings from database: %s. Continuing with config-based team mapping.", e ) return team_mappings @@ -1232,10 +1246,10 @@ async def _setup_role_mappings() -> Optional["RoleMappings"]: role_mappings = role_mappings_data if role_mappings: - verbose_proxy_logger.debug(f"Loaded role_mappings for provider '{role_mappings.provider}'") + verbose_proxy_logger.debug("Loaded role_mappings for provider '%s'", role_mappings.provider) except Exception as e: verbose_proxy_logger.debug( - f"Could not load role_mappings from database: {e}. Continuing with existing role logic." + "Could not load role_mappings from database: %s. Continuing with existing role logic.", e ) generic_role_mappings = os.getenv("GENERIC_ROLE_MAPPINGS_ROLES", None) @@ -1257,12 +1271,12 @@ async def _setup_role_mappings() -> Optional["RoleMappings"]: role_mappings = RoleMappings(**role_mappings_data) verbose_proxy_logger.debug( - f"Loaded role_mappings from environments for provider '{role_mappings.provider}'." + "Loaded role_mappings from environments for provider '%s'.", role_mappings.provider ) return role_mappings except TypeError as e: verbose_proxy_logger.warning( - f"Error decoding role mappings from environment variables: {e}. Continuing with existing role logic." + "Error decoding role mappings from environment variables: %s. Continuing with existing role logic.", e ) return role_mappings @@ -1534,7 +1548,7 @@ async def create_team_member_add_task(team_id, user_info): user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), ) except Exception as e: - verbose_proxy_logger.debug(f"[Non-Blocking] Error trying to add sso user to db: {e}") + verbose_proxy_logger.debug("[Non-Blocking] Error trying to add sso user to db: %s", e) async def add_missing_team_member(user_info: NewUserResponse | LiteLLM_UserTable, sso_teams: list[str]): @@ -1552,7 +1566,7 @@ async def add_missing_team_member(user_info: NewUserResponse | LiteLLM_UserTable try: await asyncio.gather(*tasks) except Exception as e: - verbose_proxy_logger.debug(f"[Non-Blocking] Error trying to add sso user to db: {e}") + verbose_proxy_logger.debug("[Non-Blocking] Error trying to add sso user to db: %s", e) def get_disabled_non_admin_personal_key_creation(): @@ -1583,7 +1597,7 @@ async def get_existing_user_info_from_db( sso_user_id=user_id, ) except Exception as e: - verbose_proxy_logger.debug(f"Error getting user object: {e}") + verbose_proxy_logger.debug("Error getting user object: %s", e) user_info = None return user_info @@ -1629,7 +1643,7 @@ async def get_user_info_from_db( break verbose_proxy_logger.debug( - f"user_info: {user_info}; litellm.default_internal_user_params: {litellm.default_internal_user_params}" + "user_info: %s; litellm.default_internal_user_params: %s", user_info, litellm.default_internal_user_params ) # Upsert SSO User to LiteLLM DB @@ -1648,7 +1662,7 @@ async def get_user_info_from_db( return user_info except Exception as e: - verbose_proxy_logger.exception(f"[Non-Blocking] Error trying to add sso user to db: {e}") + verbose_proxy_logger.exception("[Non-Blocking] Error trying to add sso user to db: %s", e) return None @@ -1660,8 +1674,8 @@ def _should_use_role_from_sso_response(sso_role: str | None) -> bool: if not is_valid_litellm_user_role(sso_role): verbose_proxy_logger.debug( - f"SSO role '{sso_role}' is not a valid LiteLLM user role. " - "Ignoring role from SSO response. See LitellmUserRoles enum for valid roles." + "SSO role '%s' is not a valid LiteLLM user role. Ignoring role from SSO response. See LitellmUserRoles enum for valid roles.", + sso_role, ) return False return True @@ -1694,7 +1708,7 @@ def _build_sso_user_update_data( # Only include if it's a valid LiteLLM role if _should_use_role_from_sso_response(sso_role_str): update_data["user_role"] = sso_role_str - verbose_proxy_logger.info(f"Updating user {user_id} role from SSO: {sso_role_str}") + verbose_proxy_logger.info("Updating user %s role from SSO: %s", user_id, sso_role_str) return update_data @@ -1726,7 +1740,7 @@ async def _sync_user_role_from_jwt_role_map( if mapped_role is None: return - verbose_proxy_logger.info(f"SSO jwt_litellm_role_map matched role: {mapped_role.value}") + verbose_proxy_logger.info("SSO jwt_litellm_role_map matched role: %s", mapped_role.value) # Update user_defined_values so downstream code uses the mapped role if user_defined_values is not None: @@ -1762,7 +1776,7 @@ def apply_user_info_values_to_sso_user_defined_values( if _should_use_role_from_sso_response(sso_role): # SSO provided a valid role, keep it and log that we're using it - verbose_proxy_logger.info(f"Using SSO role: {sso_role} (DB role was: {db_role})") + verbose_proxy_logger.info("Using SSO role: %s (DB role was: %s)", sso_role, db_role) else: # SSO didn't provide a valid role, fall back to DB role or default if user_info is None or user_info.user_role is None: @@ -1770,7 +1784,7 @@ def apply_user_info_values_to_sso_user_defined_values( verbose_proxy_logger.debug("No SSO or DB role found, using default: INTERNAL_USER_VIEW_ONLY") else: user_defined_values["user_role"] = user_info.user_role - verbose_proxy_logger.debug(f"Using DB role: {user_info.user_role}") + verbose_proxy_logger.debug("Using DB role: %s", user_info.user_role) # Preserve the user's existing models from the database if user_info is not None and hasattr(user_info, "models") and user_info.models: @@ -1803,13 +1817,13 @@ async def check_and_update_if_proxy_admin_id(user_role: str, user_id: str, prism @router.get("/sso/callback", tags=["experimental"], include_in_schema=False) async def auth_callback(request: Request, state: str | None = None): """Verify login""" - verbose_proxy_logger.info(f"Starting SSO callback with state: {state}") + verbose_proxy_logger.info("Starting SSO callback with state: %s", state) oauth_error = request.query_params.get("error") if oauth_error: oauth_error_description = request.query_params.get("error_description") verbose_proxy_logger.warning( - f"SSO callback received OAuth error: {oauth_error}, description: {oauth_error_description}" + "SSO callback received OAuth error: %s, description: %s", oauth_error, oauth_error_description ) raise HTTPException( status_code=401, @@ -1861,7 +1875,7 @@ async def auth_callback(request: Request, state: str | None = None): ) redirect_url = SSOAuthenticationHandler.get_redirect_url_for_sso(request=request, sso_callback_route="sso/callback") - verbose_proxy_logger.info(f"Redirecting to {redirect_url}") + verbose_proxy_logger.info("Redirecting to %s", redirect_url) result = None if google_client_id is not None: result = await GoogleSSOHandler.get_google_callback_response( @@ -2044,7 +2058,7 @@ async def _fetch_cli_sso_team_details( } ) except Exception as e: - verbose_proxy_logger.error(f"Error fetching team details for CLI SSO session: {e}") + verbose_proxy_logger.error("Error fetching team details for CLI SSO session: %s", e) return team_details @@ -2111,7 +2125,7 @@ async def _complete_cli_sso_callback_session( _set_cli_sso_flow(login_id=key, cache=cli_sso_session_cache, flow=flow) verbose_proxy_logger.info( - f"Stored CLI SSO session for user: {user_info.user_id}, teams: {teams}, num_teams: {len(teams)}" + "Stored CLI SSO session for user: %s, teams: %s, num_teams: %s", user_info.user_id, teams, len(teams) ) verify_url = get_custom_url( request_base_url=str(request.base_url), @@ -2165,7 +2179,7 @@ async def cli_sso_callback( result=result_non_none, generic_client_id=os.getenv("GENERIC_CLIENT_ID", None), ) - verbose_proxy_logger.debug(f"parsed_openid_result: {parsed_openid_result}") + verbose_proxy_logger.debug("parsed_openid_result: %s", parsed_openid_result) user_defined_values = await _build_cli_sso_user_defined_values( result=result_non_none, parsed_openid_result=parsed_openid_result, @@ -2196,7 +2210,7 @@ async def cli_sso_callback( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error with CLI SSO callback: {e}") + verbose_proxy_logger.error("Error with CLI SSO callback: %s", e) raise HTTPException(status_code=500, detail=f"Failed to process CLI SSO: {e}") @@ -2236,7 +2250,11 @@ async def cli_poll_key( user_id = session_data["user_id"] verbose_proxy_logger.info( - f"CLI poll: user={user_id}, team_id={team_id}, user_teams={user_teams}, num_teams={len(user_teams)}" + "CLI poll: user=%s, team_id=%s, user_teams=%s, num_teams=%s", + user_id, + team_id, + user_teams, + len(user_teams), ) # If no team_id provided and user has teams, return teams list for selection @@ -2244,7 +2262,7 @@ async def cli_poll_key( # clients we return rich team details (id + alias); older clients # can continue to rely on the simple "teams" list. if team_id is None and len(user_teams) > 1: - verbose_proxy_logger.info(f"Returning teams list for user {user_id} to select from: {user_teams}") + verbose_proxy_logger.info("Returning teams list for user %s to select from: %s", user_id, user_teams) # Best-effort construction of team_details if it wasn't # already cached for some reason. team_details_response: list[dict[str, Any]] | None = None @@ -2298,7 +2316,7 @@ async def cli_poll_key( # Delete cache entry (single-use) cli_sso_session_cache.delete_cache(key=_get_cli_sso_flow_cache_key(key_id)) - verbose_proxy_logger.info(f"CLI JWT generated for user: {user_id}, team: {team_id}") + verbose_proxy_logger.info("CLI JWT generated for user: %s, team: %s", user_id, team_id) poll_response = { "status": "ready", "key": jwt_token, @@ -2319,7 +2337,7 @@ async def cli_poll_key( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error polling for CLI JWT: {e}") + verbose_proxy_logger.error("Error polling for CLI JWT: %s", e) raise HTTPException(status_code=500, detail=f"Error checking session status: {e}") @@ -2337,7 +2355,7 @@ async def insert_sso_user( Returns: Tuple[str, str]: User ID and User Role """ - verbose_proxy_logger.debug(f"Inserting SSO user into DB. User values: {user_defined_values}") + verbose_proxy_logger.debug("Inserting SSO user into DB. User values: %s", user_defined_values) if result_openid is None: raise ValueError("result_openid is None") if isinstance(result_openid, dict): @@ -2357,7 +2375,7 @@ async def insert_sso_user( preserved_role = sso_role user_defined_values.update(litellm.default_internal_user_params) # type: ignore user_defined_values["user_role"] = preserved_role # Restore preserved role - verbose_proxy_logger.debug(f"Preserved SSO-extracted role '{preserved_role}'") + verbose_proxy_logger.debug("Preserved SSO-extracted role '%s'", preserved_role) else: # SSO didn't provide a valid role, apply all defaults including role user_defined_values.update(litellm.default_internal_user_params) # type: ignore @@ -2671,7 +2689,9 @@ class SSOAuthenticationHandler: redirect_uri=redirect_url, ) verbose_proxy_logger.info( - f"In /google-login/key/generate, \nGOOGLE_REDIRECT_URI: {redirect_url}\nGOOGLE_CLIENT_ID: {google_client_id}" + "In /google-login/key/generate, \nGOOGLE_REDIRECT_URI: %s\nGOOGLE_CLIENT_ID: %s", + redirect_url, + google_client_id, ) with google_sso: return await google_sso.get_login_redirect(state=state) @@ -2733,10 +2753,13 @@ class SSOAuthenticationHandler: code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) verbose_proxy_logger.debug( - f"authorization_endpoint: {generic_authorization_endpoint}\ntoken_endpoint: {generic_token_endpoint}\nuserinfo_endpoint: {generic_userinfo_endpoint}" + "authorization_endpoint: %s\ntoken_endpoint: %s\nuserinfo_endpoint: %s", + generic_authorization_endpoint, + generic_token_endpoint, + generic_userinfo_endpoint, ) verbose_proxy_logger.debug( - f"GENERIC_REDIRECT_URI: {redirect_url}\nGENERIC_CLIENT_ID: {generic_client_id}\n" + "GENERIC_REDIRECT_URI: %s\nGENERIC_CLIENT_ID: %s\n", redirect_url, generic_client_id ) discovery = DiscoveryDocument( authorization_endpoint=generic_authorization_endpoint, @@ -2992,7 +3015,7 @@ class SSOAuthenticationHandler: ) return user_info except Exception as e: - verbose_proxy_logger.exception(f"Error upserting SSO user into LiteLLM DB: {e}") + verbose_proxy_logger.exception("Error upserting SSO user into LiteLLM DB: %s", e) return user_info @staticmethod @@ -3075,11 +3098,11 @@ class SSOAuthenticationHandler: ) try: team_obj = await TeamRepository(prisma_client).table.find_first(where={"team_id": litellm_team_id}) - verbose_proxy_logger.debug(f"Team object: {team_obj}") + verbose_proxy_logger.debug("Team object: %s", team_obj) # only create a new team if it doesn't exist if team_obj: - verbose_proxy_logger.debug(f"Team already exists: {litellm_team_id} - {litellm_team_name}") + verbose_proxy_logger.debug("Team already exists: %s - %s", litellm_team_id, litellm_team_name) return team_request: NewTeamRequest = NewTeamRequest( @@ -3104,7 +3127,7 @@ class SSOAuthenticationHandler: ), ) except Exception as e: - verbose_proxy_logger.exception(f"Error creating Litellm Team: {e}") + verbose_proxy_logger.exception("Error creating Litellm Team: %s", e) @staticmethod def _cast_and_deepcopy_litellm_default_team_params( @@ -3188,7 +3211,7 @@ class SSOAuthenticationHandler: if _user_role is not None: # Convert enum to string if needed user_role = _user_role.value if isinstance(_user_role, LitellmUserRoles) else _user_role - verbose_proxy_logger.debug(f"Extracted user_role from SSO result: {user_role}") + verbose_proxy_logger.debug("Extracted user_role from SSO result: %s", user_role) # generic client id - override with custom attribute name if specified if generic_client_id is not None and result is not None: @@ -3252,7 +3275,7 @@ class SSOAuthenticationHandler: user_email = parsed_openid_result.get("user_email") user_id = parsed_openid_result.get("user_id") user_role = parsed_openid_result.get("user_role") - verbose_proxy_logger.info(f"SSO callback result: {result}") + verbose_proxy_logger.info("SSO callback result: %s", result) user_info = None user_id_models: list = [] @@ -3318,7 +3341,7 @@ class SSOAuthenticationHandler: "Unable to map user identity to known values. 'user_defined_values' is None. File an issue - https://github.com/BerriAI/litellm/issues" ) - verbose_proxy_logger.info(f"user_defined_values for creating ui key: {user_defined_values}") + verbose_proxy_logger.info("user_defined_values for creating ui key: %s", user_defined_values) response = await generate_key_helper_fn( request_type="key", @@ -3346,7 +3369,7 @@ class SSOAuthenticationHandler: user_role=user_role, user_id=user_id, prisma_client=prisma_client ) - verbose_proxy_logger.debug(f"user_role: {user_role}; ui_access_mode: {ui_access_mode}") + verbose_proxy_logger.debug("user_role: %s; ui_access_mode: %s", user_role, ui_access_mode) ## CHECK IF ROLE ALLOWED TO USE PROXY ## is_admin_only_access = check_is_admin_only_access(ui_access_mode or {}) if is_admin_only_access: @@ -3412,7 +3435,7 @@ class SSOAuthenticationHandler: if user_id is not None and isinstance(user_id, str): litellm_dashboard_ui += "?login=success" - verbose_proxy_logger.info(f"Redirecting to {litellm_dashboard_ui}") + verbose_proxy_logger.info("Redirecting to %s", litellm_dashboard_ui) redirect_response = RedirectResponse(url=litellm_dashboard_ui, status_code=303) redirect_response.set_cookie(key="token", value=jwt_token) return redirect_response @@ -4012,7 +4035,7 @@ class MicrosoftSSOHandler: # Extract app roles from the id_token JWT app_roles = MicrosoftSSOHandler.get_app_roles_from_id_token(id_token=microsoft_sso.id_token) - verbose_proxy_logger.debug(f"Extracted app roles from id_token: {app_roles}") + verbose_proxy_logger.debug("Extracted app roles from id_token: %s", app_roles) # Combine groups and app roles user_role: LitellmUserRoles | None = None @@ -4022,10 +4045,10 @@ class MicrosoftSSOHandler: role = get_litellm_user_role(role_str) if role is not None: user_role = role - verbose_proxy_logger.debug(f"Found valid LitellmUserRoles '{role.value}' in app_roles") + verbose_proxy_logger.debug("Found valid LitellmUserRoles '%s' in app_roles", role.value) break - verbose_proxy_logger.debug(f"Combined team_ids (groups + app roles): {user_team_ids}") + verbose_proxy_logger.debug("Combined team_ids (groups + app roles): %s", user_team_ids) # if user is trying to get the raw sso response for debugging, return the raw sso response if return_raw_sso_response: @@ -4047,7 +4070,7 @@ class MicrosoftSSOHandler: user_role: LitellmUserRoles | None, ) -> CustomOpenID: response = response or {} - verbose_proxy_logger.debug(f"Microsoft SSO Callback Response: {response}") + verbose_proxy_logger.debug("Microsoft SSO Callback Response: %s", response) openid_response = CustomOpenID( email=normalize_email(response.get(MICROSOFT_USER_EMAIL_ATTRIBUTE) or response.get("mail")), display_name=response.get(MICROSOFT_USER_DISPLAY_NAME_ATTRIBUTE), @@ -4058,7 +4081,7 @@ class MicrosoftSSOHandler: team_ids=team_ids, user_role=user_role, ) - verbose_proxy_logger.debug(f"Microsoft SSO OpenID Response: {openid_response}") + verbose_proxy_logger.debug("Microsoft SSO OpenID Response: %s", openid_response) return openid_response @staticmethod @@ -4091,14 +4114,14 @@ class MicrosoftSSOHandler: roles = decoded_token.get("app_roles", []) or decoded_token.get("roles", []) if roles and isinstance(roles, list): - verbose_proxy_logger.debug(f"Found {len(roles)} app role(s) in id_token: {roles}") + verbose_proxy_logger.debug("Found %s app role(s) in id_token: %s", len(roles), roles) return roles else: verbose_proxy_logger.debug("No app roles found in id_token or roles claim is not a list") return [] except Exception as e: - verbose_proxy_logger.error(f"Error extracting app roles from id_token: {e}") + verbose_proxy_logger.error("Error extracting app roles from id_token: %s", e) return [] @staticmethod @@ -4130,7 +4153,7 @@ class MicrosoftSSOHandler: async_client=async_client, access_token=access_token, ) - verbose_proxy_logger.debug(f"Service principal group IDs: {service_principal_group_ids}") + verbose_proxy_logger.debug("Service principal group IDs: %s", service_principal_group_ids) if len(service_principal_group_ids) > 0: await MicrosoftSSOHandler.create_litellm_teams_from_service_principal_team_ids( service_principal_teams=service_principal_teams, @@ -4151,7 +4174,8 @@ class MicrosoftSSOHandler: if next_link is not None and page_count >= MicrosoftSSOHandler.MAX_GRAPH_API_PAGES: verbose_proxy_logger.warning( - f"Reached maximum page limit of {MicrosoftSSOHandler.MAX_GRAPH_API_PAGES}. Some groups may not be included." + "Reached maximum page limit of %s. Some groups may not be included.", + MicrosoftSSOHandler.MAX_GRAPH_API_PAGES, ) # If service_principal_group_ids is not empty, only return group_ids that are in both all_group_ids and service_principal_group_ids @@ -4161,7 +4185,7 @@ class MicrosoftSSOHandler: return all_group_ids except Exception as e: - verbose_proxy_logger.error(f"Error getting user groups from Microsoft Graph API: {e}") + verbose_proxy_logger.error("Error getting user groups from Microsoft Graph API: %s", e) return [] @staticmethod @@ -4238,7 +4262,7 @@ class MicrosoftSSOHandler: while next_link is not None and page_count < MicrosoftSSOHandler.MAX_GRAPH_API_PAGES: response = await async_client.get(next_link, headers=headers) response_json = response.json() - verbose_proxy_logger.debug(f"Response from service principal app role assigned to: {response_json}") + verbose_proxy_logger.debug("Response from service principal app role assigned to: %s", response_json) for _object in response_json.get("value", []): if _object.get("principalType") == "Group": @@ -4257,7 +4281,8 @@ class MicrosoftSSOHandler: if next_link is not None and page_count >= MicrosoftSSOHandler.MAX_GRAPH_API_PAGES: verbose_proxy_logger.warning( - f"Reached maximum page limit of {MicrosoftSSOHandler.MAX_GRAPH_API_PAGES}. Some service principal group assignments may not be included." + "Reached maximum page limit of %s. Some service principal group assignments may not be included.", + MicrosoftSSOHandler.MAX_GRAPH_API_PAGES, ) return group_ids, service_principal_teams @@ -4271,13 +4296,13 @@ class MicrosoftSSOHandler: When a user sets a `SERVICE_PRINCIPAL_ID` in the env, litellm will fetch groups under that service principal and create Litellm Teams from them """ - verbose_proxy_logger.debug(f"Creating Litellm Teams from Service Principal Teams: {service_principal_teams}") + verbose_proxy_logger.debug("Creating Litellm Teams from Service Principal Teams: %s", service_principal_teams) for service_principal_team in service_principal_teams: litellm_team_id: str | None = service_principal_team.get("principalId") litellm_team_name: str | None = service_principal_team.get("principalDisplayName") if not litellm_team_id: verbose_proxy_logger.debug( - f"Skipping team creation for {litellm_team_name} because it has no principalId" + "Skipping team creation for %s because it has no principalId", litellm_team_name ) continue diff --git a/litellm/proxy/management_helpers/audit_logs.py b/litellm/proxy/management_helpers/audit_logs.py index 8677627c607..081bd6af866 100644 --- a/litellm/proxy/management_helpers/audit_logs.py +++ b/litellm/proxy/management_helpers/audit_logs.py @@ -231,4 +231,4 @@ async def create_audit_log_for_update(request_data: LiteLLM_AuditLogs): ) except Exception as e: # [Non-Blocking Exception. Do not allow blocking LLM API call] - verbose_proxy_logger.error(f"Failed Creating audit log {e}") + verbose_proxy_logger.error("Failed Creating audit log %s", e) diff --git a/litellm/proxy/management_helpers/object_permission_utils.py b/litellm/proxy/management_helpers/object_permission_utils.py index 24bfbe2f9a9..69665151f07 100644 --- a/litellm/proxy/management_helpers/object_permission_utils.py +++ b/litellm/proxy/management_helpers/object_permission_utils.py @@ -166,7 +166,7 @@ async def handle_update_object_permission_common( }, ) - verbose_proxy_logger.debug(f"created_object_permission_row: {created_object_permission_row}") + verbose_proxy_logger.debug("created_object_permission_row: %s", created_object_permission_row) return created_object_permission_row.object_permission_id @@ -572,8 +572,8 @@ async def validate_key_mcp_servers_against_team( } if stale_identifiers: verbose_proxy_logger.warning( - "validate_key_mcp_servers_against_team: ignoring stale MCP server " - f"identifiers (no longer in registry or DB): {sorted(stale_identifiers)}" + "validate_key_mcp_servers_against_team: ignoring stale MCP server identifiers (no longer in registry or DB): %s", + sorted(stale_identifiers), ) _rewrite_object_permission_mcp_identifiers( object_permission=object_permission, diff --git a/litellm/proxy/ocr_endpoints/endpoints.py b/litellm/proxy/ocr_endpoints/endpoints.py index ca25be9d92c..acc3f85076f 100644 --- a/litellm/proxy/ocr_endpoints/endpoints.py +++ b/litellm/proxy/ocr_endpoints/endpoints.py @@ -96,9 +96,10 @@ async def _parse_multipart_form(request: Request) -> dict[str, Any]: data[field_name] = field_value verbose_proxy_logger.debug( - f"OCR multipart form request parsed - model: {data.get('model')}, " - f"document_type: {document['type']}, " - f"filename: {uploaded_file.filename}" + "OCR multipart form request parsed - model: %s, document_type: %s, filename: %s", + data.get("model"), + document["type"], + uploaded_file.filename, ) return data diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 87514b46dbd..76b956f31b2 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -983,10 +983,10 @@ async def ensure_batch_response_managed_file_ids( user_api_key_dict=user_api_key_dict, ) setattr(response, file_attr, new_unified_file_id) - verbose_proxy_logger.debug(f"Converted batch {file_attr} {raw_file_id!r} to managed ID before DB write") + verbose_proxy_logger.debug("Converted batch %s %r to managed ID before DB write", file_attr, raw_file_id) except Exception as e: verbose_proxy_logger.warning( - f"Failed to convert batch {file_attr}={raw_file_id!r} to managed ID before DB write: {e}" + "Failed to convert batch %s=%r to managed ID before DB write: %s", file_attr, raw_file_id, e ) @@ -1042,12 +1042,16 @@ async def get_batch_from_database( # The stored batch object has the raw provider input_file_id. Resolve to unified ID. await resolve_input_file_id_to_unified(response, prisma_client) - verbose_proxy_logger.debug(f"Retrieved batch {batch_id} from ManagedObjectTable with status={response.status}") + verbose_proxy_logger.debug( + "Retrieved batch %s from ManagedObjectTable with status=%s", batch_id, response.status + ) return db_batch_object, response except Exception as e: - verbose_proxy_logger.warning(f"Failed to retrieve batch from ManagedObjectTable: {e}, falling back to provider") + verbose_proxy_logger.warning( + "Failed to retrieve batch from ManagedObjectTable: %s, falling back to provider", e + ) return None, None @@ -1103,10 +1107,10 @@ async def update_batch_in_database( if db_batch_object: verbose_proxy_logger.info( - f"Updating batch {batch_id} status from {db_batch_object.status} to {response.status}" + "Updating batch %s status from %s to %s", batch_id, db_batch_object.status, response.status ) else: - verbose_proxy_logger.info(f"Updating batch {batch_id} status to {response.status} after {operation}") + verbose_proxy_logger.info("Updating batch %s status to %s after %s", batch_id, response.status, operation) # Normalize status for database storage db_status = response.status if response.status != "completed" else "complete" @@ -1138,7 +1142,9 @@ async def update_batch_in_database( # retry without it so the status update still succeeds. err_str = str(col_err).lower() if "batch_processed" in err_str and update_data.get("batch_processed") is not None: - verbose_proxy_logger.warning(f"batch_processed column not found, retrying update without it: {col_err}") + verbose_proxy_logger.warning( + "batch_processed column not found, retrying update without it: %s", col_err + ) update_data.pop("batch_processed", None) await ManagedObjectRepository(prisma_client).table.update( where={"unified_object_id": batch_id}, @@ -1147,4 +1153,4 @@ async def update_batch_in_database( else: raise except Exception as e: - verbose_proxy_logger.error(f"Failed to update batch status in ManagedObjectTable: {e}") + verbose_proxy_logger.error("Failed to update batch status in ManagedObjectTable: %s", e) diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 5d4c3c04818..51b9139cff1 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -208,7 +208,7 @@ async def route_create_file( original_id = response.id encoded_id = encode_file_id_with_model(file_id=original_id, model=model) response.id = encoded_id - verbose_proxy_logger.debug(f"Encoded file ID: {original_id} -> {encoded_id} (model: {model})") + verbose_proxy_logger.debug("Encoded file ID: %s -> %s (model: %s)", original_id, encoded_id, model) return response @@ -549,7 +549,7 @@ async def create_file( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.create_file(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.create_file(): Exception occured - %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "message", str(e.detail)), @@ -845,7 +845,7 @@ async def get_file_content( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.retrieve_file_content(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.retrieve_file_content(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -1032,7 +1032,7 @@ async def get_file( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.retrieve_file(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.retrieve_file(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -1238,7 +1238,7 @@ async def delete_file( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.delete_file(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.delete_file(): Exception occured - %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "message", str(e.detail)), @@ -1337,7 +1337,7 @@ async def list_files( **data, # type: ignore ) - verbose_proxy_logger.debug(f"Listed files using model: {model_used}") + verbose_proxy_logger.debug("Listed files using model: %s", model_used) elif target_model_names and isinstance(target_model_names, str): target_model_names_list = target_model_names.split(",") @@ -1427,7 +1427,7 @@ async def list_files( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.list_files(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.list_files(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( diff --git a/litellm/proxy/openai_files_endpoints/storage_backend_service.py b/litellm/proxy/openai_files_endpoints/storage_backend_service.py index b9658d8efee..200bad13a9f 100644 --- a/litellm/proxy/openai_files_endpoints/storage_backend_service.py +++ b/litellm/proxy/openai_files_endpoints/storage_backend_service.py @@ -82,7 +82,7 @@ class StorageBackendFileService: file_naming_strategy="uuid", ) - verbose_proxy_logger.debug(f"Storage backend upload complete: backend={target_storage}, url={storage_url}") + verbose_proxy_logger.debug("Storage backend upload complete: backend=%s, url=%s", target_storage, storage_url) # Create file object with storage metadata file_object = StorageBackendFileService._create_file_object_with_storage_metadata( @@ -223,8 +223,10 @@ class StorageBackendFileService: file_object.id = base64_unified_file_id verbose_proxy_logger.debug( - f"Storing file in managed files: unified_id={base64_unified_file_id}, " - f"storage_backend={target_storage}, storage_url={storage_url}" + "Storing file in managed files: unified_id=%s, storage_backend=%s, storage_url=%s", + base64_unified_file_id, + target_storage, + storage_url, ) # Store in managed files diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 0d9b0ab9c49..026a76a767c 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -752,7 +752,7 @@ async def handle_bedrock_passthrough_router_model( is_streaming = any(action in endpoint for action in BEDROCK_STREAMING_ACTIONS) verbose_proxy_logger.debug( - f"Bedrock router passthrough: model='{model}', endpoint='{endpoint}', streaming={is_streaming}" + "Bedrock router passthrough: model='%s', endpoint='%s', streaming=%s", model, endpoint, is_streaming ) # Use the common processing path (same as non-router models) @@ -843,8 +843,8 @@ async def handle_bedrock_count_tokens( if key != "user_api_key_dict": # Don't overwrite user_api_key_dict litellm_params[key] = value # type: ignore - verbose_proxy_logger.debug(f"Count tokens litellm_params: {litellm_params}") - verbose_proxy_logger.debug(f"Resolved model: {resolved_model}") + verbose_proxy_logger.debug("Count tokens litellm_params: %s", litellm_params) + verbose_proxy_logger.debug("Resolved model: %s", resolved_model) # Handle the count tokens request result = await handler.handle_count_tokens_request( @@ -857,13 +857,13 @@ async def handle_bedrock_count_tokens( except BedrockError as e: # Convert BedrockError to HTTPException for FastAPI - verbose_proxy_logger.error(f"BedrockError in handle_bedrock_count_tokens: {e}") + verbose_proxy_logger.error("BedrockError in handle_bedrock_count_tokens: %s", e) raise HTTPException(status_code=e.status_code, detail={"error": e.message}) except HTTPException: # Re-raise HTTP exceptions as-is raise except Exception as e: - verbose_proxy_logger.error(f"Error in handle_bedrock_count_tokens: {e}") + verbose_proxy_logger.error("Error in handle_bedrock_count_tokens: %s", e) raise HTTPException(status_code=500, detail={"error": f"CountTokens processing error: {e}"}) @@ -947,7 +947,9 @@ async def bedrock_llm_proxy_route( ) # Fall back to existing implementation for direct Bedrock models - verbose_proxy_logger.debug(f"Bedrock passthrough: Using direct Bedrock model '{model}' for endpoint '{endpoint}'") + verbose_proxy_logger.debug( + "Bedrock passthrough: Using direct Bedrock model '%s' for endpoint '%s'", model, endpoint + ) data: dict[str, Any] = {} base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) @@ -1148,7 +1150,7 @@ def _resolve_vertex_model_from_router( endpoint = endpoint.replace(model_id, actual_model) except Exception as e: - verbose_proxy_logger.debug(f"Error resolving vertex model from router for model {model_id}: {e}") + verbose_proxy_logger.debug("Error resolving vertex model from router for model %s: %s", model_id, e) return encoded_endpoint, endpoint, vertex_project, vertex_location diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index 5d045ff2852..275283e3e56 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -920,7 +920,7 @@ class AnthropicPassthroughLoggingHandler: } except Exception as e: - verbose_proxy_logger.error(f"Error in batch_creation_handler: {e}") + verbose_proxy_logger.error("Error in batch_creation_handler: %s", e) # Return basic response on error litellm_model_response = ModelResponse() litellm_model_response.id = str(uuid.uuid4()) @@ -1017,7 +1017,9 @@ class AnthropicPassthroughLoggingHandler: ) verbose_proxy_logger.info( - f"Stored Anthropic batch managed object with unified_object_id={unified_object_id}, batch_id={model_object_id}" + "Stored Anthropic batch managed object with unified_object_id=%s, batch_id=%s", + unified_object_id, + model_object_id, ) else: verbose_proxy_logger.warning( @@ -1025,7 +1027,7 @@ class AnthropicPassthroughLoggingHandler: ) except Exception as e: - verbose_proxy_logger.error(f"Error storing Anthropic batch managed object: {e}") + verbose_proxy_logger.error("Error storing Anthropic batch managed object: %s", e) @staticmethod def get_actual_model_id_from_router(model_name: str) -> str: @@ -1038,14 +1040,14 @@ class AnthropicPassthroughLoggingHandler: if model_ids and len(model_ids) > 0: # Use the first model ID found actual_model_id = model_ids[0] - verbose_proxy_logger.info(f"Found model ID in router: {actual_model_id}") + verbose_proxy_logger.info("Found model ID in router: %s", actual_model_id) return actual_model_id else: # Fallback to model name actual_model_id = model_name - verbose_proxy_logger.warning(f"Model not found in router, using model name: {actual_model_id}") + verbose_proxy_logger.warning("Model not found in router, using model name: %s", actual_model_id) return actual_model_id else: # Fallback if router is not available - verbose_proxy_logger.warning(f"Router not available, using model name: {model_name}") + verbose_proxy_logger.warning("Router not available, using model name: %s", model_name) return model_name diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py index 397f1d94a34..97cbe9a0615 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py @@ -203,7 +203,7 @@ class AssemblyAIPassthroughLoggingHandler: return response.json() except Exception as e: - verbose_proxy_logger.exception(f"[Non blocking logging error] Error getting AssemblyAI transcript: {e}") + verbose_proxy_logger.exception("[Non blocking logging error] Error getting AssemblyAI transcript: %s", e) return None def _poll_assembly_for_transcript_response( @@ -275,7 +275,7 @@ class AssemblyAIPassthroughLoggingHandler: return None except Exception as e: - verbose_proxy_logger.exception(f"[Non blocking logging error] Error getting AssemblyAI model info: {e}") + verbose_proxy_logger.exception("[Non blocking logging error] Error getting AssemblyAI model info: %s", e) return None @staticmethod diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index 63414a1c19e..18e56d914c6 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -183,7 +183,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): return cost except Exception as e: - verbose_proxy_logger.warning(f"Error calculating image generation cost: {e}") + verbose_proxy_logger.warning("Error calculating image generation cost: %s", e) return 0.0 @staticmethod @@ -217,7 +217,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): return cost except Exception as e: - verbose_proxy_logger.warning(f"Error calculating image editing cost: {e}") + verbose_proxy_logger.warning("Error calculating image editing cost: %s", e) return 0.0 @staticmethod @@ -445,7 +445,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): } except Exception as e: - verbose_proxy_logger.error(f"Error in OpenAI passthrough cost tracking: {e}") + verbose_proxy_logger.error("Error in OpenAI passthrough cost tracking: %s", e) # Fall back to base handler without cost tracking base_handler = OpenAIPassthroughLoggingHandler() return base_handler.passthrough_chat_handler( @@ -501,7 +501,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): all_openai_chunks.append(transformed_chunk) except (StopIteration, StopAsyncIteration, Exception) as e: - verbose_proxy_logger.debug(f"Error parsing streaming chunk: {e}") + verbose_proxy_logger.debug("Error parsing streaming chunk: %s", e) continue if not all_openai_chunks: @@ -514,7 +514,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): return complete_streaming_response except Exception as e: - verbose_proxy_logger.error(f"Error building complete streaming response: {e}") + verbose_proxy_logger.error("Error building complete streaming response: %s", e) return None @staticmethod @@ -608,7 +608,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): } except Exception as e: - verbose_proxy_logger.error(f"Error in OpenAI streaming passthrough cost tracking: {e}") + verbose_proxy_logger.error("Error in OpenAI streaming passthrough cost tracking: %s", e) return { "result": None, "kwargs": {}, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_ai_live_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_ai_live_passthrough_logging_handler.py index 11672571f3f..fd703b22549 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_ai_live_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_ai_live_passthrough_logging_handler.py @@ -148,11 +148,11 @@ class VertexAILivePassthroughLoggingHandler(BasePassthroughLoggingHandler): # Get model pricing information model_info = get_model_info(model=model, custom_llm_provider=custom_llm_provider) - verbose_proxy_logger.debug(f"Vertex AI Live API model info for '{model}': {model_info}") + verbose_proxy_logger.debug("Vertex AI Live API model info for '%s': %s", model, model_info) # Check if pricing info is available if not model_info or not model_info.get("input_cost_per_token"): - verbose_proxy_logger.error(f"No pricing info found for {model} in local model pricing database") + verbose_proxy_logger.error("No pricing info found for %s in local model pricing database", model) return 0.0 total_cost = 0.0 @@ -221,7 +221,7 @@ class VertexAILivePassthroughLoggingHandler(BasePassthroughLoggingHandler): return total_cost except Exception as e: - verbose_proxy_logger.error(f"Error calculating Vertex AI Live API cost: {e}") + verbose_proxy_logger.error("Error calculating Vertex AI Live API cost: %s", e) return 0.0 @staticmethod @@ -302,7 +302,9 @@ class VertexAILivePassthroughLoggingHandler(BasePassthroughLoggingHandler): # Extract model from request body or kwargs model = kwargs.get("model", "gemini-2.0-flash-live-preview-04-09") custom_llm_provider = kwargs.get("custom_llm_provider", "vertex_ai") - verbose_proxy_logger.debug(f"Vertex AI Live API model: {model}, custom_llm_provider: {custom_llm_provider}") + verbose_proxy_logger.debug( + "Vertex AI Live API model: %s, custom_llm_provider: %s", model, custom_llm_provider + ) # Extract usage metadata from WebSocket messages usage_metadata = self._extract_usage_metadata_from_websocket_messages(websocket_messages) @@ -360,7 +362,7 @@ class VertexAILivePassthroughLoggingHandler(BasePassthroughLoggingHandler): } except Exception as e: - verbose_proxy_logger.error(f"Error in Vertex AI Live API passthrough handler: {e}") + verbose_proxy_logger.error("Error in Vertex AI Live API passthrough handler: %s", e) return { "result": None, "kwargs": kwargs, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index 233127c3fef..bab16d0ba54 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -744,7 +744,7 @@ class VertexPassthroughLoggingHandler: } except Exception as e: - verbose_proxy_logger.error(f"Error in batch_prediction_jobs_handler: {e}") + verbose_proxy_logger.error("Error in batch_prediction_jobs_handler: %s", e) # Return basic response on error litellm_model_response = ModelResponse() litellm_model_response.id = str(uuid.uuid4()) @@ -841,7 +841,9 @@ class VertexPassthroughLoggingHandler: ) verbose_proxy_logger.info( - f"Stored batch managed object with unified_object_id={unified_object_id}, batch_id={model_object_id}" + "Stored batch managed object with unified_object_id=%s, batch_id=%s", + unified_object_id, + model_object_id, ) else: verbose_proxy_logger.warning( @@ -849,7 +851,7 @@ class VertexPassthroughLoggingHandler: ) except Exception as e: - verbose_proxy_logger.error(f"Error storing batch managed object: {e}") + verbose_proxy_logger.error("Error storing batch managed object: %s", e) @staticmethod def get_actual_model_id_from_router(model_name: str) -> str: @@ -864,15 +866,15 @@ class VertexPassthroughLoggingHandler: if model_ids and len(model_ids) > 0: # Use the first model ID found actual_model_id = model_ids[0] - verbose_proxy_logger.info(f"Found model ID in router: {actual_model_id}") + verbose_proxy_logger.info("Found model ID in router: %s", actual_model_id) return actual_model_id else: # Fallback to constructed model name actual_model_id = extracted_model_name - verbose_proxy_logger.warning(f"Model not found in router, using constructed name: {actual_model_id}") + verbose_proxy_logger.warning("Model not found in router, using constructed name: %s", actual_model_id) return actual_model_id else: # Fallback if router is not available extracted_model_name = VertexPassthroughLoggingHandler.extract_model_name_from_vertex_path(model_name) - verbose_proxy_logger.warning(f"Router not available, using constructed model name: {extracted_model_name}") + verbose_proxy_logger.warning("Router not available, using constructed model name: %s", extracted_model_name) return extracted_model_name diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index b8aba215d10..171df5b4d2c 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -294,7 +294,7 @@ async def chat_completion_pass_through_endpoint( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.completion(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.completion(): Exception occured - %s", e) error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), @@ -890,7 +890,7 @@ async def pass_through_request( if "metadata" not in _parsed_body: _parsed_body["metadata"] = {} _parsed_body["metadata"]["guardrails"] = guardrails_to_run - verbose_proxy_logger.debug(f"Added guardrails to passthrough request metadata: {guardrails_to_run}") + verbose_proxy_logger.debug("Added guardrails to passthrough request metadata: %s", guardrails_to_run) ## LOGGING OBJECT ## - initialize before pre_call_hook so guardrails can access it # Surface the requested model (when the body carries one) so logging/spans @@ -1502,7 +1502,7 @@ async def pass_through_request( ) else: verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.pass_through_endpoint(): Exception occured - {e}" + "litellm.proxy.proxy_server.pass_through_endpoint(): Exception occured - %s", e ) ######################################################### @@ -1887,12 +1887,12 @@ async def websocket_passthrough_request( websocket_messages: list[dict[str, Any]] = [] litellm_call_id = str(uuid.uuid4()) - verbose_proxy_logger.info(f"WebSocket passthrough ({endpoint}): Starting WebSocket connection to {target}") + verbose_proxy_logger.info("WebSocket passthrough (%s): Starting WebSocket connection to %s", endpoint, target) # Only accept the WebSocket if requested (for generic usage) if accept_websocket: await websocket.accept() - verbose_proxy_logger.debug(f"WebSocket passthrough ({endpoint}): WebSocket connection accepted") + verbose_proxy_logger.debug("WebSocket passthrough (%s): WebSocket connection accepted", endpoint) # Prepare headers for the upstream connection upstream_headers = custom_headers.copy() @@ -1985,13 +1985,15 @@ async def websocket_passthrough_request( ) try: - verbose_proxy_logger.debug(f"WebSocket passthrough ({endpoint}): Establishing upstream connection to {target}") + verbose_proxy_logger.debug( + "WebSocket passthrough (%s): Establishing upstream connection to %s", endpoint, target + ) async with connect( target, additional_headers=upstream_headers, ) as upstream_ws: verbose_proxy_logger.info( - f"WebSocket passthrough ({endpoint}): Upstream connection established successfully" + "WebSocket passthrough (%s): Upstream connection established successfully", endpoint ) async def forward_client_to_upstream() -> None: @@ -2011,14 +2013,17 @@ async def websocket_passthrough_request( # Try to extract model from client setup message for Vertex AI Live if endpoint and "/vertex_ai/live" in endpoint: verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Processing client message for model extraction" + "WebSocket passthrough (%s): Processing client message for model extraction", + endpoint, ) try: client_message = json.loads(text_data) if isinstance(client_message, dict) and "setup" in client_message: setup_data = client_message["setup"] verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Found setup data in client message: {setup_data}" + "WebSocket passthrough (%s): Found setup data in client message: %s", + endpoint, + setup_data, ) if isinstance(setup_data, dict) and "model" in setup_data: extracted_model = _extract_model_from_vertex_ai_setup(setup_data) @@ -2030,23 +2035,32 @@ async def websocket_passthrough_request( logging_obj.model_call_details["model"] = extracted_model logging_obj.model_call_details["custom_llm_provider"] = "vertex_ai" verbose_proxy_logger.info( - f"WebSocket passthrough ({endpoint}): Successfully extracted model '{extracted_model}' and set provider to 'vertex_ai' from client setup message" + "WebSocket passthrough (%s): Successfully extracted model '%s' and set provider to 'vertex_ai' from client setup message", + endpoint, + extracted_model, ) else: verbose_proxy_logger.warning( - f"WebSocket passthrough ({endpoint}): Failed to extract model from client setup data: {setup_data}" + "WebSocket passthrough (%s): Failed to extract model from client setup data: %s", + endpoint, + setup_data, ) else: verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Setup data does not contain model field: {setup_data}" + "WebSocket passthrough (%s): Setup data does not contain model field: %s", + endpoint, + setup_data, ) else: verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Client message does not contain setup data" + "WebSocket passthrough (%s): Client message does not contain setup data", + endpoint, ) except (json.JSONDecodeError, KeyError, TypeError) as e: verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Client message is not a valid setup message: {e}" + "WebSocket passthrough (%s): Client message is not a valid setup message: %s", + endpoint, + e, ) # Not a JSON message or doesn't contain setup data @@ -2057,7 +2071,7 @@ async def websocket_passthrough_request( raise except Exception: verbose_proxy_logger.exception( - f"WebSocket passthrough ({endpoint}): error forwarding client message" + "WebSocket passthrough (%s): error forwarding client message", endpoint ) await upstream_ws.close() @@ -2070,12 +2084,13 @@ async def websocket_passthrough_request( if isinstance(raw_response, str): raw_response = raw_response.encode("ascii") setup_response = json.loads(raw_response.decode("ascii")) - verbose_proxy_logger.debug(f"Setup response: {setup_response}") + verbose_proxy_logger.debug("Setup response: %s", setup_response) # Extract model and provider from setup response for Vertex AI Live if endpoint and "/vertex_ai/live" in endpoint: verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Processing server setup response for model extraction" + "WebSocket passthrough (%s): Processing server setup response for model extraction", + endpoint, ) extracted_model = _extract_model_from_vertex_ai_setup(setup_response) if extracted_model: @@ -2086,15 +2101,20 @@ async def websocket_passthrough_request( logging_obj.model_call_details["model"] = extracted_model logging_obj.model_call_details["custom_llm_provider"] = "vertex_ai_language_models" verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Successfully extracted model '{extracted_model}' and set provider to 'vertex_ai' from server setup response" + "WebSocket passthrough (%s): Successfully extracted model '%s' and set provider to 'vertex_ai' from server setup response", + endpoint, + extracted_model, ) else: verbose_proxy_logger.warning( - f"WebSocket passthrough ({endpoint}): Failed to extract model from server setup response: {setup_response}" + "WebSocket passthrough (%s): Failed to extract model from server setup response: %s", + endpoint, + setup_response, ) else: verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Not a Vertex AI Live endpoint, skipping model extraction" + "WebSocket passthrough (%s): Not a Vertex AI Live endpoint, skipping model extraction", + endpoint, ) # Send the setup response to the client @@ -2120,14 +2140,14 @@ async def websocket_passthrough_request( pass except (ConnectionClosedOK, ConnectionClosedError) as e: - verbose_proxy_logger.debug(f"Upstream WebSocket connection closed: {e}") + verbose_proxy_logger.debug("Upstream WebSocket connection closed: %s", e) except asyncio.CancelledError: verbose_proxy_logger.debug("asyncio.CancelledError in forward_upstream_to_client") raise except Exception as e: - verbose_proxy_logger.debug(f"Exception in forward_upstream_to_client: {e}") + verbose_proxy_logger.debug("Exception in forward_upstream_to_client: %s", e) verbose_proxy_logger.exception( - f"WebSocket passthrough ({endpoint}): error forwarding upstream message" + "WebSocket passthrough (%s): error forwarding upstream message", endpoint ) raise @@ -2218,7 +2238,7 @@ async def websocket_passthrough_request( ) except InvalidStatus as exc: - verbose_proxy_logger.exception(f"WebSocket passthrough ({endpoint}): upstream rejected WebSocket connection") + verbose_proxy_logger.exception("WebSocket passthrough (%s): upstream rejected WebSocket connection", endpoint) # Prepare request payload for logging request_payload = {} @@ -2244,7 +2264,9 @@ async def websocket_passthrough_request( reason="Upstream connection rejected", ) except Exception as e: - verbose_proxy_logger.exception(f"WebSocket passthrough ({endpoint}): unexpected error while proxying WebSocket") + verbose_proxy_logger.exception( + "WebSocket passthrough (%s): unexpected error while proxying WebSocket", endpoint + ) # Prepare request payload for logging request_payload = {} @@ -2321,8 +2343,9 @@ async def _relay_passthrough_response_bytes( finally: if not upstream_fully_relayed: verbose_proxy_logger.warning( - f"Passthrough stream for {url_route} ended before upstream body was fully relayed; " - f"{bytes_relayed} bytes were sent to the client" + "Passthrough stream for %s ended before upstream body was fully relayed; %s bytes were sent to the client", + url_route, + bytes_relayed, ) await response.aclose() GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( @@ -2370,7 +2393,7 @@ def _extract_model_from_vertex_ai_setup(setup_response: dict) -> str | None: model_name = model_path.split("/models/")[-1] return model_name except Exception as e: - verbose_proxy_logger.debug(f"Error extracting model from setup response: {e}") + verbose_proxy_logger.debug("Error extracting model from setup response: %s", e) return None diff --git a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py index ed35e3d1b46..bd7f7b94f7a 100644 --- a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py +++ b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py @@ -49,12 +49,14 @@ class PassthroughEndpointRouter: custom_llm_provider=custom_llm_provider, region_name=region_name, ) - verbose_router_logger.debug(f"Pass-through llm endpoints router, looking for credentials for {credential_name}") + verbose_router_logger.debug( + "Pass-through llm endpoints router, looking for credentials for %s", credential_name + ) if credential_name in self.credentials: - verbose_router_logger.debug(f"Found credentials for {credential_name}") + verbose_router_logger.debug("Found credentials for %s", credential_name) return self.credentials[credential_name] else: - verbose_router_logger.debug(f"No credentials found for {credential_name}, looking for env variable") + verbose_router_logger.debug("No credentials found for %s, looking for env variable", credential_name) _env_variable_name = self._get_default_env_variable_name_passthrough_endpoint( custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index 9a4a28c7678..b8fd6d757b5 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -89,7 +89,7 @@ class PassThroughStreamingHandler: yield chunk except Exception as e: - verbose_proxy_logger.error(f"Error in chunk_processor: {e}") + verbose_proxy_logger.error("Error in chunk_processor: %s", e) raise finally: # GeneratorExit (raised on client disconnect) is not caught by @@ -115,7 +115,7 @@ class PassThroughStreamingHandler: ) ) except Exception as e: - verbose_proxy_logger.error(f"Error scheduling chunk_processor logging: {e}") + verbose_proxy_logger.error("Error scheduling chunk_processor logging: %s", e) @staticmethod async def _route_streaming_logging_to_handler( @@ -165,7 +165,7 @@ class PassThroughStreamingHandler: **kwargs, ) except Exception as e: - verbose_proxy_logger.error(f"Error in _route_streaming_logging_to_handler: {e}") + verbose_proxy_logger.error("Error in _route_streaming_logging_to_handler: %s", e) @staticmethod def _build_passthrough_logging_result( diff --git a/litellm/proxy/policy_engine/attachment_registry.py b/litellm/proxy/policy_engine/attachment_registry.py index 797f72f7667..9bbe5405965 100644 --- a/litellm/proxy/policy_engine/attachment_registry.py +++ b/litellm/proxy/policy_engine/attachment_registry.py @@ -58,14 +58,14 @@ class AttachmentRegistry: try: attachment = self._parse_attachment(attachment_data) self._attachments.append(attachment) - verbose_proxy_logger.debug(f"Loaded attachment for policy: {attachment.policy}") + verbose_proxy_logger.debug("Loaded attachment for policy: %s", attachment.policy) except Exception as e: - verbose_proxy_logger.error(f"Error loading attachment: {e}") + verbose_proxy_logger.error("Error loading attachment: %s", e) raise ValueError(f"Invalid attachment: {e}") from e self._config_attachments = tuple(self._attachments) self._initialized = True - verbose_proxy_logger.info(f"Loaded {len(self._attachments)} policy attachments") + verbose_proxy_logger.info("Loaded %s policy attachments", len(self._attachments)) def _parse_attachment(self, attachment_data: dict[str, Any]) -> PolicyAttachment: """ @@ -123,9 +123,12 @@ class AttachmentRegistry: } ) verbose_proxy_logger.debug( - f"Attachment matched: policy={attachment.policy}, " - f"matched_via={matched_via}, " - f"context=(team={context.team_alias}, key={context.key_alias}, model={context.model})" + "Attachment matched: policy=%s, matched_via=%s, context=(team=%s, key=%s, model=%s)", + attachment.policy, + matched_via, + context.team_alias, + context.key_alias, + context.model, ) return results @@ -222,7 +225,7 @@ class AttachmentRegistry: """ self._attachments.append(attachment) self._initialized = True - verbose_proxy_logger.debug(f"Added attachment for policy: {attachment.policy}") + verbose_proxy_logger.debug("Added attachment for policy: %s", attachment.policy) def remove_attachments_for_policy(self, policy_name: str) -> int: """ @@ -238,7 +241,7 @@ class AttachmentRegistry: self._attachments = [a for a in self._attachments if a.policy != policy_name] removed_count = original_count - len(self._attachments) if removed_count > 0: - verbose_proxy_logger.debug(f"Removed {removed_count} attachment(s) for policy: {policy_name}") + verbose_proxy_logger.debug("Removed %s attachment(s) for policy: %s", removed_count, policy_name) return removed_count def remove_attachment_by_id(self, attachment_id: str) -> bool: @@ -317,7 +320,7 @@ class AttachmentRegistry: updated_by=created_attachment.updated_by, ) except Exception as e: - verbose_proxy_logger.exception(f"Error adding attachment to DB: {e}") + verbose_proxy_logger.exception("Error adding attachment to DB: %s", e) raise Exception(f"Error adding attachment to DB: {e}") async def delete_attachment_from_db( @@ -353,7 +356,7 @@ class AttachmentRegistry: return {"message": f"Attachment {attachment_id} deleted successfully"} except Exception as e: - verbose_proxy_logger.exception(f"Error deleting attachment from DB: {e}") + verbose_proxy_logger.exception("Error deleting attachment from DB: %s", e) raise Exception(f"Error deleting attachment from DB: {e}") async def get_attachment_by_id_from_db( @@ -393,7 +396,7 @@ class AttachmentRegistry: updated_by=attachment.updated_by, ) except Exception as e: - verbose_proxy_logger.exception(f"Error getting attachment from DB: {e}") + verbose_proxy_logger.exception("Error getting attachment from DB: %s", e) raise Exception(f"Error getting attachment from DB: {e}") async def get_all_attachments_from_db( @@ -431,7 +434,7 @@ class AttachmentRegistry: for a in attachments ] except Exception as e: - verbose_proxy_logger.exception(f"Error getting attachments from DB: {e}") + verbose_proxy_logger.exception("Error getting attachments from DB: %s", e) raise Exception(f"Error getting attachments from DB: {e}") async def sync_attachments_from_db( @@ -463,11 +466,12 @@ class AttachmentRegistry: self._initialized = True verbose_proxy_logger.info( - f"Synced {len(attachments)} attachments from DB to in-memory registry " - f"({len(self._config_attachments)} config-defined attachments preserved)" + "Synced %s attachments from DB to in-memory registry (%s config-defined attachments preserved)", + len(attachments), + len(self._config_attachments), ) except Exception as e: - verbose_proxy_logger.exception(f"Error syncing attachments from DB: {e}") + verbose_proxy_logger.exception("Error syncing attachments from DB: %s", e) raise Exception(f"Error syncing attachments from DB: {e}") diff --git a/litellm/proxy/policy_engine/condition_evaluator.py b/litellm/proxy/policy_engine/condition_evaluator.py index 02268fcd721..2f3a4fa8015 100644 --- a/litellm/proxy/policy_engine/condition_evaluator.py +++ b/litellm/proxy/policy_engine/condition_evaluator.py @@ -48,7 +48,9 @@ class ConditionEvaluator: condition=condition.model, model=context.model, ): - verbose_proxy_logger.debug(f"Condition failed: model={context.model} did not match {condition.model}") + verbose_proxy_logger.debug( + "Condition failed: model=%s did not match %s", context.model, condition.model + ) return False return True diff --git a/litellm/proxy/policy_engine/init_policies.py b/litellm/proxy/policy_engine/init_policies.py index 1facec0898f..c3016afbd6c 100644 --- a/litellm/proxy/policy_engine/init_policies.py +++ b/litellm/proxy/policy_engine/init_policies.py @@ -124,7 +124,7 @@ async def init_policies( Raises: ValueError: If fail_on_error is True and validation errors are found """ - verbose_proxy_logger.info(f"Initializing {len(policies_config)} policies...") + verbose_proxy_logger.info("Initializing %s policies...", len(policies_config)) # Print policies to console on startup _print_policies_on_startup(policies_config, policy_attachments_config) @@ -146,13 +146,13 @@ async def init_policies( if validation_result.errors: for error in validation_result.errors: verbose_proxy_logger.error( - f"Policy validation error in '{error.policy_name}': [{error.error_type}] {error.message}" + "Policy validation error in '%s': [%s] %s", error.policy_name, error.error_type, error.message ) if validation_result.warnings: for warning in validation_result.warnings: verbose_proxy_logger.warning( - f"Policy validation warning in '{warning.policy_name}': [{warning.error_type}] {warning.message}" + "Policy validation warning in '%s': [%s] %s", warning.policy_name, warning.error_type, warning.message ) # Fail if there are errors and fail_on_error is True @@ -165,18 +165,18 @@ async def init_policies( # Load policies into registry (even with warnings) try: policy_registry.load_policies(policies_config) - verbose_proxy_logger.info(f"Successfully loaded {len(policies_config)} policies") + verbose_proxy_logger.info("Successfully loaded %s policies", len(policies_config)) except Exception as e: - verbose_proxy_logger.error(f"Failed to load policies: {e}") + verbose_proxy_logger.error("Failed to load policies: %s", e) raise # Load attachments if provided if policy_attachments_config: try: attachment_registry.load_attachments(policy_attachments_config) - verbose_proxy_logger.info(f"Successfully loaded {len(policy_attachments_config)} policy attachments") + verbose_proxy_logger.info("Successfully loaded %s policy attachments", len(policy_attachments_config)) except Exception as e: - verbose_proxy_logger.error(f"Failed to load policy attachments: {e}") + verbose_proxy_logger.error("Failed to load policy attachments: %s", e) raise return validation_result diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index 983d2da124b..7feb5123093 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -92,7 +92,12 @@ class PipelineExecutor: step_results.append(step_result) verbose_proxy_logger.debug( - f"Pipeline '{policy_name}' step {i}: guardrail={step.guardrail}, outcome={outcome}, action={action}" + "Pipeline '%s' step %s: guardrail=%s, outcome=%s, action=%s", + policy_name, + i, + step.guardrail, + outcome, + action, ) # Forward modified data to next step if pass_data is True @@ -158,7 +163,7 @@ class PipelineExecutor: """ callback = PipelineExecutor.find_guardrail_callback(step.guardrail) if callback is None: - verbose_proxy_logger.warning(f"Pipeline: guardrail '{step.guardrail}' not found in callbacks") + verbose_proxy_logger.warning("Pipeline: guardrail '%s' not found in callbacks", step.guardrail) return ("error", None, f"Guardrail '{step.guardrail}' not found", None) try: @@ -205,7 +210,7 @@ class PipelineExecutor: error_msg = _extract_error_message(e) return ("fail", None, error_msg, e) else: - verbose_proxy_logger.error(f"Pipeline: unexpected error from guardrail '{step.guardrail}': {e}") + verbose_proxy_logger.error("Pipeline: unexpected error from guardrail '%s': %s", step.guardrail, e) return ("error", None, str(e), e) @staticmethod diff --git a/litellm/proxy/policy_engine/policy_endpoints.py b/litellm/proxy/policy_engine/policy_endpoints.py index 1d0fd67cfc8..9959c2302b5 100644 --- a/litellm/proxy/policy_engine/policy_endpoints.py +++ b/litellm/proxy/policy_engine/policy_endpoints.py @@ -142,7 +142,7 @@ async def list_policies(version_status: str | None = None): policies = db_policies + config_policies return PolicyListDBResponse(policies=policies, total_count=len(policies)) except Exception as e: - verbose_proxy_logger.exception(f"Error listing policies: {e}") + verbose_proxy_logger.exception("Error listing policies: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -201,7 +201,7 @@ async def create_policy( ) return result except Exception as e: - verbose_proxy_logger.exception(f"Error creating policy: {e}") + verbose_proxy_logger.exception("Error creating policy: %s", e) if "unique constraint" in str(e).lower(): raise HTTPException( status_code=400, @@ -236,7 +236,7 @@ async def list_policy_versions(policy_name: str): prisma_client=prisma_client, ) except Exception as e: - verbose_proxy_logger.exception(f"Error listing policy versions: {e}") + verbose_proxy_logger.exception("Error listing policy versions: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -269,7 +269,7 @@ async def create_policy_version( created_by=created_by, ) except Exception as e: - verbose_proxy_logger.exception(f"Error creating policy version: {e}") + verbose_proxy_logger.exception("Error creating policy version: %s", e) if "not found" in str(e).lower() or "no production" in str(e).lower(): raise HTTPException(status_code=404, detail=str(e)) raise HTTPException(status_code=500, detail=str(e)) @@ -308,7 +308,7 @@ async def update_policy_version_status( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error updating version status: {e}") + verbose_proxy_logger.exception("Error updating version status: %s", e) if "invalid status" in str(e).lower() or "only draft" in str(e).lower() or "cannot promote" in str(e).lower(): raise HTTPException(status_code=400, detail=str(e)) if "not found" in str(e).lower(): @@ -341,7 +341,7 @@ async def compare_policy_versions( prisma_client=prisma_client, ) except Exception as e: - verbose_proxy_logger.exception(f"Error comparing versions: {e}") + verbose_proxy_logger.exception("Error comparing versions: %s", e) if "not found" in str(e).lower(): raise HTTPException(status_code=404, detail=str(e)) raise HTTPException(status_code=500, detail=str(e)) @@ -367,7 +367,7 @@ async def delete_all_policy_versions(policy_name: str): prisma_client=prisma_client, ) except Exception as e: - verbose_proxy_logger.exception(f"Error deleting all versions: {e}") + verbose_proxy_logger.exception("Error deleting all versions: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -408,7 +408,7 @@ async def get_policy(policy_id: str): except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error getting policy: {e}") + verbose_proxy_logger.exception("Error getting policy: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -467,7 +467,7 @@ async def update_policy( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error updating policy: {e}") + verbose_proxy_logger.exception("Error updating policy: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -516,7 +516,7 @@ async def delete_policy(policy_id: str): except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error deleting policy: {e}") + verbose_proxy_logger.exception("Error deleting policy: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -577,7 +577,7 @@ async def get_resolved_guardrails(policy_id: str): except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: - verbose_proxy_logger.exception(f"Error resolving guardrails: {e}") + verbose_proxy_logger.exception("Error resolving guardrails: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -640,7 +640,7 @@ async def test_pipeline( ) return result.model_dump() except Exception as e: - verbose_proxy_logger.exception(f"Error testing pipeline: {e}") + verbose_proxy_logger.exception("Error testing pipeline: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -699,7 +699,7 @@ async def list_policy_attachments(): attachments = db_attachments + config_attachments return PolicyAttachmentListResponse(attachments=attachments, total_count=len(attachments)) except Exception as e: - verbose_proxy_logger.exception(f"Error listing policy attachments: {e}") + verbose_proxy_logger.exception("Error listing policy attachments: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -791,7 +791,7 @@ async def create_policy_attachment( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error creating policy attachment: {e}") + verbose_proxy_logger.exception("Error creating policy attachment: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -830,7 +830,7 @@ async def get_policy_attachment(attachment_id: str): except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error getting policy attachment: {e}") + verbose_proxy_logger.exception("Error getting policy attachment: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -881,5 +881,5 @@ async def delete_policy_attachment(attachment_id: str): except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error deleting policy attachment: {e}") + verbose_proxy_logger.exception("Error deleting policy attachment: %s", e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/litellm/proxy/policy_engine/policy_registry.py b/litellm/proxy/policy_engine/policy_registry.py index 07a4c2abac6..6031f4a372e 100644 --- a/litellm/proxy/policy_engine/policy_registry.py +++ b/litellm/proxy/policy_engine/policy_registry.py @@ -185,15 +185,15 @@ class PolicyRegistry: try: policy = self._parse_policy(policy_name, policy_data) self._policies[policy_name] = policy - verbose_proxy_logger.debug(f"Loaded policy: {policy_name}") + verbose_proxy_logger.debug("Loaded policy: %s", policy_name) except Exception as e: - verbose_proxy_logger.error(f"Error loading policy '{policy_name}': {e}") + verbose_proxy_logger.error("Error loading policy '%s': %s", policy_name, e) raise ValueError(f"Invalid policy '{policy_name}': {e}") from e self._config_policies = dict(self._policies) self._sources = {policy_name: "config" for policy_name in self._policies} self._initialized = True - verbose_proxy_logger.info(f"Loaded {len(self._policies)} policies") + verbose_proxy_logger.info("Loaded %s policies", len(self._policies)) def _parse_policy(self, policy_name: str, policy_data: dict[str, Any]) -> Policy: """ @@ -336,7 +336,7 @@ class PolicyRegistry: if source == "config": self._config_policies = {**self._config_policies, policy_name: policy} self._initialized = True - verbose_proxy_logger.debug(f"Added/updated policy: {policy_name}") + verbose_proxy_logger.debug("Added/updated policy: %s", policy_name) def remove_policy(self, policy_name: str) -> bool: """ @@ -355,11 +355,11 @@ class PolicyRegistry: if config_fallback is not None: self._policies[policy_name] = config_fallback self._sources = {**self._sources, policy_name: "config"} - verbose_proxy_logger.debug(f"Removed policy: {policy_name}; restored config-defined version") + verbose_proxy_logger.debug("Removed policy: %s; restored config-defined version", policy_name) return True del self._policies[policy_name] self._sources = {name: source for name, source in self._sources.items() if name != policy_name} - verbose_proxy_logger.debug(f"Removed policy: {policy_name}") + verbose_proxy_logger.debug("Removed policy: %s", policy_name) return True # ───────────────────────────────────────────────────────────────────────── @@ -432,7 +432,7 @@ class PolicyRegistry: return _row_to_policy_db_response(created_policy) except Exception as e: - verbose_proxy_logger.exception(f"Error adding policy to DB: {e}") + verbose_proxy_logger.exception("Error adding policy to DB: %s", e) raise Exception(f"Error adding policy to DB: {e}") async def update_policy_in_db( @@ -496,7 +496,7 @@ class PolicyRegistry: return _row_to_policy_db_response(updated_policy) except Exception as e: - verbose_proxy_logger.exception(f"Error updating policy in DB: {e}") + verbose_proxy_logger.exception("Error updating policy in DB: %s", e) raise Exception(f"Error updating policy in DB: {e}") async def delete_policy_from_db( @@ -546,7 +546,7 @@ class PolicyRegistry: return result except Exception as e: - verbose_proxy_logger.exception(f"Error deleting policy from DB: {e}") + verbose_proxy_logger.exception("Error deleting policy from DB: %s", e) raise Exception(f"Error deleting policy from DB: {e}") async def get_policy_by_id_from_db( @@ -572,7 +572,7 @@ class PolicyRegistry: return _row_to_policy_db_response(policy) except Exception as e: - verbose_proxy_logger.exception(f"Error getting policy from DB: {e}") + verbose_proxy_logger.exception("Error getting policy from DB: %s", e) raise Exception(f"Error getting policy from DB: {e}") def get_policy_by_id_for_request(self, policy_id: str) -> tuple[str, Policy] | None: @@ -619,7 +619,7 @@ class PolicyRegistry: return [_row_to_policy_db_response(p) for p in policies] except Exception as e: - verbose_proxy_logger.exception(f"Error getting policies from DB: {e}") + verbose_proxy_logger.exception("Error getting policies from DB: %s", e) raise Exception(f"Error getting policies from DB: {e}") async def sync_policies_from_db( @@ -653,7 +653,8 @@ class PolicyRegistry: } for policy_name in set(db_policies) & set(self._config_policies): verbose_proxy_logger.warning( - f"Policy '{policy_name}' is defined in both config.yaml and the DB; the DB version takes precedence" + "Policy '%s' is defined in both config.yaml and the DB; the DB version takes precedence", + policy_name, ) config_sources: Mapping[str, Literal["db", "config"]] = {name: "config" for name in self._config_policies} db_sources: Mapping[str, Literal["db", "config"]] = {name: "db" for name in db_policies} @@ -683,12 +684,13 @@ class PolicyRegistry: self._initialized = True verbose_proxy_logger.info( - f"Synced {len(production)} production policies and {len(non_production)} " - "draft/published (by ID) from DB to in-memory registry " - f"({len(self._config_policies)} config-defined policies preserved)" + "Synced %s production policies and %s draft/published (by ID) from DB to in-memory registry (%s config-defined policies preserved)", + len(production), + len(non_production), + len(self._config_policies), ) except Exception as e: - verbose_proxy_logger.exception(f"Error syncing policies from DB: {e}") + verbose_proxy_logger.exception("Error syncing policies from DB: %s", e) raise Exception(f"Error syncing policies from DB: {e}") async def resolve_guardrails_from_db( @@ -741,7 +743,7 @@ class PolicyRegistry: return sorted(resolved_policy.guardrails) except Exception as e: - verbose_proxy_logger.exception(f"Error resolving guardrails from DB: {e}") + verbose_proxy_logger.exception("Error resolving guardrails from DB: %s", e) raise Exception(f"Error resolving guardrails from DB: {e}") async def get_versions_by_policy_name( @@ -771,7 +773,7 @@ class PolicyRegistry: total_count=len(versions), ) except Exception as e: - verbose_proxy_logger.exception(f"Error getting versions: {e}") + verbose_proxy_logger.exception("Error getting versions: %s", e) raise Exception(f"Error getting versions: {e}") async def create_new_version( @@ -857,7 +859,7 @@ class PolicyRegistry: created = await _policy_table(prisma_client).create(data=data) return _row_to_policy_db_response(created) except Exception as e: - verbose_proxy_logger.exception(f"Error creating new version: {e}") + verbose_proxy_logger.exception("Error creating new version: %s", e) raise Exception(f"Error creating new version: {e}") async def update_version_status( @@ -962,7 +964,7 @@ class PolicyRegistry: return _row_to_policy_db_response(updated) except Exception as e: - verbose_proxy_logger.exception(f"Error updating version status: {e}") + verbose_proxy_logger.exception("Error updating version status: %s", e) raise Exception(f"Error updating version status: {e}") async def compare_versions( @@ -1015,7 +1017,7 @@ class PolicyRegistry: field_diffs=field_diffs, ) except Exception as e: - verbose_proxy_logger.exception(f"Error comparing versions: {e}") + verbose_proxy_logger.exception("Error comparing versions: %s", e) raise Exception(f"Error comparing versions: {e}") async def delete_all_versions( @@ -1046,7 +1048,7 @@ class PolicyRegistry: } return {"message": message} except Exception as e: - verbose_proxy_logger.exception(f"Error deleting all versions: {e}") + verbose_proxy_logger.exception("Error deleting all versions: %s", e) raise Exception(f"Error deleting all versions: {e}") diff --git a/litellm/proxy/policy_engine/policy_resolve_endpoints.py b/litellm/proxy/policy_engine/policy_resolve_endpoints.py index e5c2693cf21..575a9cbacc2 100644 --- a/litellm/proxy/policy_engine/policy_resolve_endpoints.py +++ b/litellm/proxy/policy_engine/policy_resolve_endpoints.py @@ -298,7 +298,7 @@ async def resolve_policies_for_context( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error resolving policies: {e}") + verbose_proxy_logger.exception("Error resolving policies: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -408,5 +408,5 @@ async def estimate_attachment_impact( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error estimating attachment impact: {e}") + verbose_proxy_logger.exception("Error estimating attachment impact: %s", e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/litellm/proxy/policy_engine/policy_resolver.py b/litellm/proxy/policy_engine/policy_resolver.py index 65c8236a0b3..355bc10c003 100644 --- a/litellm/proxy/policy_engine/policy_resolver.py +++ b/litellm/proxy/policy_engine/policy_resolver.py @@ -47,7 +47,7 @@ class PolicyResolver: visited = set() if policy_name in visited: - verbose_proxy_logger.warning(f"Circular inheritance detected for policy '{policy_name}'") + verbose_proxy_logger.warning("Circular inheritance detected for policy '%s'", policy_name) return [] policy = policies.get(policy_name) @@ -106,7 +106,7 @@ class PolicyResolver: context=context, ): verbose_proxy_logger.debug( - f"Policy '{chain_policy_name}' condition did not match, skipping guardrails" + "Policy '%s' condition did not match, skipping guardrails", chain_policy_name ) continue @@ -163,8 +163,10 @@ class PolicyResolver: if not matching_policy_names: verbose_proxy_logger.debug( - f"No policies match context: team_alias={context.team_alias}, " - f"key_alias={context.key_alias}, model={context.model}" + "No policies match context: team_alias=%s, key_alias=%s, model=%s", + context.team_alias, + context.key_alias, + context.model, ) return [] @@ -178,10 +180,10 @@ class PolicyResolver: context=context, ) all_guardrails.update(resolved.guardrails) - verbose_proxy_logger.debug(f"Policy '{policy_name}' contributes guardrails: {resolved.guardrails}") + verbose_proxy_logger.debug("Policy '%s' contributes guardrails: %s", policy_name, resolved.guardrails) result = list(all_guardrails) - verbose_proxy_logger.debug(f"Final guardrails for context: {result}") + verbose_proxy_logger.debug("Final guardrails for context: %s", result) return result @@ -229,7 +231,7 @@ class PolicyResolver: if policy.pipeline is not None: pipelines.append((policy_name, policy.pipeline)) verbose_proxy_logger.debug( - f"Policy '{policy_name}' has pipeline with {len(policy.pipeline.steps)} steps" + "Policy '%s' has pipeline with %s steps", policy_name, len(policy.pipeline.steps) ) return pipelines diff --git a/litellm/proxy/policy_engine/policy_validator.py b/litellm/proxy/policy_engine/policy_validator.py index 67f7b37472c..c8799a71307 100644 --- a/litellm/proxy/policy_engine/policy_validator.py +++ b/litellm/proxy/policy_engine/policy_validator.py @@ -78,7 +78,7 @@ class PolicyValidator: guardrails = IN_MEMORY_GUARDRAIL_HANDLER.list_in_memory_guardrails() return {g.get("guardrail_name", "") for g in guardrails if g.get("guardrail_name")} except Exception as e: - verbose_proxy_logger.warning(f"Could not get guardrails from registry: {e}") + verbose_proxy_logger.warning("Could not get guardrails from registry: %s", e) return set() async def check_team_alias_exists(self, team_alias: str) -> bool: @@ -100,7 +100,7 @@ class PolicyValidator: ) return team is not None except Exception as e: - verbose_proxy_logger.warning(f"Could not check team alias '{team_alias}': {e}") + verbose_proxy_logger.warning("Could not check team alias '%s': %s", team_alias, e) return True # Assume valid on error async def check_key_alias_exists(self, key_alias: str) -> bool: @@ -122,7 +122,7 @@ class PolicyValidator: ) return key is not None except Exception as e: - verbose_proxy_logger.warning(f"Could not check key alias '{key_alias}': {e}") + verbose_proxy_logger.warning("Could not check key alias '%s': %s", key_alias, e) return True # Assume valid on error def check_model_exists(self, model: str) -> bool: @@ -151,7 +151,7 @@ class PolicyValidator: return False except Exception as e: - verbose_proxy_logger.warning(f"Could not check model '{model}': {e}") + verbose_proxy_logger.warning("Could not check model '%s': %s", model, e) return True # Assume valid on error @staticmethod diff --git a/litellm/proxy/prisma_migration.py b/litellm/proxy/prisma_migration.py index 0f6944fac79..b09a49cbff5 100644 --- a/litellm/proxy/prisma_migration.py +++ b/litellm/proxy/prisma_migration.py @@ -16,9 +16,9 @@ run_server(["--skip_server_startup"], standalone_mode=False) # run prisma generate verbose_proxy_logger.info("Running 'prisma generate'...") result = subprocess.run(["prisma", "generate"], capture_output=True, text=True) -verbose_proxy_logger.info(f"'prisma generate' stdout: {result.stdout}") # Log stdout +verbose_proxy_logger.info("'prisma generate' stdout: %s", result.stdout) # Log stdout exit_code = result.returncode if exit_code != 0: - verbose_proxy_logger.info(f"'prisma generate' failed with exit code {exit_code}.") - verbose_proxy_logger.error(f"'prisma generate' stderr: {result.stderr}") # Log stderr + verbose_proxy_logger.info("'prisma generate' failed with exit code %s.", exit_code) + verbose_proxy_logger.error("'prisma generate' stderr: %s", result.stderr) # Log stderr diff --git a/litellm/proxy/prometheus_cleanup.py b/litellm/proxy/prometheus_cleanup.py index 2a22b1c5fae..7cf27193b9a 100644 --- a/litellm/proxy/prometheus_cleanup.py +++ b/litellm/proxy/prometheus_cleanup.py @@ -21,9 +21,9 @@ def wipe_directory(directory: str) -> None: os.remove(filepath) deleted += 1 except OSError as e: - verbose_proxy_logger.warning(f"Failed to delete stale prometheus file {filepath}: {e}") + verbose_proxy_logger.warning("Failed to delete stale prometheus file %s: %s", filepath, e) if deleted: - verbose_proxy_logger.info(f"Prometheus cleanup: wiped {deleted} stale .db files from {directory}") + verbose_proxy_logger.info("Prometheus cleanup: wiped %s stale .db files from %s", deleted, directory) def mark_worker_exit(worker_pid: int) -> None: @@ -34,6 +34,6 @@ def mark_worker_exit(worker_pid: int) -> None: from prometheus_client import multiprocess multiprocess.mark_process_dead(worker_pid) - verbose_proxy_logger.info(f"Prometheus cleanup: marked worker {worker_pid} as dead") + verbose_proxy_logger.info("Prometheus cleanup: marked worker %s as dead", worker_pid) except Exception as e: - verbose_proxy_logger.warning(f"Failed to mark prometheus worker {worker_pid} as dead: {e}") + verbose_proxy_logger.warning("Failed to mark prometheus worker %s as dead: %s", worker_pid, e) diff --git a/litellm/proxy/prompts/init_prompts.py b/litellm/proxy/prompts/init_prompts.py index 67e961b2da8..752762f07b9 100644 --- a/litellm/proxy/prompts/init_prompts.py +++ b/litellm/proxy/prompts/init_prompts.py @@ -23,4 +23,4 @@ def init_prompts( if initialized_prompt: prompt_list.append(initialized_prompt) - verbose_proxy_logger.debug(f"\nPrompt List:{prompt_list}\n") + verbose_proxy_logger.debug("\nPrompt List:%s\n", prompt_list) diff --git a/litellm/proxy/prompts/prompt_endpoints.py b/litellm/proxy/prompts/prompt_endpoints.py index 89087a3fdd5..fd0437a4265 100644 --- a/litellm/proxy/prompts/prompt_endpoints.py +++ b/litellm/proxy/prompts/prompt_endpoints.py @@ -755,7 +755,7 @@ async def create_prompt( return initialized_prompt except Exception as e: - verbose_proxy_logger.exception(f"Error creating prompt: {e}") + verbose_proxy_logger.exception("Error creating prompt: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -871,7 +871,7 @@ async def update_prompt( except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.exception(f"Error updating prompt: {e}") + verbose_proxy_logger.exception("Error updating prompt: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -970,7 +970,7 @@ async def delete_prompt( except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.exception(f"Error deleting prompt: {e}") + verbose_proxy_logger.exception("Error deleting prompt: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -1111,7 +1111,7 @@ async def patch_prompt( except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.exception(f"Error patching prompt: {e}") + verbose_proxy_logger.exception("Error patching prompt: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -1252,7 +1252,7 @@ async def test_prompt( except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: - verbose_proxy_logger.exception(f"Error testing prompt: {e}") + verbose_proxy_logger.exception("Error testing prompt: %s", e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/litellm/proxy/prompts/prompt_registry.py b/litellm/proxy/prompts/prompt_registry.py index 9e00d4e7c63..78b449873a8 100644 --- a/litellm/proxy/prompts/prompt_registry.py +++ b/litellm/proxy/prompts/prompt_registry.py @@ -51,7 +51,7 @@ def get_prompt_initializer_from_integrations(): module_path = f"litellm.integrations.{item}" try: # Import the module - verbose_proxy_logger.debug(f"Discovering prompt integrations in: {module_path}") + verbose_proxy_logger.debug("Discovering prompt integrations in: %s", module_path) module = importlib.import_module(module_path) @@ -61,22 +61,22 @@ def get_prompt_initializer_from_integrations(): if isinstance(registry, dict): discovered_initializers.update(registry) verbose_proxy_logger.debug( - f"Found prompt_initializer_registry in {module_path}: {list(registry.keys())}" + "Found prompt_initializer_registry in %s: %s", module_path, list(registry.keys()) ) except ImportError as e: - verbose_proxy_logger.error(f"Could not import {module_path}: {e}") + verbose_proxy_logger.error("Could not import %s: %s", module_path, e) continue except Exception as e: - verbose_proxy_logger.error(f"Error processing {module_path}: {e}") + verbose_proxy_logger.error("Error processing %s: %s", module_path, e) continue verbose_proxy_logger.debug( - f"Discovered {len(discovered_initializers)} prompt initializers: {list(discovered_initializers.keys())}" + "Discovered %s prompt initializers: %s", len(discovered_initializers), list(discovered_initializers.keys()) ) except Exception as e: - verbose_proxy_logger.error(f"Error discovering prompt initializers: {e}") + verbose_proxy_logger.error("Error discovering prompt initializers: %s", e) return discovered_initializers diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 58b4ec88192..82cbb7fda48 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -163,8 +163,8 @@ def _with_query_value(url: str, key: str, value: str) -> str: def append_query_params(url: str | None, params: dict) -> str: from litellm._logging import verbose_proxy_logger - verbose_proxy_logger.debug(f"url: {url}") - verbose_proxy_logger.debug(f"params: {params}") + verbose_proxy_logger.debug("url: %s", url) + verbose_proxy_logger.debug("params: %s", params) if not isinstance(url, str) or url == "": # Preserve previous startup behavior when DATABASE_URL is absent. # Returning an empty string avoids urlparse type errors in test/dev flows. diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 2ff39164d80..16a63540d4d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -862,12 +862,16 @@ async def _initialize_shared_aiohttp_session(): session = ClientSession(connector=connector) verbose_proxy_logger.info( - f"SESSION REUSE: Created shared aiohttp session for connection pooling (ID: {id(session)}, " - f"limit={AIOHTTP_CONNECTOR_LIMIT}, limit_per_host={AIOHTTP_CONNECTOR_LIMIT_PER_HOST})" + "SESSION REUSE: Created shared aiohttp session for connection pooling (ID: %s, limit=%s, limit_per_host=%s)", + id(session), + AIOHTTP_CONNECTOR_LIMIT, + AIOHTTP_CONNECTOR_LIMIT_PER_HOST, ) return session except Exception as e: - verbose_proxy_logger.warning(f"Failed to create shared aiohttp session: {e}. Continuing without session reuse.") + verbose_proxy_logger.warning( + "Failed to create shared aiohttp session: %s. Continuing without session reuse.", e + ) return None @@ -918,7 +922,7 @@ async def proxy_startup_event(app: FastAPI): raise ## CHECK PREMIUM USER - verbose_proxy_logger.debug(f"litellm.proxy.proxy_server.py::startup() - CHECKING PREMIUM USER - {premium_user}") + verbose_proxy_logger.debug("litellm.proxy.proxy_server.py::startup() - CHECKING PREMIUM USER - %s", premium_user) if premium_user is False: premium_user = _license_check.is_premium() @@ -973,9 +977,9 @@ async def proxy_startup_event(app: FastAPI): async def _run_pw_migration(): try: result = await migrate_passwords_to_scrypt_async(prisma_client) - verbose_proxy_logger.info(f"Password migration: {result}") + verbose_proxy_logger.info("Password migration: %s", result) except Exception as e: - verbose_proxy_logger.warning(f"Password migration skipped: {e}") + verbose_proxy_logger.warning("Password migration skipped: %s", e) asyncio.create_task(_run_pw_migration()) @@ -1049,14 +1053,14 @@ async def proxy_startup_event(app: FastAPI): verbose_proxy_logger.debug("About to initialize semantic tool filter") _config = proxy_config.get_config_state() _litellm_settings = _config.get("litellm_settings", {}) - verbose_proxy_logger.debug(f"litellm_settings keys = {list(_litellm_settings.keys())}") + verbose_proxy_logger.debug("litellm_settings keys = %s", list(_litellm_settings.keys())) await ProxyStartupEvent._initialize_semantic_tool_filter( llm_router=llm_router, litellm_settings=_litellm_settings, ) verbose_proxy_logger.debug("After semantic tool filter initialization") except Exception as e: - verbose_proxy_logger.error(f"Semantic filter init failed: {e}", exc_info=True) + verbose_proxy_logger.error("Semantic filter init failed: %s", e, exc_info=True) ## JWT AUTH ## ProxyStartupEvent._initialize_jwt_auth( @@ -1133,7 +1137,7 @@ async def proxy_startup_event(app: FastAPI): await shared_aiohttp_session.close() verbose_proxy_logger.info("SESSION REUSE: Closed shared aiohttp session") except Exception as e: - verbose_proxy_logger.error(f"Error closing shared aiohttp session: {e}") + verbose_proxy_logger.error("Error closing shared aiohttp session: %s", e) # Shutdown event - stop RDS IAM token refresh background task if ( @@ -1144,14 +1148,14 @@ async def proxy_startup_event(app: FastAPI): try: await prisma_client.db.stop_token_refresh_task() except Exception as e: - verbose_proxy_logger.error(f"Error stopping token refresh task: {e}") + verbose_proxy_logger.error("Error stopping token refresh task: %s", e) # Shutdown event - stop Prisma DB health watchdog task if prisma_client is not None and hasattr(prisma_client, "stop_db_health_watchdog_task"): try: await prisma_client.stop_db_health_watchdog_task() except Exception as e: - verbose_proxy_logger.error(f"Error stopping DB health watchdog task: {e}") + verbose_proxy_logger.error("Error stopping DB health watchdog task: %s", e) await proxy_config.stop_config_sync_subscriber() @@ -1598,7 +1602,7 @@ try: # Primary signal: marker file created by Dockerfile marker_file = os.path.join(ui_dir, ".litellm_ui_ready") if os.path.exists(marker_file): - verbose_proxy_logger.debug(f"Found UI ready marker: {marker_file}") + verbose_proxy_logger.debug("Found UI ready marker: %s", marker_file) return True # Fallback signal: Detect restructuring pattern @@ -1617,11 +1621,11 @@ try: if os.path.exists(index_path): # Found at least one restructured route - this proves the pattern verbose_proxy_logger.debug( - f"Detected restructured UI via pattern: found {entry.name}/index.html" + "Detected restructured UI via pattern: found %s/index.html", entry.name ) return True except (PermissionError, OSError) as e: - verbose_proxy_logger.debug(f"Could not scan {ui_dir} for restructuring detection: {e}") + verbose_proxy_logger.debug("Could not scan %s for restructuring detection: %s", ui_dir, e) return False # No restructured routes found @@ -1641,7 +1645,7 @@ try: target_path, dirs_exist_ok=True, ) - verbose_proxy_logger.info(f"Successfully populated UI at {target_path}") + verbose_proxy_logger.info("Successfully populated UI at %s", target_path) return True, "" else: return False, "Source or target directory state invalid" @@ -1665,7 +1669,7 @@ try: # Validate packaged UI before proceeding if not _validate_ui_directory(packaged_ui_path): verbose_proxy_logger.error( - f"Packaged UI at {packaged_ui_path} is invalid or incomplete. UI may not function correctly." + "Packaged UI at %s is invalid or incomplete. UI may not function correctly.", packaged_ui_path ) # Decision tree for UI path selection: @@ -1684,20 +1688,20 @@ try: # Case 2: Runtime UI exists and is ready if has_content and is_pre_restructured: - verbose_proxy_logger.info(f"Using pre-restructured UI at {runtime_ui_path}") + verbose_proxy_logger.info("Using pre-restructured UI at %s", runtime_ui_path) ui_path = runtime_ui_path # Case 3: Runtime UI exists but needs restructuring elif has_content and not is_pre_restructured: verbose_proxy_logger.warning( - f"UI at {runtime_ui_path} has content but is not properly restructured. " - f"Will attempt to restructure in place." + "UI at %s has content but is not properly restructured. Will attempt to restructure in place.", + runtime_ui_path, ) ui_path = runtime_ui_path # Case 4: Runtime UI missing - try to populate else: - verbose_proxy_logger.info(f"UI not found at {runtime_ui_path}. Attempting to populate from packaged UI.") + verbose_proxy_logger.info("UI not found at %s. Attempting to populate from packaged UI.", runtime_ui_path) success, error = _try_populate_ui_directory(packaged_ui_path, runtime_ui_path) @@ -1707,20 +1711,20 @@ try: else: # Case 4b: Population failed - fall back to packaged UI verbose_proxy_logger.warning( - f"Failed to populate UI at {runtime_ui_path}: {error}. " - f"Falling back to packaged UI at {packaged_ui_path}. " - f"For read-only deployments, pre-build UI in Dockerfile " - f"or set LITELLM_UI_PATH to a writable emptyDir volume." + "Failed to populate UI at %s: %s. Falling back to packaged UI at %s. For read-only deployments, pre-build UI in Dockerfile or set LITELLM_UI_PATH to a writable emptyDir volume.", + runtime_ui_path, + error, + packaged_ui_path, ) ui_path = packaged_ui_path else: # Case 1: Using packaged UI directly (local development) - verbose_proxy_logger.info(f"Using packaged UI directory: {packaged_ui_path}") + verbose_proxy_logger.info("Using packaged UI directory: %s", packaged_ui_path) ui_path = packaged_ui_path # Validate final UI path if not _validate_ui_directory(ui_path): - verbose_proxy_logger.error(f"Selected UI path {ui_path} is invalid or incomplete. UI may not work correctly.") + verbose_proxy_logger.error("Selected UI path %s is invalid or incomplete. UI may not work correctly.", ui_path) # Only modify files if a custom server root path is set AND filesystem is writable if server_root_path and server_root_path != "/": @@ -1729,9 +1733,8 @@ try: if not is_writable: verbose_proxy_logger.warning( - f"Cannot apply server_root_path replacements to UI at {ui_path}: " - f"path is not writable. Ensure server_root_path is '/' or pre-process " - f"UI files in Dockerfile with custom server_root_path." + "Cannot apply server_root_path replacements to UI at %s: path is not writable. Ensure server_root_path is '/' or pre-process UI files in Dockerfile with custom server_root_path.", + ui_path, ) else: # Iterate through files in the UI directory @@ -1825,20 +1828,19 @@ try: is_writable = os.access(ui_path, os.W_OK) if is_pre_restructured: - verbose_proxy_logger.info(f"Skipping UI restructuring: {ui_path} is already pre-restructured") + verbose_proxy_logger.info("Skipping UI restructuring: %s is already pre-restructured", ui_path) elif not is_writable: verbose_proxy_logger.warning( - f"Cannot restructure UI at {ui_path}: path is not writable. " - f"UI may not work correctly for extensionless routes. " - f"Pre-build and restructure UI in Dockerfile for read-only deployments." + "Cannot restructure UI at %s: path is not writable. UI may not work correctly for extensionless routes. Pre-build and restructure UI in Dockerfile for read-only deployments.", + ui_path, ) else: _restructure_ui_html_files(ui_path) - verbose_proxy_logger.info(f"Restructured UI directory: {ui_path}") + verbose_proxy_logger.info("Restructured UI directory: %s", ui_path) except PermissionError as e: - verbose_proxy_logger.exception(f"Permission error while restructuring UI directory {ui_path}: {e}") + verbose_proxy_logger.exception("Permission error while restructuring UI directory %s: %s", ui_path, e) except Exception as e: - verbose_proxy_logger.exception(f"Error while restructuring UI directory {ui_path}: {e}") + verbose_proxy_logger.exception("Error while restructuring UI directory %s: %s", ui_path, e) except Exception: pass @@ -2826,7 +2828,7 @@ async def update_cache( hashed_token = token verbose_proxy_logger.debug("_update_key_cache: hashed_token=%s", hashed_token) existing_spend_obj = await user_api_key_cache.async_get_cache(key=hashed_token, model_type=UserAPIKeyAuth) - verbose_proxy_logger.debug(f"_update_key_cache: existing_spend_obj={existing_spend_obj}") + verbose_proxy_logger.debug("_update_key_cache: existing_spend_obj=%s", existing_spend_obj) if existing_spend_obj is None: return @@ -2889,7 +2891,7 @@ async def update_cache( if existing_spend_obj is None: return verbose_proxy_logger.debug( - f"_update_user_db: existing spend: {existing_spend_obj}; response_cost: {response_cost}" + "_update_user_db: existing spend: %s; response_cost: %s", existing_spend_obj, response_cost ) existing_spend = existing_spend_obj.spend or 0.0 @@ -2939,7 +2941,7 @@ async def update_cache( if existing_spend_obj is None: return verbose_proxy_logger.debug( - f"_update_end_user_db: existing spend: {existing_spend_obj}; response_cost: {response_cost}" + "_update_end_user_db: existing spend: %s; response_cost: %s", existing_spend_obj, response_cost ) existing_spend = existing_spend_obj.spend or 0.0 @@ -2981,7 +2983,7 @@ async def update_cache( if existing_spend_obj is None: return verbose_proxy_logger.debug( - f"_update_team_db: existing spend: {existing_spend_obj}; response_cost: {response_cost}" + "_update_team_db: existing spend: %s; response_cost: %s", existing_spend_obj, response_cost ) existing_spend: float = existing_spend_obj.spend or 0.0 @@ -3031,7 +3033,10 @@ async def update_cache( continue verbose_proxy_logger.debug( - f"_update_tag_cache: existing spend for tag={tag_name}: {existing_tag_obj}; response_cost: {response_cost}" + "_update_tag_cache: existing spend for tag=%s: %s; response_cost: %s", + tag_name, + existing_tag_obj, + response_cost, ) existing_spend = existing_tag_obj.spend or 0.0 @@ -3101,9 +3106,10 @@ def run_ollama_serve(): with open(os.devnull, "w") as devnull: subprocess.Popen(command, stdout=devnull, stderr=devnull) except Exception as e: - verbose_proxy_logger.debug(f""" - LiteLLM Warning: proxy started with `ollama` model\n`ollama serve` failed with Exception{e}. \nEnsure you run `ollama serve` - """) + verbose_proxy_logger.debug( + "\n LiteLLM Warning: proxy started with `ollama` model\n`ollama serve` failed with Exception%s. \nEnsure you run `ollama serve`\n ", + e, + ) def _get_process_rss_mb() -> float | None: @@ -4026,7 +4032,7 @@ class ProxyConfig: dict: Processed configuration dictionary. """ if depth > max_depth: - verbose_proxy_logger.warning(f"Maximum recursion depth ({max_depth}) reached while processing config.") + verbose_proxy_logger.warning("Maximum recursion depth (%s) reached while processing config.", max_depth) return config for key, value in config.items(): @@ -4230,7 +4236,9 @@ class ProxyConfig: return copy.deepcopy(self.config) except Exception as e: verbose_proxy_logger.debug( - f"ProxyConfig:get_config_state(): Error returning copy of config state. self.config={self.config}\nError: {e}" + "ProxyConfig:get_config_state(): Error returning copy of config state. self.config=%s\nError: %s", + self.config, + e, ) return {} @@ -4294,7 +4302,7 @@ class ProxyConfig: search_tool_typed: SearchToolTypedDict = SearchToolTypedDict(**search_tool) # type: ignore search_tools_parsed.append(search_tool_typed) except Exception as e: - verbose_proxy_logger.error(f"Error parsing search tool {search_tool_name}: {e}") + verbose_proxy_logger.error("Error parsing search tool %s: %s", search_tool_name, e) continue return search_tools_parsed if search_tools_parsed else None @@ -4484,7 +4492,7 @@ class ProxyConfig: ) ) if litellm.cache is not None: - verbose_proxy_logger.debug(f"{blue_color_code}Set Cache on LiteLLM Proxy{reset_color_code}") + verbose_proxy_logger.debug("%sSet Cache on LiteLLM Proxy%s", blue_color_code, reset_color_code) elif key == "cache" and value is False: pass elif key == "guardrails": @@ -4504,7 +4512,7 @@ class ProxyConfig: set_global_prompt_directory(value) verbose_proxy_logger.info( - f"{blue_color_code}Set Global Prompt Directory on LiteLLM Proxy{reset_color_code}" + "%sSet Global Prompt Directory on LiteLLM Proxy%s", blue_color_code, reset_color_code ) elif key == "global_bitbucket_config": from litellm.integrations.bitbucket import ( @@ -4513,14 +4521,14 @@ class ProxyConfig: set_global_bitbucket_config(value) verbose_proxy_logger.info( - f"{blue_color_code}Set Global BitBucket Config on LiteLLM Proxy{reset_color_code}" + "%sSet Global BitBucket Config on LiteLLM Proxy%s", blue_color_code, reset_color_code ) elif key == "global_gitlab_config": from litellm.integrations.gitlab import set_global_gitlab_config set_global_gitlab_config(value) verbose_proxy_logger.info( - f"{blue_color_code}Set Global Gitlab Config on LiteLLM Proxy{reset_color_code}" + "%sSet Global Gitlab Config on LiteLLM Proxy%s", blue_color_code, reset_color_code ) elif key == "priority_reservation_settings": from litellm.types.utils import PriorityReservationSettings @@ -4542,7 +4550,7 @@ class ProxyConfig: elif key == "post_call_rules": litellm.post_call_rules = [get_instance_fn(value=value, config_file_path=config_file_path)] - verbose_proxy_logger.debug(f"litellm.post_call_rules: {litellm.post_call_rules}") + verbose_proxy_logger.debug("litellm.post_call_rules: %s", litellm.post_call_rules) elif key == "max_budget": litellm.max_budget = float(value) elif key == "max_internal_user_budget": @@ -4558,7 +4566,11 @@ class ProxyConfig: else value ) verbose_proxy_logger.debug( - f"{blue_color_code} setting litellm.{key}={_redact_general_setting_value(key, litellm.default_internal_user_params, is_full_admin=False)}{reset_color_code}" + "%s setting litellm.%s=%s%s", + blue_color_code, + key, + _redact_general_setting_value(key, litellm.default_internal_user_params, is_full_admin=False), + reset_color_code, ) elif key == "custom_provider_map": from litellm.utils import custom_llm_setup @@ -4662,12 +4674,20 @@ class ProxyConfig: native_background_mode = background_mode.get("native_background_mode", []) polling_cache_ttl = background_mode.get("ttl", 3600) verbose_proxy_logger.debug( - f"{blue_color_code} Initialized polling via cache: enabled={polling_via_cache_enabled}, native_background_mode={native_background_mode}, ttl={polling_cache_ttl}{reset_color_code}" + "%s Initialized polling via cache: enabled=%s, native_background_mode=%s, ttl=%s%s", + blue_color_code, + polling_via_cache_enabled, + native_background_mode, + polling_cache_ttl, + reset_color_code, ) elif key == "max_ui_session_budget": litellm.max_ui_session_budget = float(value) if value is not None else None verbose_proxy_logger.debug( - f"{blue_color_code} setting litellm.max_ui_session_budget={litellm.max_ui_session_budget}{reset_color_code}" + "%s setting litellm.max_ui_session_budget=%s%s", + blue_color_code, + litellm.max_ui_session_budget, + reset_color_code, ) elif key == "default_team_settings": for idx, team_setting in enumerate(value): # run through pydantic validation @@ -4682,7 +4702,11 @@ class ProxyConfig: f"team_id missing from default_team_settings at index={idx}\npassed in value={type(team_setting)}" ) verbose_proxy_logger.debug( - f"{blue_color_code} setting litellm.{key}={_redact_general_setting_value(key, value, is_full_admin=False)}{reset_color_code}" + "%s setting litellm.%s=%s%s", + blue_color_code, + key, + _redact_general_setting_value(key, value, is_full_admin=False), + reset_color_code, ) setattr(litellm, key, value) elif key == "upperbound_key_generate_params": @@ -4696,7 +4720,9 @@ class ProxyConfig: elif key == "json_logs" and value is True: litellm.json_logs = True litellm._turn_on_json() - verbose_proxy_logger.debug(f"{blue_color_code} Enabled JSON logging via config{reset_color_code}") + verbose_proxy_logger.debug( + "%s Enabled JSON logging via config%s", blue_color_code, reset_color_code + ) elif key == "budget_reset_time": from litellm.proxy.common_utils.timezone_utils import ( parse_budget_reset_time, @@ -4706,7 +4732,11 @@ class ProxyConfig: setattr(litellm, key, value) else: verbose_proxy_logger.debug( - f"{blue_color_code} setting litellm.{key}={_redact_general_setting_value(key, value, is_full_admin=False)}{reset_color_code}" + "%s setting litellm.%s=%s%s", + blue_color_code, + key, + _redact_general_setting_value(key, value, is_full_admin=False), + reset_color_code, ) setattr(litellm, key, value) if key == "request_timeout": @@ -5035,7 +5065,7 @@ class ProxyConfig: ) else: verbose_proxy_logger.warning( - f"Key '{k}' is not a valid argument for Router.__init__(). Ignoring this key." + "Key '%s' is not a valid argument for Router.__init__(). Ignoring this key.", k ) router = litellm.Router( **router_params, @@ -5168,7 +5198,7 @@ class ProxyConfig: policy_attachments_config = config.get("policy_attachments", None) - verbose_proxy_logger.info(f"Policy engine: found {len(policies_config)} policies in config") + verbose_proxy_logger.info("Policy engine: found %s policies in config", len(policies_config)) # Initialize policies await init_policies( @@ -5191,7 +5221,7 @@ class ProxyConfig: # Ensure proxy_logging_obj.alerting is set for all alerting types _alerting_value = general_settings.get("alerting", None) - verbose_proxy_logger.debug(f"_load_alerting_settings: Calling update_values with alerting={_alerting_value}") + verbose_proxy_logger.debug("_load_alerting_settings: Calling update_values with alerting=%s", _alerting_value) proxy_logging_obj.update_values( alerting=_alerting_value, alerting_threshold=general_settings.get("alerting_threshold", 600), @@ -5409,7 +5439,7 @@ class ProxyConfig: else: verbose_proxy_logger.error( - f"Invalid model added to proxy db. Invalid litellm params. litellm_params={_litellm_params}" + "Invalid model added to proxy db. Invalid litellm params. litellm_params=%s", _litellm_params ) continue # skip to next model _model_info = self.get_model_info_with_id(model=m, db_model=True) ## 👈 FLAG = True for db_models @@ -5439,7 +5469,7 @@ class ProxyConfig: _litellm_params = LiteLLM_Params.model_validate(_litellm_params) else: verbose_proxy_logger.error( - f"Invalid model added to proxy db. Invalid litellm params. litellm_params={_litellm_params}" + "Invalid model added to proxy db. Invalid litellm params. litellm_params=%s", _litellm_params ) continue # skip to next model @@ -5488,13 +5518,13 @@ class ProxyConfig: models_list: list = new_models if isinstance(new_models, list) else [] if llm_router is None and master_key is not None: - verbose_proxy_logger.debug(f"len new_models: {len(models_list)}") + verbose_proxy_logger.debug("len new_models: %s", len(models_list)) _model_list: list = self.decrypt_model_list_from_db(new_models=models_list) # Only create router if we have models or search_tools to route # Router can function with model_list=[] if search_tools are configured if len(_model_list) > 0 or search_tools: - verbose_proxy_logger.debug(f"_model_list: {_model_list}") + verbose_proxy_logger.debug("_model_list: %s", _model_list) llm_router = litellm.Router( model_list=_model_list, router_general_settings=RouterGeneralSettings( @@ -5503,9 +5533,9 @@ class ProxyConfig: search_tools=search_tools, ignore_invalid_deployments=True, ) - verbose_proxy_logger.debug(f"updated llm_router: {llm_router}") + verbose_proxy_logger.debug("updated llm_router: %s", llm_router) else: - verbose_proxy_logger.debug(f"len new_models: {len(models_list)}") + verbose_proxy_logger.debug("len new_models: %s", len(models_list)) if search_tools is not None and llm_router is not None: llm_router.search_tools = search_tools ## DELETE MODEL LOGIC @@ -5515,7 +5545,7 @@ class ProxyConfig: self._add_deployment(db_models=models_list) except Exception as e: - verbose_proxy_logger.exception(f"Error adding/deleting model to llm_router: {e}") + verbose_proxy_logger.exception("Error adding/deleting model to llm_router: %s", e) if llm_router is not None: llm_model_list = llm_router.get_model_list() @@ -5800,7 +5830,10 @@ class ProxyConfig: item for item in _general_settings["alerting"] if item not in general_settings["alerting"] ] verbose_proxy_logger.debug( - f"Merging alerting values: YAML={general_settings['alerting']}, DB={_general_settings['alerting']}, Merged={_merged_alerting}" + "Merging alerting values: YAML=%s, DB=%s, Merged=%s", + general_settings["alerting"], + _general_settings["alerting"], + _merged_alerting, ) general_settings["alerting"] = _merged_alerting # Use update_values to properly set alerting for both slack and email @@ -5879,9 +5912,9 @@ class ProxyConfig: replace_existing=True, misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, ) - verbose_proxy_logger.info(f"Spend log cleanup rescheduled with cron: {cleanup_cron}") + verbose_proxy_logger.info("Spend log cleanup rescheduled with cron: %s", cleanup_cron) except ValueError: - verbose_proxy_logger.error(f"Invalid maximum_spend_logs_cleanup_cron value: {cleanup_cron}") + verbose_proxy_logger.error("Invalid maximum_spend_logs_cleanup_cron value: %s", cleanup_cron) else: # Interval-based scheduling (existing behavior) from litellm.litellm_core_utils.duration_parser import ( @@ -5900,7 +5933,7 @@ class ProxyConfig: replace_existing=True, misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, ) - verbose_proxy_logger.info(f"Spend log cleanup rescheduled with interval: {retention_interval}") + verbose_proxy_logger.info("Spend log cleanup rescheduled with interval: %s", retention_interval) except ValueError: verbose_proxy_logger.error("Invalid maximum_spend_logs_retention_interval value") @@ -6135,7 +6168,7 @@ class ProxyConfig: # If supported_db_objects is set, only load specified objects if not isinstance(supported_db_objects, list): verbose_proxy_logger.warning( - f"supported_db_objects is not a list, got {type(supported_db_objects)}. Loading all objects." + "supported_db_objects is not a list, got %s. Loading all objects.", type(supported_db_objects) ) return True @@ -6159,7 +6192,7 @@ class ProxyConfig: return new_models except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy_server.py::add_deployment() - Error getting new models from DB - {e}" + "litellm.proxy_server.py::add_deployment() - Error getting new models from DB - %s", e ) return None @@ -6216,7 +6249,7 @@ class ProxyConfig: await self._init_non_llm_objects_in_db(prisma_client=prisma_client) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - %s", e) return still_desired_ids @@ -6250,7 +6283,7 @@ class ProxyConfig: try: await subscriber.stop() except Exception as e: - verbose_proxy_logger.error(f"Error stopping config sync subscriber: {e}") + verbose_proxy_logger.error("Error stopping config sync subscriber: %s", e) async def _init_non_llm_objects_in_db(self, prisma_client: PrismaClient): """ @@ -6371,7 +6404,7 @@ class ProxyConfig: self._last_semantic_filter_config = mcp_semantic_filter_config.copy() except Exception as e: - verbose_proxy_logger.exception(f"Error initializing semantic filter settings from DB: {e}") + verbose_proxy_logger.exception("Error initializing semantic filter settings from DB: %s", e) async def _init_sso_settings_in_db(self, prisma_client: PrismaClient): """ @@ -6391,7 +6424,9 @@ class ProxyConfig: uppercase_sso_settings = {key.upper(): value for key, value in sso_settings.sso_settings.items()} self._decrypt_and_set_db_env_variables(environment_variables=uppercase_sso_settings) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_sso_settings_in_db - {e}") + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.py::ProxyConfig:_init_sso_settings_in_db - %s", e + ) async def _init_hashicorp_vault_config_override(self, prisma_client: PrismaClient): """ @@ -6492,7 +6527,7 @@ class ProxyConfig: f"Model cost map reload triggered by interval. Hours since last reload: {hours_since_last_reload:.2f}, Interval: {interval_hours}" ) except Exception as e: - verbose_proxy_logger.warning(f"Error parsing last reload time: {e}") + verbose_proxy_logger.warning("Error parsing last reload time: %s", e) # If we can't parse the last reload time, reload anyway should_reload = True else: @@ -6544,11 +6579,12 @@ class ProxyConfig: await evict_config_param("model_cost_map_reload_config") verbose_proxy_logger.info( - f"Model cost map reloaded successfully. Models count: {len(new_model_cost_map) if new_model_cost_map else 0}" + "Model cost map reloaded successfully. Models count: %s", + len(new_model_cost_map) if new_model_cost_map else 0, ) except Exception as e: - verbose_proxy_logger.exception(f"Error in _check_and_reload_model_cost_map: {e}") + verbose_proxy_logger.exception("Error in _check_and_reload_model_cost_map: %s", e) async def _check_and_reload_anthropic_beta_headers(self, prisma_client: PrismaClient): """ @@ -6592,7 +6628,7 @@ class ProxyConfig: f"Anthropic beta headers reload triggered by interval. Hours since last reload: {hours_since_last_reload:.2f}, Interval: {interval_hours}" ) except Exception as e: - verbose_proxy_logger.warning(f"Error parsing last reload time: {e}") + verbose_proxy_logger.warning("Error parsing last reload time: %s", e) # If we can't parse the last reload time, reload anyway should_reload = True else: @@ -6641,11 +6677,11 @@ class ProxyConfig: # Count providers in config provider_count = sum(1 for k in new_config.keys() if k != "provider_aliases" and k != "description") verbose_proxy_logger.info( - f"Anthropic beta headers config reloaded successfully. Providers: {provider_count}" + "Anthropic beta headers config reloaded successfully. Providers: %s", provider_count ) except Exception as e: - verbose_proxy_logger.exception(f"Error in _check_and_reload_anthropic_beta_headers: {e}") + verbose_proxy_logger.exception("Error in _check_and_reload_anthropic_beta_headers: %s", e) def _get_prompt_spec_for_db_prompt(self, db_prompt): """ @@ -6674,7 +6710,7 @@ class ProxyConfig: prompt_spec = self._get_prompt_spec_for_db_prompt(db_prompt=prompt) IN_MEMORY_PROMPT_REGISTRY.initialize_prompt(prompt=prompt_spec) except Exception as e: - verbose_proxy_logger.debug(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_prompts_in_db - {e}") + verbose_proxy_logger.debug("litellm.proxy.proxy_server.py::ProxyConfig:_init_prompts_in_db - %s", e) async def _init_guardrails_in_db(self, prisma_client: PrismaClient): from litellm.proxy.guardrails.guardrail_registry import ( @@ -6701,7 +6737,7 @@ class ProxyConfig: # pod. Config-loaded entries are never touched. IN_MEMORY_GUARDRAIL_HANDLER.reconcile_db_guardrails(db_guardrail_ids=db_guardrail_ids) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_guardrails_in_db - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:_init_guardrails_in_db - %s", e) async def _init_policies_in_db(self, prisma_client: PrismaClient): """ @@ -6725,7 +6761,7 @@ class ProxyConfig: verbose_proxy_logger.debug("Successfully synced policies and attachments from DB") except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_policies_in_db - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:_init_policies_in_db - %s", e) async def _init_tool_policy_in_db(self, prisma_client: PrismaClient): """ @@ -6739,7 +6775,7 @@ class ProxyConfig: await registry.sync_tool_policy_from_db(prisma_client=prisma_client) verbose_proxy_logger.debug("Successfully synced tool policy from DB") except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_tool_policy_in_db - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:_init_tool_policy_in_db - %s", e) async def _init_vector_stores_in_db(self, prisma_client: PrismaClient): from litellm.vector_stores.vector_store_registry import VectorStoreRegistry @@ -6757,7 +6793,7 @@ class ProxyConfig: litellm.vector_store_registry.add_vector_store_to_registry(vector_store=vector_store) except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.py::ProxyConfig:_init_vector_stores_in_db - {e}" + "litellm.proxy.proxy_server.py::ProxyConfig:_init_vector_stores_in_db - %s", e ) async def _init_vector_store_indexes_in_db(self, prisma_client: PrismaClient): @@ -6781,7 +6817,7 @@ class ProxyConfig: litellm.vector_store_index_registry.upsert_vector_store_index(vector_store_index=vector_store_index) except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.py::ProxyConfig:_init_vector_stores_in_db - {e}" + "litellm.proxy.proxy_server.py::ProxyConfig:_init_vector_stores_in_db - %s", e ) async def _init_mcp_servers_in_db(self): @@ -6806,7 +6842,7 @@ class ProxyConfig: await backfill_null_oauth2_flows(prisma_client) except Exception as e: # noqa: BLE001 verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db backfill - {e}" + "litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db backfill - %s", e ) try: @@ -6814,13 +6850,13 @@ class ProxyConfig: await backfill_discovery_stamped_issuers(prisma_client) except Exception as e: # noqa: BLE001 verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db issuer stamp backfill - {e}" + "litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db issuer stamp backfill - %s", e ) try: await global_mcp_server_manager.reload_servers_from_database() except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db - %s", e) async def init_mcp_servers_from_db(self) -> None: if self._should_load_db_object(object_type="mcp"): @@ -6848,7 +6884,7 @@ class ProxyConfig: await global_mcp_server_manager.reload_servers_from_database() except Exception as e: # noqa: BLE001 # scheduled job: a reload failure must not kill the recurring retry verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.py::ProxyConfig:reload_mcp_servers_from_db - {e}" + "litellm.proxy.proxy_server.py::ProxyConfig:reload_mcp_servers_from_db - %s", e ) async def _init_agents_in_db(self, prisma_client: PrismaClient): @@ -6860,7 +6896,7 @@ class ProxyConfig: db_agents = await AGENT_REGISTRY.get_all_agents_from_db(prisma_client=prisma_client) AGENT_REGISTRY.load_agents_from_db_and_config(db_agents=db_agents) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_agents_in_db - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:_init_agents_in_db - %s", e) async def _init_search_tools_in_db(self, prisma_client: PrismaClient): """ @@ -6885,13 +6921,15 @@ class ProxyConfig: ) verbose_proxy_logger.info( - f"Loading {len(search_tools)} search tool(s) into router " - f"({len(config_search_tools)} from config, {len(db_search_tools)} from database)" + "Loading %s search tool(s) into router (%s from config, %s from database)", + len(search_tools), + len(config_search_tools), + len(db_search_tools), ) if llm_router is not None and search_tools: await SearchAPIRouter.update_router_search_tools(router_instance=llm_router, search_tools=search_tools) - verbose_proxy_logger.info(f"Successfully loaded {len(search_tools)} search tool(s) into router") + verbose_proxy_logger.info("Successfully loaded %s search tool(s) into router", len(search_tools)) elif llm_router is not None: verbose_proxy_logger.debug("No search tools found in config or database, skipping router update") else: @@ -6900,7 +6938,9 @@ class ProxyConfig: ) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_search_tools_in_db - {e}") + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.py::ProxyConfig:_init_search_tools_in_db - %s", e + ) @staticmethod def _merge_config_and_db_search_tools( @@ -6966,7 +7006,7 @@ class ProxyConfig: CredentialAccessor.upsert_credentials(credentials) # upsert credentials that are in the all-up list except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy_server.py::get_credentials() - Error getting credentials from DB - {e}" + "litellm.proxy_server.py::get_credentials() - Error getting credentials from DB - %s", e ) return [] @@ -7153,7 +7193,7 @@ async def async_assistants_data_generator(response, user_api_key_dict: UserAPIKe yield f"data: {done_message}\n\n" except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.async_assistants_data_generator(): Exception occured - {e}" + "litellm.proxy.proxy_server.async_assistants_data_generator(): Exception occured - %s", e ) await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, @@ -7161,7 +7201,8 @@ async def async_assistants_data_generator(response, user_api_key_dict: UserAPIKe request_data=request_data, ) verbose_proxy_logger.debug( - f"\033[1;31mAn error occurred: {e}\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`" + "\x1b[1;31mAn error occurred: %s\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`", + e, ) if isinstance(e, HTTPException): raise e @@ -7632,14 +7673,15 @@ async def async_data_generator( client_disconnected = True raise except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.async_data_generator(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.async_data_generator(): Exception occured - %s", e) await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=request_data, ) verbose_proxy_logger.debug( - f"\033[1;31mAn error occurred: {e}\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`" + "\x1b[1;31mAn error occurred: %s\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`", + e, ) if isinstance(e, HTTPException): @@ -7881,8 +7923,9 @@ class ProxyStartupEvent: return verbose_proxy_logger.debug( - f"Initializing semantic tool filter: llm_router={llm_router is not None}, " - f"config={mcp_semantic_filter_config}" + "Initializing semantic tool filter: llm_router=%s, config=%s", + llm_router is not None, + mcp_semantic_filter_config, ) hook = await SemanticToolFilterHook.initialize_from_config( config=mcp_semantic_filter_config, @@ -8274,9 +8317,9 @@ class ProxyStartupEvent: replace_existing=True, misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, ) - verbose_proxy_logger.info(f"Spend log cleanup scheduled with cron: {cleanup_cron}") + verbose_proxy_logger.info("Spend log cleanup scheduled with cron: %s", cleanup_cron) except ValueError: - verbose_proxy_logger.error(f"Invalid maximum_spend_logs_cleanup_cron value: {cleanup_cron}") + verbose_proxy_logger.error("Invalid maximum_spend_logs_cleanup_cron value: %s", cleanup_cron) else: # Interval-based scheduling (existing behavior) retention_interval = general_settings.get("maximum_spend_logs_retention_interval", "1d") @@ -8318,7 +8361,7 @@ class ProxyStartupEvent: verbose_proxy_logger.info("Batch cost check job scheduled successfully") except Exception as e: - verbose_proxy_logger.debug(f"Failed to setup batch cost checking: {e}") + verbose_proxy_logger.debug("Failed to setup batch cost checking: %s", e) verbose_proxy_logger.debug( "Checking batch cost for LiteLLM Managed Files is an Enterprise Feature. Skipping..." ) @@ -8347,7 +8390,7 @@ class ProxyStartupEvent: verbose_proxy_logger.info("Responses cost check job scheduled successfully") except Exception as e: - verbose_proxy_logger.debug(f"Failed to setup responses cost checking: {e}") + verbose_proxy_logger.debug("Failed to setup responses cost checking: %s", e) verbose_proxy_logger.debug( "Checking responses cost for LiteLLM Managed Files is an Enterprise Feature. Skipping..." ) @@ -8359,8 +8402,8 @@ class ProxyStartupEvent: # Start the scheduler immediately without processing backlogs scheduler.start(paused=False) verbose_proxy_logger.info( - f"APScheduler started with memory leak prevention settings: " - f"removed jitter, increased intervals, misfire_grace_time={APSCHEDULER_MISFIRE_GRACE_TIME}" + "APScheduler started with memory leak prevention settings: removed jitter, increased intervals, misfire_grace_time=%s", + APSCHEDULER_MISFIRE_GRACE_TIME, ) @classmethod @@ -8448,7 +8491,7 @@ class ProxyStartupEvent: ) key_rotation_enabled: bool | None = str_to_bool(LITELLM_KEY_ROTATION_ENABLED) - verbose_proxy_logger.debug(f"key_rotation_enabled: {key_rotation_enabled}") + verbose_proxy_logger.debug("key_rotation_enabled: %s", key_rotation_enabled) if key_rotation_enabled is True: try: @@ -8465,7 +8508,8 @@ class ProxyStartupEvent: pod_lock_manager=pod_lock_manager, ) verbose_proxy_logger.debug( - f"Key rotation background job scheduled every {LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS} seconds (LITELLM_KEY_ROTATION_ENABLED=true)" + "Key rotation background job scheduled every %s seconds (LITELLM_KEY_ROTATION_ENABLED=true)", + LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS, ) scheduler.add_job( key_rotation_manager.process_rotations, @@ -8476,7 +8520,7 @@ class ProxyStartupEvent: else: verbose_proxy_logger.warning("Key rotation enabled but prisma_client not available") except Exception as e: - verbose_proxy_logger.warning(f"Failed to setup key rotation job: {e}") + verbose_proxy_logger.warning("Failed to setup key rotation job: %s", e) else: verbose_proxy_logger.debug("Key rotation disabled (set LITELLM_KEY_ROTATION_ENABLED=true to enable)") @@ -8503,7 +8547,7 @@ class ProxyStartupEvent: expired_ui_session_key_cleanup_enabled: bool | None = str_to_bool( LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_ENABLED ) - verbose_proxy_logger.debug(f"expired_ui_session_key_cleanup_enabled: {expired_ui_session_key_cleanup_enabled}") + verbose_proxy_logger.debug("expired_ui_session_key_cleanup_enabled: %s", expired_ui_session_key_cleanup_enabled) if expired_ui_session_key_cleanup_enabled is True: try: @@ -8519,11 +8563,8 @@ class ProxyStartupEvent: pod_lock_manager=pod_lock_manager, ) verbose_proxy_logger.debug( - "Expired UI session key cleanup background job scheduled " - "every " - f"{LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_INTERVAL_SECONDS} " - "seconds " - "(LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_ENABLED=true)" + "Expired UI session key cleanup background job scheduled every %s seconds (LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_ENABLED=true)", + LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_INTERVAL_SECONDS, ) scheduler.add_job( expired_ui_session_key_cleanup_manager.cleanup_expired_keys, @@ -8536,7 +8577,7 @@ class ProxyStartupEvent: "Expired UI session key cleanup enabled but prisma_client not available" ) except Exception as e: - verbose_proxy_logger.warning(f"Failed to setup expired UI session key cleanup job: {e}") + verbose_proxy_logger.warning("Failed to setup expired UI session key cleanup job: %s", e) else: verbose_proxy_logger.debug( "Expired UI session key cleanup disabled (set " @@ -9383,7 +9424,7 @@ async def completion( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.completion(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.completion(): Exception occured - %s", e) error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), @@ -9622,7 +9663,7 @@ async def moderations( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.moderations(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.moderations(): Exception occured - %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "message", str(e)), @@ -9768,7 +9809,7 @@ async def audio_speech( original_exception=e, request_data=data, ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.audio_speech(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.audio_speech(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) raise e @@ -9910,7 +9951,7 @@ async def audio_transcriptions( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.audio_transcription(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.audio_transcription(): Exception occured - %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "message", str(e.detail)), @@ -10196,7 +10237,7 @@ async def get_assistants( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.get_assistants(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.get_assistants(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10287,7 +10328,7 @@ async def create_assistant( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.create_assistant(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.create_assistant(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10376,7 +10417,7 @@ async def delete_assistant( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.delete_assistant(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.delete_assistant(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10465,7 +10506,7 @@ async def create_threads( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.create_threads(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.create_threads(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10552,7 +10593,7 @@ async def get_thread( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.get_thread(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.get_thread(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10643,7 +10684,7 @@ async def add_messages( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.add_messages(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.add_messages(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10730,7 +10771,7 @@ async def get_messages( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.get_messages(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.get_messages(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10831,7 +10872,7 @@ async def run_thread( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.run_thread(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.run_thread(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10970,8 +11011,9 @@ async def _try_provider_token_count( code=result.status_code or 500, ) verbose_proxy_logger.warning( - f"Provider token counting failed ({result.status_code}): {result.error_message}. " - "Falling back to local tokenizer." + "Provider token counting failed (%s): %s. Falling back to local tokenizer.", + result.status_code, + result.error_message, ) return None return result @@ -11768,7 +11810,7 @@ async def _apply_search_filter_to_models( ) search_total_count = router_models_count + db_models_total_count except Exception as e: - verbose_proxy_logger.exception(f"Error querying database models with search: {e}") + verbose_proxy_logger.exception("Error querying database models with search: %s", e) search_total_count = router_models_count else: search_total_count = router_models_count @@ -11903,7 +11945,7 @@ def _sort_models( sorted_models = sorted(all_models, key=get_sort_key, reverse=reverse) return sorted_models except Exception as e: - verbose_proxy_logger.exception(f"Error sorting models by {sort_by}: {e}") + verbose_proxy_logger.exception("Error sorting models by %s: %s", sort_by, e) return all_models @@ -11951,7 +11993,12 @@ def _paginate_models_response( paginated_models = all_models[skip : skip + size] verbose_proxy_logger.debug( - f"Pagination: skip={skip}, take={size}, total_count={total_count}, total_pages={total_pages}, search={search}" + "Pagination: skip=%s, take=%s, total_count=%s, total_pages=%s, search=%s", + skip, + size, + total_count, + total_pages, + search, ) return { @@ -11979,11 +12026,11 @@ async def _load_team_object_for_model_filter(team_id: str, prisma_client: Prisma try: team_db_object = await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id}) if team_db_object is None: - verbose_proxy_logger.warning(f"Team {team_id} not found in database") + verbose_proxy_logger.warning("Team %s not found in database", team_id) return None return LiteLLM_TeamTable.model_validate(team_db_object.model_dump()) except Exception as e: - verbose_proxy_logger.exception(f"Error fetching team {team_id}: {e}") + verbose_proxy_logger.exception("Error fetching team %s: %s", team_id, e) return None @@ -12033,7 +12080,7 @@ async def _gather_team_accessible_model_ids( if db_model.model_id: team_accessible_model_ids.add(db_model.model_id) except Exception as e: - verbose_proxy_logger.debug(f"Error querying database models for team {team_id}: {e}") + verbose_proxy_logger.debug("Error querying database models for team %s: %s", team_id, e) return team_accessible_model_ids @@ -12171,7 +12218,7 @@ async def _find_model_by_id( if decrypted_models: found_model = decrypted_models[0] except Exception as e: - verbose_proxy_logger.exception(f"Error querying database for modelId {model_id}: {e}") + verbose_proxy_logger.exception("Error querying database for modelId %s: %s", model_id, e) # If model found, verify search filter if provided if found_model is not None: @@ -13787,7 +13834,7 @@ async def login_v2(request: Request): json_response.set_cookie(key="token", value=jwt_token) return json_response except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.login_v2(): Exception occurred - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.login_v2(): Exception occurred - %s", e) if isinstance(e, ProxyException): raise e elif isinstance(e, HTTPException): @@ -13864,7 +13911,7 @@ async def login_v3(request: Request): status_code=status.HTTP_200_OK, ) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.login_v3(): Exception occurred - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.login_v3(): Exception occurred - %s", e) if isinstance(e, ProxyException): raise e elif isinstance(e, HTTPException): @@ -13937,7 +13984,7 @@ async def login_v3_exchange(request: Request): except ProxyException: raise except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.login_v3_exchange(): Exception occurred - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.login_v3_exchange(): Exception occurred - %s", e) raise ProxyException( message=str(e), type=ProxyErrorTypes.auth_error, @@ -14305,11 +14352,12 @@ async def get_image(): if not os.path.exists(assets_dir): try: os.makedirs(assets_dir, exist_ok=True) - verbose_proxy_logger.debug(f"Created assets directory at {assets_dir}") + verbose_proxy_logger.debug("Created assets directory at %s", assets_dir) except (PermissionError, OSError) as e: verbose_proxy_logger.warning( - f"Cannot create assets directory at {assets_dir}: {e}. " - f"Logo caching may not work. Using current directory for assets." + "Cannot create assets directory at %s: %s. Logo caching may not work. Using current directory for assets.", + assets_dir, + e, ) assets_dir = current_dir @@ -14764,7 +14812,7 @@ async def update_config( return {"message": "Config updated successfully"} except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.update_config(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.update_config(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -15592,7 +15640,7 @@ async def delete_callback( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.delete_callback(): Exception occurred - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.delete_callback(): Exception occurred - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) raise ProxyException( message="Error deleting callback: " + str(e), @@ -15716,7 +15764,7 @@ async def get_config( "available_callbacks": all_available_callbacks, } except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.get_config(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.get_config(): Exception occured - %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"Authentication Error({e})"), @@ -15825,7 +15873,7 @@ async def reload_model_cost_map( await invalidate_config_param("model_cost_map_reload_config") models_count = len(new_model_cost_map) if new_model_cost_map else 0 - verbose_proxy_logger.info(f"Model cost map reloaded successfully in current pod. Models count: {models_count}") + verbose_proxy_logger.info("Model cost map reloaded successfully in current pod. Models count: %s", models_count) return { "message": f"Price data reloaded successfully! {models_count} models updated.", @@ -15834,7 +15882,7 @@ async def reload_model_cost_map( "timestamp": current_time.isoformat(), } except Exception as e: - verbose_proxy_logger.exception(f"Failed to reload model cost map: {e}") + verbose_proxy_logger.exception("Failed to reload model cost map: %s", e) raise HTTPException(status_code=500, detail=f"Failed to reload model cost map: {e}") @@ -15882,7 +15930,7 @@ async def schedule_model_cost_map_reload( ) await invalidate_config_param("model_cost_map_reload_config") - verbose_proxy_logger.info(f"Model cost map reload scheduled for every {hours} hours") + verbose_proxy_logger.info("Model cost map reload scheduled for every %s hours", hours) return { "message": f"Model cost map reload scheduled for every {hours} hours", @@ -15891,7 +15939,7 @@ async def schedule_model_cost_map_reload( "timestamp": datetime.utcnow().isoformat(), } except Exception as e: - verbose_proxy_logger.exception(f"Failed to schedule model cost map reload: {e}") + verbose_proxy_logger.exception("Failed to schedule model cost map reload: %s", e) raise HTTPException( status_code=500, detail=f"Failed to schedule model cost map reload: {e}", @@ -15936,7 +15984,7 @@ async def cancel_model_cost_map_reload( "timestamp": datetime.utcnow().isoformat(), } except Exception as e: - verbose_proxy_logger.exception(f"Failed to cancel model cost map reload: {e}") + verbose_proxy_logger.exception("Failed to cancel model cost map reload: %s", e) raise HTTPException(status_code=500, detail=f"Failed to cancel model cost map reload: {e}") @@ -15964,7 +16012,7 @@ async def get_model_cost_map_reload_status( try: global prisma_client, last_model_cost_map_reload - verbose_proxy_logger.info(f"Checking model cost map reload status. Last reload: {last_model_cost_map_reload}") + verbose_proxy_logger.info("Checking model cost map reload status. Last reload: %s", last_model_cost_map_reload) if prisma_client is None: verbose_proxy_logger.info("No database connection, returning not scheduled") @@ -16014,7 +16062,7 @@ async def get_model_cost_map_reload_status( if hours_since_last_reload < interval_hours: next_run = (last_reload_time + timedelta(hours=interval_hours)).isoformat() except Exception as e: - verbose_proxy_logger.warning(f"Error parsing last reload time: {e}") + verbose_proxy_logger.warning("Error parsing last reload time: %s", e) return { "scheduled": True, @@ -16023,7 +16071,7 @@ async def get_model_cost_map_reload_status( "next_run": next_run, } except Exception as e: - verbose_proxy_logger.exception(f"Failed to get model cost map reload status: {e}") + verbose_proxy_logger.exception("Failed to get model cost map reload status: %s", e) raise HTTPException( status_code=500, detail=f"Failed to get model cost map reload status: {e}", @@ -16071,7 +16119,7 @@ async def get_model_cost_map_source( "model_count": model_count, } except Exception as e: - verbose_proxy_logger.exception(f"Failed to get model cost map source info: {e}") + verbose_proxy_logger.exception("Failed to get model cost map source info: %s", e) raise HTTPException( status_code=500, detail=f"Failed to get model cost map source info: {e}", @@ -16140,7 +16188,7 @@ async def reload_anthropic_beta_headers( provider_count = sum(1 for k in new_config.keys() if k not in ["provider_aliases", "description"]) verbose_proxy_logger.info( - f"Anthropic beta headers config reloaded successfully in current pod. Providers: {provider_count}" + "Anthropic beta headers config reloaded successfully in current pod. Providers: %s", provider_count ) return { @@ -16150,7 +16198,7 @@ async def reload_anthropic_beta_headers( "timestamp": current_time.isoformat(), } except Exception as e: - verbose_proxy_logger.exception(f"Failed to reload anthropic beta headers: {e}") + verbose_proxy_logger.exception("Failed to reload anthropic beta headers: %s", e) raise HTTPException(status_code=500, detail=f"Failed to reload anthropic beta headers: {e}") @@ -16198,7 +16246,7 @@ async def schedule_anthropic_beta_headers_reload( ) await invalidate_config_param("anthropic_beta_headers_reload_config") - verbose_proxy_logger.info(f"Anthropic beta headers reload scheduled for every {hours} hours") + verbose_proxy_logger.info("Anthropic beta headers reload scheduled for every %s hours", hours) return { "message": f"Anthropic beta headers reload scheduled for every {hours} hours", @@ -16207,7 +16255,7 @@ async def schedule_anthropic_beta_headers_reload( "timestamp": datetime.utcnow().isoformat(), } except Exception as e: - verbose_proxy_logger.exception(f"Failed to schedule anthropic beta headers reload: {e}") + verbose_proxy_logger.exception("Failed to schedule anthropic beta headers reload: %s", e) raise HTTPException( status_code=500, detail=f"Failed to schedule anthropic beta headers reload: {e}", @@ -16252,7 +16300,7 @@ async def cancel_anthropic_beta_headers_reload( "timestamp": datetime.utcnow().isoformat(), } except Exception as e: - verbose_proxy_logger.exception(f"Failed to cancel anthropic beta headers reload: {e}") + verbose_proxy_logger.exception("Failed to cancel anthropic beta headers reload: %s", e) raise HTTPException( status_code=500, detail=f"Failed to cancel anthropic beta headers reload: {e}", @@ -16284,7 +16332,7 @@ async def get_anthropic_beta_headers_reload_status( global prisma_client, last_anthropic_beta_headers_reload verbose_proxy_logger.info( - f"Checking anthropic beta headers reload status. Last reload: {last_anthropic_beta_headers_reload}" + "Checking anthropic beta headers reload status. Last reload: %s", last_anthropic_beta_headers_reload ) if prisma_client is None: @@ -16335,7 +16383,7 @@ async def get_anthropic_beta_headers_reload_status( if hours_since_last_reload < interval_hours: next_run = (last_reload_time + timedelta(hours=interval_hours)).isoformat() except Exception as e: - verbose_proxy_logger.warning(f"Error parsing last reload time: {e}") + verbose_proxy_logger.warning("Error parsing last reload time: %s", e) return { "scheduled": True, @@ -16344,7 +16392,7 @@ async def get_anthropic_beta_headers_reload_status( "next_run": next_run, } except Exception as e: - verbose_proxy_logger.exception(f"Failed to get anthropic beta headers reload status: {e}") + verbose_proxy_logger.exception("Failed to get anthropic beta headers reload status: %s", e) raise HTTPException( status_code=500, detail=f"Failed to get anthropic beta headers reload status: {e}", diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index a4ef30fb3c4..129ff8df3bd 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -191,7 +191,7 @@ async def _save_vector_store_to_db_from_rag_ingest( elif hasattr(response, "vector_store_id"): vector_store_id = response.vector_store_id else: - verbose_proxy_logger.warning(f"Unable to extract vector_store_id from response type: {type(response)}") + verbose_proxy_logger.warning("Unable to extract vector_store_id from response type: %s", type(response)) return if vector_store_id is None or not isinstance(vector_store_id, str): @@ -229,7 +229,7 @@ async def _save_vector_store_to_db_from_rag_ingest( # Only create if it doesn't exist if existing_vector_store is None: - verbose_proxy_logger.info(f"Saving newly created vector store {vector_store_id} to database") + verbose_proxy_logger.info("Saving newly created vector store %s to database", vector_store_id) # Initialize metadata with first file initial_metadata = {"ingested_files": [file_entry]} @@ -250,9 +250,9 @@ async def _save_vector_store_to_db_from_rag_ingest( user_id=user_api_key_dict.user_id, ) - verbose_proxy_logger.info(f"Vector store {vector_store_id} saved to database successfully") + verbose_proxy_logger.info("Vector store %s saved to database successfully", vector_store_id) else: - verbose_proxy_logger.info(f"Vector store {vector_store_id} already exists, appending file to metadata") + verbose_proxy_logger.info("Vector store %s already exists, appending file to metadata", vector_store_id) # Update existing vector store with new file existing_metadata = existing_vector_store.vector_store_metadata or {} @@ -274,11 +274,13 @@ async def _save_vector_store_to_db_from_rag_ingest( ) verbose_proxy_logger.info( - f"Added file {file_entry.get('filename') or file_entry.get('file_url', 'Unknown')} to vector store {vector_store_id} metadata" + "Added file %s to vector store %s metadata", + file_entry.get("filename") or file_entry.get("file_url", "Unknown"), + vector_store_id, ) except Exception as db_error: # Log the error but don't fail the request since ingestion succeeded - verbose_proxy_logger.exception(f"Failed to save vector store {vector_store_id} to database: {db_error}") + verbose_proxy_logger.exception("Failed to save vector store %s to database: %s", vector_store_id, db_error) async def parse_rag_ingest_request( @@ -495,7 +497,7 @@ async def rag_ingest( proxy_config=proxy_config, ) - verbose_proxy_logger.debug(f"RAG Ingest - options: {ingest_options}") + verbose_proxy_logger.debug("RAG Ingest - options: %s", ingest_options) # Call ingest response = await litellm.aingest( @@ -509,7 +511,10 @@ async def rag_ingest( # Save vector store to database if it was newly created and prisma_client is available verbose_proxy_logger.debug( - f"RAG Ingest - Checking database save conditions: prisma_client={prisma_client is not None}, response={response is not None}, response_type={type(response)}" + "RAG Ingest - Checking database save conditions: prisma_client=%s, response=%s, response_type=%s", + prisma_client is not None, + response is not None, + type(response), ) if prisma_client is not None and response is not None: @@ -523,7 +528,7 @@ async def rag_ingest( ) else: verbose_proxy_logger.warning( - f"Skipping database save: prisma_client={prisma_client is not None}, response={response is not None}" + "Skipping database save: prisma_client=%s, response=%s", prisma_client is not None, response is not None ) return response @@ -531,7 +536,7 @@ async def rag_ingest( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"RAG Ingest failed: {e}") + verbose_proxy_logger.exception("RAG Ingest failed: %s", e) raise HTTPException( status_code=500, detail={"error": str(e)}, @@ -663,7 +668,7 @@ async def rag_query( proxy_config=proxy_config, ) - verbose_proxy_logger.debug(f"RAG Query - model: {model}, retrieval_config: {retrieval_config}") + verbose_proxy_logger.debug("RAG Query - model: %s, retrieval_config: %s", model, retrieval_config) # Call query response = await litellm.aquery( @@ -706,7 +711,7 @@ async def rag_query( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"RAG Query failed: {e}") + verbose_proxy_logger.exception("RAG Query failed: %s", e) raise HTTPException( status_code=500, detail={"error": str(e)}, diff --git a/litellm/proxy/rerank_endpoints/endpoints.py b/litellm/proxy/rerank_endpoints/endpoints.py index f1c138fa1bf..e91b3b18f01 100644 --- a/litellm/proxy/rerank_endpoints/endpoints.py +++ b/litellm/proxy/rerank_endpoints/endpoints.py @@ -103,7 +103,7 @@ async def rerank( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.rerank(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.rerank(): Exception occured - %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "message", str(e)), diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 9fa634dc12e..f03db48b4a7 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -116,7 +116,7 @@ async def responses_api( ResponsePollingHandler, ) - verbose_proxy_logger.info(f"Starting background response with polling for model={data.get('model')}") + verbose_proxy_logger.info("Starting background response with polling for model=%s", data.get("model")) # Run pre-call checks (rate limits, guardrails, budget) BEFORE creating # polling ID. This ensures rate-limited requests get a synchronous 429 @@ -233,7 +233,8 @@ async def responses_api( if not model_id: verbose_proxy_logger.warning( - f"No model_id found in response hidden params for response {response.id}, skipping managed object storage" + "No model_id found in response hidden params for response %s, skipping managed object storage", + response.id, ) raise Exception("No model_id found in response hidden params") # Store in managed objects table @@ -247,10 +248,14 @@ async def responses_api( ) verbose_proxy_logger.info( - f"Stored background response {response.id} in managed objects table with unified_id={response.id}" + "Stored background response %s in managed objects table with unified_id=%s", + response.id, + response.id, ) except Exception as e: - verbose_proxy_logger.error(f"Failed to store background response in managed objects table: {e}") + verbose_proxy_logger.error( + "Failed to store background response in managed objects table: %s", e + ) return response except ModifyResponseException as e: diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index b744396e850..7d81390866a 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -51,7 +51,7 @@ async def background_streaming_task( """ try: - verbose_proxy_logger.info(f"Starting background streaming for {polling_id}") + verbose_proxy_logger.info("Starting background streaming for %s", polling_id) # Update status to in_progress (OpenAI format) await polling_handler.update_state( @@ -146,8 +146,8 @@ async def background_streaming_task( # Handle StreamingResponse if not hasattr(response, "body_iterator"): verbose_proxy_logger.warning( - f"background_streaming_task: response for {polling_id} has no " - "body_iterator; this may indicate a misconfiguration or provider error" + "background_streaming_task: response for %s has no body_iterator; this may indicate a misconfiguration or provider error", + polling_id, ) if hasattr(response, "body_iterator"): @@ -293,7 +293,7 @@ async def background_streaming_task( await flush_state_if_needed() except json.JSONDecodeError as e: - verbose_proxy_logger.warning(f"Failed to parse streaming chunk: {e}") + verbose_proxy_logger.warning("Failed to parse streaming chunk: %s", e) # Final flush to ensure all accumulated state is saved await flush_state_if_needed(force=True) @@ -324,11 +324,16 @@ async def background_streaming_task( ) verbose_proxy_logger.info( - f"Finished background streaming for {polling_id}, status={final_status}, error={terminal_error}, incomplete_details={incomplete_details_data}, output_items={len(output_items)}" + "Finished background streaming for %s, status=%s, error=%s, incomplete_details=%s, output_items=%s", + polling_id, + final_status, + terminal_error, + incomplete_details_data, + len(output_items), ) except Exception as e: - verbose_proxy_logger.error(f"Error in background streaming task for {polling_id}: {e}") + verbose_proxy_logger.error("Error in background streaming task for %s: %s", polling_id, e) import traceback verbose_proxy_logger.error(traceback.format_exc()) diff --git a/litellm/proxy/response_polling/polling_handler.py b/litellm/proxy/response_polling/polling_handler.py index 4f2ad70cc7d..c39e5949f19 100644 --- a/litellm/proxy/response_polling/polling_handler.py +++ b/litellm/proxy/response_polling/polling_handler.py @@ -77,7 +77,7 @@ class ResponsePollingHandler: value=response.model_dump_json(), # Pydantic v2 method ttl=self.ttl, ) - verbose_proxy_logger.debug(f"Created initial polling state for {polling_id} with TTL={self.ttl}s") + verbose_proxy_logger.debug("Created initial polling state for %s with TTL=%ss", polling_id, self.ttl) return response @@ -141,7 +141,7 @@ class ResponsePollingHandler: # Get current state cached_state = await self.redis_cache.async_get_cache(cache_key) if not cached_state: - verbose_proxy_logger.warning(f"No cached state found for polling_id: {polling_id}") + verbose_proxy_logger.warning("No cached state found for polling_id: %s", polling_id) return # Parse existing ResponsesAPIResponse from cache @@ -209,7 +209,7 @@ class ResponsePollingHandler: output_count = len(state.get("output", [])) verbose_proxy_logger.debug( - f"Updated polling state for {polling_id}: status={state['status']}, output_items={output_count}" + "Updated polling state for %s: status=%s, output_items=%s", polling_id, state["status"], output_count ) async def get_state(self, polling_id: str) -> dict[str, Any] | None: @@ -277,7 +277,7 @@ def should_use_polling_for_request( # Check if model is in native_background_mode list - these use native provider background mode if native_background_mode and model in native_background_mode: - verbose_proxy_logger.debug(f"Model {model} is in native_background_mode list, skipping polling via cache") + verbose_proxy_logger.debug("Model %s is in native_background_mode list, skipping polling via cache", model) return False # "all" enables polling for all providers @@ -311,9 +311,9 @@ def should_use_polling_for_request( # If ANY deployment's provider matches, enable polling if dep_provider and dep_provider in polling_via_cache_enabled: - verbose_proxy_logger.debug(f"Polling enabled for model={model}, provider={dep_provider}") + verbose_proxy_logger.debug("Polling enabled for model=%s, provider=%s", model, dep_provider) return True except Exception as e: - verbose_proxy_logger.debug(f"Could not resolve provider for model {model}: {e}") + verbose_proxy_logger.debug("Could not resolve provider for model %s: %s", model, e) return False diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 5e775d1cbf7..97832b0b6c4 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -262,7 +262,7 @@ async def add_shared_session_to_data(data: dict) -> None: if session is not None and not session.closed: data["shared_session"] = session - verbose_proxy_logger.info(f"SESSION REUSE: Attached shared aiohttp session to request (ID: {id(session)})") + verbose_proxy_logger.info("SESSION REUSE: Attached shared aiohttp session to request (ID: %s)", id(session)) elif session is not None and session.closed: # Session was created at startup but has since closed — recreate it # Use lock to prevent concurrent recreation (avoids session/connector leak) @@ -278,7 +278,7 @@ async def add_shared_session_to_data(data: dict) -> None: # or closed — either way we need to recreate if session is not None: verbose_proxy_logger.warning( - f"SESSION REUSE: Shared aiohttp session is closed (ID: {id(session)}), recreating..." + "SESSION REUSE: Shared aiohttp session is closed (ID: %s), recreating...", id(session) ) else: verbose_proxy_logger.warning( diff --git a/litellm/proxy/search_endpoints/endpoints.py b/litellm/proxy/search_endpoints/endpoints.py index 0032083b09c..09814672ccf 100644 --- a/litellm/proxy/search_endpoints/endpoints.py +++ b/litellm/proxy/search_endpoints/endpoints.py @@ -170,14 +170,15 @@ async def search( team_object=team_object, ) except Exception as e: - verbose_proxy_logger.error(f"Search tool authorization failed for {search_tool_name_value}: {e}") + verbose_proxy_logger.error("Search tool authorization failed for %s: %s", search_tool_name_value, e) raise if llm_router is not None and hasattr(llm_router, "search_tools"): verbose_proxy_logger.debug( - f"Search endpoint - Looking for search_tool_name: {search_tool_name_value}. " - f"Available search tools in router: {[tool.get('search_tool_name') for tool in llm_router.search_tools]}. " - f"Total search tools: {len(llm_router.search_tools)}" + "Search endpoint - Looking for search_tool_name: %s. Available search tools in router: %s. Total search tools: %s", + search_tool_name_value, + [tool.get("search_tool_name") for tool in llm_router.search_tools], + len(llm_router.search_tools), ) matching_tools = [ @@ -302,5 +303,5 @@ async def list_search_tools( except Exception as e: from litellm._logging import verbose_proxy_logger - verbose_proxy_logger.exception(f"Error listing search tools: {e}") + verbose_proxy_logger.exception("Error listing search tools: %s", e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/litellm/proxy/search_endpoints/search_tool_management.py b/litellm/proxy/search_endpoints/search_tool_management.py index 6b0bbacd131..1b3462b48a4 100644 --- a/litellm/proxy/search_endpoints/search_tool_management.py +++ b/litellm/proxy/search_endpoints/search_tool_management.py @@ -161,7 +161,7 @@ async def list_search_tools( if parsed_tools: config_search_tools = parsed_tools except Exception as e: - verbose_proxy_logger.debug(f"Could not get config-defined search tools: {e}") + verbose_proxy_logger.debug("Could not get config-defined search tools: %s", e) for config_search_tool in config_search_tools: tool_name = config_search_tool.get("search_tool_name") @@ -214,7 +214,7 @@ async def list_search_tools( return ListSearchToolsResponse(search_tools=visible_search_tools) except Exception as e: - verbose_proxy_logger.exception(f"Error getting search tools: {e}") + verbose_proxy_logger.exception("Error getting search tools: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -278,13 +278,13 @@ async def create_search_tool(request: CreateSearchToolRequest): ) verbose_proxy_logger.debug( - f"Successfully added search tool '{result.get('search_tool_name')}' to database. " - f"Router will be updated by the cron job." + "Successfully added search tool '%s' to database. Router will be updated by the cron job.", + result.get("search_tool_name"), ) return result except Exception as e: - verbose_proxy_logger.exception(f"Error adding search tool to db: {e}") + verbose_proxy_logger.exception("Error adding search tool to db: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -361,15 +361,15 @@ async def update_search_tool(search_tool_id: str, request: UpdateSearchToolReque ) verbose_proxy_logger.debug( - f"Successfully updated search tool '{result.get('search_tool_name')}' in database. " - f"Router will be updated by the cron job." + "Successfully updated search tool '%s' in database. Router will be updated by the cron job.", + result.get("search_tool_name"), ) return result except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.exception(f"Error updating search tool: {e}") + verbose_proxy_logger.exception("Error updating search tool: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -425,7 +425,7 @@ async def delete_search_tool(search_tool_id: str): except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.exception(f"Error deleting search tool: {e}") + verbose_proxy_logger.exception("Error deleting search tool: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -498,7 +498,7 @@ async def get_search_tool_info(search_tool_id: str): except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.exception(f"Error getting search tool info: {e}") + verbose_proxy_logger.exception("Error getting search tool info: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -561,7 +561,7 @@ async def test_search_tool_connection(request: TestSearchToolConnectionRequest): if not search_provider: raise HTTPException(status_code=400, detail="search_provider is required in litellm_params") - verbose_proxy_logger.debug(f"Testing connection to search provider: {search_provider}") + verbose_proxy_logger.debug("Testing connection to search provider: %s", search_provider) # Make a simple test search query with max_results=1 to minimize cost test_query = "test" @@ -574,7 +574,7 @@ async def test_search_tool_connection(request: TestSearchToolConnectionRequest): timeout=10.0, # 10 second timeout for test ) - verbose_proxy_logger.debug(f"Successfully tested connection to {search_provider} search provider") + verbose_proxy_logger.debug("Successfully tested connection to %s search provider", search_provider) return { "status": "success", @@ -587,7 +587,7 @@ async def test_search_tool_connection(request: TestSearchToolConnectionRequest): error_message = str(e) error_type = type(e).__name__ - verbose_proxy_logger.exception(f"Failed to connect to search provider: {error_message}") + verbose_proxy_logger.exception("Failed to connect to search provider: %s", error_message) # Return error details in a structured format return { @@ -652,10 +652,10 @@ async def get_available_search_providers(): } ) except Exception as e: - verbose_proxy_logger.debug(f"Could not get config for search provider {provider.value}: {e}") + verbose_proxy_logger.debug("Could not get config for search provider %s: %s", provider.value, e) continue return {"providers": available_providers} except Exception as e: - verbose_proxy_logger.exception(f"Error getting available search providers: {e}") + verbose_proxy_logger.exception("Error getting available search providers: %s", e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/litellm/proxy/search_endpoints/search_tool_registry.py b/litellm/proxy/search_endpoints/search_tool_registry.py index d7e5efa6d1e..ad8c8b6fe42 100644 --- a/litellm/proxy/search_endpoints/search_tool_registry.py +++ b/litellm/proxy/search_endpoints/search_tool_registry.py @@ -78,7 +78,7 @@ class SearchToolRegistry: return search_tool_dict except Exception as e: - verbose_proxy_logger.exception(f"Error adding search tool to DB: {e}") + verbose_proxy_logger.exception("Error adding search tool to DB: %s", e) raise Exception(f"Error adding search tool to DB: {e}") async def delete_search_tool_from_db(self, search_tool_id: str, prisma_client: PrismaClient): @@ -109,7 +109,7 @@ class SearchToolRegistry: "search_tool_name": existing_tool.search_tool_name, } except Exception as e: - verbose_proxy_logger.exception(f"Error deleting search tool from DB: {e}") + verbose_proxy_logger.exception("Error deleting search tool from DB: %s", e) raise Exception(f"Error deleting search tool from DB: {e}") async def update_search_tool_in_db(self, search_tool_id: str, search_tool: SearchTool, prisma_client: PrismaClient): @@ -143,7 +143,7 @@ class SearchToolRegistry: # Convert to dict with ISO formatted datetimes return self._convert_prisma_to_dict(updated_search_tool) except Exception as e: - verbose_proxy_logger.exception(f"Error updating search tool in DB: {e}") + verbose_proxy_logger.exception("Error updating search tool in DB: %s", e) raise Exception(f"Error updating search tool in DB: {e}") @staticmethod @@ -176,7 +176,7 @@ class SearchToolRegistry: return search_tools except Exception as e: - verbose_proxy_logger.exception(f"Error getting search tools from DB: {e}") + verbose_proxy_logger.exception("Error getting search tools from DB: %s", e) raise Exception(f"Error getting search tools from DB: {e}") async def get_search_tool_by_id_from_db( @@ -204,7 +204,7 @@ class SearchToolRegistry: search_tool_dict = self._convert_prisma_to_dict(search_tool) return SearchTool(**search_tool_dict) # type: ignore except Exception as e: - verbose_proxy_logger.exception(f"Error getting search tool from DB: {e}") + verbose_proxy_logger.exception("Error getting search tool from DB: %s", e) raise Exception(f"Error getting search tool from DB: {e}") async def get_search_tool_by_name_from_db( @@ -232,5 +232,5 @@ class SearchToolRegistry: search_tool_dict = self._convert_prisma_to_dict(search_tool) return SearchTool(**search_tool_dict) # type: ignore except Exception as e: - verbose_proxy_logger.exception(f"Error getting search tool from DB: {e}") + verbose_proxy_logger.exception("Error getting search tool from DB: %s", e) raise Exception(f"Error getting search tool from DB: {e}") diff --git a/litellm/proxy/spend_tracking/cloudzero_endpoints.py b/litellm/proxy/spend_tracking/cloudzero_endpoints.py index 7b573b2fad7..fdeb176aa99 100644 --- a/litellm/proxy/spend_tracking/cloudzero_endpoints.py +++ b/litellm/proxy/spend_tracking/cloudzero_endpoints.py @@ -161,7 +161,7 @@ async def get_cloudzero_settings( # Re-raise HTTPExceptions as-is raise e except Exception as e: - verbose_proxy_logger.error(f"Error retrieving CloudZero settings: {e}") + verbose_proxy_logger.error("Error retrieving CloudZero settings: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to retrieve CloudZero settings: {e}"}, @@ -238,7 +238,7 @@ async def update_cloudzero_settings( ) raise e except Exception as e: - verbose_proxy_logger.error(f"Error updating CloudZero settings: {e}") + verbose_proxy_logger.error("Error updating CloudZero settings: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to update CloudZero settings: {e}"}, @@ -275,7 +275,7 @@ async def is_cloudzero_setup_in_db() -> bool: return cloudzero_config is not None and cloudzero_config.param_value is not None except Exception as e: - verbose_proxy_logger.error(f"Error checking CloudZero status: {e}") + verbose_proxy_logger.error("Error checking CloudZero status: %s", e) return False @@ -317,7 +317,7 @@ async def is_cloudzero_setup() -> bool: return False except Exception as e: - verbose_proxy_logger.error(f"Error checking CloudZero setup: {e}") + verbose_proxy_logger.error("Error checking CloudZero setup: %s", e) return False @@ -364,7 +364,7 @@ async def init_cloudzero_settings( return CloudZeroInitResponse(message="CloudZero settings initialized successfully", status="success") except Exception as e: - verbose_proxy_logger.error(f"Error initializing CloudZero settings: {e}") + verbose_proxy_logger.error("Error initializing CloudZero settings: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to initialize CloudZero settings: {e}"}, @@ -422,7 +422,7 @@ async def cloudzero_dry_run_export( ) except Exception as e: - verbose_proxy_logger.error(f"Error performing CloudZero dry run export: {e}") + verbose_proxy_logger.error("Error performing CloudZero dry run export: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to perform CloudZero dry run export: {e}"}, @@ -487,7 +487,7 @@ async def cloudzero_export( ) except Exception as e: - verbose_proxy_logger.error(f"Error performing CloudZero export: {e}") + verbose_proxy_logger.error("Error performing CloudZero export: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to perform CloudZero export: {e}"}, @@ -550,7 +550,7 @@ async def delete_cloudzero_settings( except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.error(f"Error deleting CloudZero settings: {e}") + verbose_proxy_logger.error("Error deleting CloudZero settings: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to delete CloudZero settings: {e}"}, diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 0bcc2b9994b..aa26a45ae0e 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1648,7 +1648,7 @@ async def _get_spend_report_for_time_range( return response, spend_per_tag except Exception as e: - verbose_proxy_logger.error(f"Exception in _get_daily_spend_reports {e}") + verbose_proxy_logger.error("Exception in _get_daily_spend_reports %s", e) @router.post( @@ -2261,7 +2261,7 @@ async def ui_view_spend_logs( total_is_capped=total_is_capped, ) except Exception as e: - verbose_proxy_logger.exception(f"Error in ui_view_spend_logs: {e}") + verbose_proxy_logger.exception("Error in ui_view_spend_logs: %s", e) raise handle_exception_on_proxy(e) @@ -2789,7 +2789,7 @@ async def global_spend_refresh(): } except Exception as e: - verbose_proxy_logger.exception(f"Failed to refresh materialized view - {e}") + verbose_proxy_logger.exception("Failed to refresh materialized view - %s", e) return { "message": "Failed to refresh materialized view", "status": "failure", @@ -2830,7 +2830,7 @@ async def global_spend_for_internal_user( return response except Exception as e: - verbose_proxy_logger.error(f"/global/spend/logs Error: {e}") + verbose_proxy_logger.error("/global/spend/logs Error: %s", e) raise e @@ -2906,7 +2906,7 @@ async def global_spend_logs( except Exception as e: error_trace = traceback.format_exc() error_str = str(e) + "\n" + error_trace - verbose_proxy_logger.error(f"/global/spend/logs Error: {error_str}") + verbose_proxy_logger.error("/global/spend/logs Error: %s", error_str) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"/global/spend/logs Error({error_str})"), @@ -3387,7 +3387,7 @@ async def provider_budgets() -> ProviderBudgetResponse: provider_budget_response_dict[_provider] = provider_budget_response_object return ProviderBudgetResponse(providers=provider_budget_response_dict) except Exception as e: - verbose_proxy_logger.exception(f"/provider/budgets: Exception occured - {e}") + verbose_proxy_logger.exception("/provider/budgets: Exception occured - %s", e) raise handle_exception_on_proxy(e) diff --git a/litellm/proxy/spend_tracking/vantage_endpoints.py b/litellm/proxy/spend_tracking/vantage_endpoints.py index ac45594de22..d965a150ac1 100644 --- a/litellm/proxy/spend_tracking/vantage_endpoints.py +++ b/litellm/proxy/spend_tracking/vantage_endpoints.py @@ -166,7 +166,7 @@ async def get_vantage_settings( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error retrieving Vantage settings: {e}") + verbose_proxy_logger.error("Error retrieving Vantage settings: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to retrieve Vantage settings: {e}"}, @@ -235,7 +235,7 @@ async def update_vantage_settings( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error updating Vantage settings: {e}") + verbose_proxy_logger.error("Error updating Vantage settings: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to update Vantage settings: {e}"}, @@ -257,7 +257,7 @@ async def is_vantage_setup_in_db() -> bool: return vantage_config is not None and vantage_config.param_value is not None except Exception as e: - verbose_proxy_logger.error(f"Error checking Vantage status: {e}") + verbose_proxy_logger.error("Error checking Vantage status: %s", e) return False @@ -280,7 +280,7 @@ async def is_vantage_setup() -> bool: return True return False except Exception as e: - verbose_proxy_logger.error(f"Error checking Vantage setup: {e}") + verbose_proxy_logger.error("Error checking Vantage setup: %s", e) return False @@ -324,7 +324,7 @@ async def init_vantage_settings( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error initializing Vantage settings: {e}") + verbose_proxy_logger.error("Error initializing Vantage settings: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to initialize Vantage settings: {e}"}, @@ -415,7 +415,7 @@ async def vantage_dry_run_export( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error performing Vantage dry run export: {e}") + verbose_proxy_logger.error("Error performing Vantage dry run export: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to perform Vantage dry run export: {e}"}, @@ -488,7 +488,7 @@ async def vantage_export( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error performing Vantage export: {e}") + verbose_proxy_logger.error("Error performing Vantage export: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to perform Vantage export: {e}"}, @@ -548,7 +548,7 @@ async def delete_vantage_settings( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error deleting Vantage settings: {e}") + verbose_proxy_logger.error("Error deleting Vantage settings: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to delete Vantage settings: {e}"}, diff --git a/litellm/proxy/types_utils/utils.py b/litellm/proxy/types_utils/utils.py index e61fcdd859b..ae2bdbc5895 100644 --- a/litellm/proxy/types_utils/utils.py +++ b/litellm/proxy/types_utils/utils.py @@ -127,8 +127,11 @@ def _load_instance_from_remote_storage(remote_url: str, config_file_path: str | object_key = f"{module_path}.py" verbose_proxy_logger.debug( - f"Loading custom logger from {storage_type}: bucket={bucket_name}, " - f"object_key={object_key}, instance={instance_name}" + "Loading custom logger from %s: bucket=%s, object_key=%s, instance=%s", + storage_type, + bucket_name, + object_key, + instance_name, ) import tempfile @@ -170,9 +173,9 @@ def _load_instance_from_remote_storage(remote_url: str, config_file_path: str | try: os.remove(local_file_path) except Exception as cleanup_error: - verbose_proxy_logger.warning(f"Could not clean up temporary file {local_file_path}: {cleanup_error}") + verbose_proxy_logger.warning("Could not clean up temporary file %s: %s", local_file_path, cleanup_error) - verbose_proxy_logger.info(f"Successfully loaded custom logger from {remote_url}") + verbose_proxy_logger.info("Successfully loaded custom logger from %s", remote_url) return instance except Exception as e: @@ -190,7 +193,7 @@ async def _download_gcs_file_wrapper(bucket_name: str, object_key: str, local_fi except Exception as e: from litellm._logging import verbose_proxy_logger - verbose_proxy_logger.error(f"Error downloading from GCS: {e}") + verbose_proxy_logger.error("Error downloading from GCS: %s", e) return False diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 8ed848ac1bf..22759a37ae7 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -689,7 +689,10 @@ async def update_default_team_member_budget(teams: list[NewUserRequestTeam], use ) except Exception as e: verbose_proxy_logger.info( - f"Error updating team {team_id} with team member budget {max_budget_in_team} with error: {e}, skipping.." + "Error updating team %s with team member budget %s with error: %s, skipping..", + team_id, + max_budget_in_team, + e, ) continue @@ -1209,7 +1212,7 @@ async def update_mcp_semantic_filter_settings( if prisma_client is not None: await proxy_config._init_semantic_filter_settings_in_db(prisma_client=prisma_client) except Exception as e: - verbose_proxy_logger.warning(f"Failed to reinitialize MCP semantic filter settings immediately: {e}") + verbose_proxy_logger.warning("Failed to reinitialize MCP semantic filter settings immediately: %s", e) return result diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 5f18189b6b3..51c9b300b6e 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -196,7 +196,7 @@ def print_verbose(print_statement): """ import traceback - verbose_proxy_logger.debug(f"{print_statement}\n{traceback.format_exc()}") + verbose_proxy_logger.debug("%s\n%s", print_statement, traceback.format_exc()) if litellm.set_verbose: print(f"LiteLLM Proxy: {_redact_string(str(print_statement))}") # noqa: T201 @@ -722,11 +722,11 @@ class ProxyLogging: else: # Could not parse modified arguments, allow original call but warn verbose_proxy_logger.warning( - f"Could not parse modified arguments from guardrail response: {new_content}" + "Could not parse modified arguments from guardrail response: %s", new_content ) return None except Exception as e: - verbose_proxy_logger.error(f"Error parsing modified arguments: {e}") + verbose_proxy_logger.error("Error parsing modified arguments: %s", e) # Fallback: allow original call return None @@ -742,7 +742,7 @@ class ProxyLogging: """ import json - verbose_proxy_logger.debug(f"Extracting modified args from content: {masked_content}") + verbose_proxy_logger.debug("Extracting modified args from content: %s", masked_content) try: # The format should be: "Tool: \nArguments: " @@ -753,16 +753,16 @@ class ProxyLogging: # Get the arguments part - everything after "Arguments: " args_text = line[len("Arguments:") :].strip() - verbose_proxy_logger.debug(f"Found arguments text: {args_text}") + verbose_proxy_logger.debug("Found arguments text: %s", args_text) # Try to parse as JSON first try: modified_args = json.loads(args_text) - verbose_proxy_logger.debug(f"Successfully parsed JSON args: {modified_args}") + verbose_proxy_logger.debug("Successfully parsed JSON args: %s", modified_args) return modified_args except json.JSONDecodeError as e: # If JSON parsing fails, try to extract key-value pairs manually - verbose_proxy_logger.debug(f"Failed to parse JSON arguments: {args_text}, error: {e}") + verbose_proxy_logger.debug("Failed to parse JSON arguments: %s, error: %s", args_text, e) return self._parse_arguments_manually(args_text, request_obj.arguments) # If we can't find the Arguments: line, return None @@ -770,7 +770,7 @@ class ProxyLogging: return None except Exception as e: - verbose_proxy_logger.error(f"Error extracting modified arguments: {e}") + verbose_proxy_logger.error("Error extracting modified arguments: %s", e) return None def _parse_arguments_manually(self, args_text: str, original_args: dict) -> dict | None: @@ -799,7 +799,7 @@ class ProxyLogging: return modified_args except Exception as e: - verbose_proxy_logger.error(f"Error in manual argument parsing: {e}") + verbose_proxy_logger.error("Error in manual argument parsing: %s", e) return None def _convert_llm_result_to_mcp_during_response(self, llm_result, request_obj) -> Any | None: @@ -2174,10 +2174,10 @@ class ProxyLogging: except Exception as e: # Log non-HTTPException errors from callbacks but don't break the flow verbose_proxy_logger.exception( - f"[Non-Blocking] Error in async_post_call_failure_hook callback: {e}" + "[Non-Blocking] Error in async_post_call_failure_hook callback: %s", e ) except Exception as e: - verbose_proxy_logger.exception(f"[Non-Blocking] Error setting up post_call_failure_hook callback: {e}") + verbose_proxy_logger.exception("[Non-Blocking] Error setting up post_call_failure_hook callback: %s", e) return transformed_exception @@ -3019,7 +3019,7 @@ class PrismaClient: try: from prisma import Prisma # type: ignore except Exception as e: - verbose_proxy_logger.error(f"Failed to import Prisma client: {e}") + verbose_proxy_logger.error("Failed to import Prisma client: %s", e) verbose_proxy_logger.error("This usually means 'prisma generate' hasn't been run yet.") verbose_proxy_logger.error("Please run 'prisma generate' to generate the Prisma client.") raise Exception("Unable to find Prisma binaries. Please run 'prisma generate' first.") @@ -3283,7 +3283,8 @@ class PrismaClient: missing_views = expected_views_set - ret_view_names_set verbose_proxy_logger.warning( - f"\n\n\033[93mNot all views exist in db, needed for UI 'Usage' tab. Missing={missing_views}.\nRun 'create_views.py' from https://github.com/BerriAI/litellm/tree/main/db_scripts to create missing views.\033[0m\n" + "\n\n\x1b[93mNot all views exist in db, needed for UI 'Usage' tab. Missing=%s.\nRun 'create_views.py' from https://github.com/BerriAI/litellm/tree/main/db_scripts to create missing views.\x1b[0m\n", + missing_views, ) except Exception: @@ -3444,7 +3445,7 @@ class PrismaClient: if token is not None: if isinstance(token, str): hashed_token = _hash_token_if_needed(token=token) - verbose_proxy_logger.debug(f"PrismaClient: find_unique for token: {hashed_token}") + verbose_proxy_logger.debug("PrismaClient: find_unique for token: %s", hashed_token) if query_type == "find_unique" and hashed_token is not None: if token is None: raise HTTPException( @@ -3687,7 +3688,7 @@ class PrismaClient: if token is not None: if isinstance(token, str): hashed_token = _hash_token_if_needed(token=token) - verbose_proxy_logger.debug(f"PrismaClient: find_unique for token: {hashed_token}") + verbose_proxy_logger.debug("PrismaClient: find_unique for token: %s", hashed_token) if query_type == "find_unique": if token is None: raise HTTPException( @@ -3994,7 +3995,7 @@ class PrismaClient: """ Update existing data """ - verbose_proxy_logger.debug(f"PrismaClient: update_data, table_name: {table_name}") + verbose_proxy_logger.debug("PrismaClient: update_data, table_name: %s", table_name) start_time = time.time() try: db_data = self.jsonify_object(data=data) @@ -5062,7 +5063,7 @@ class PrismaClient: try: return await _fetch_row_count() except Exception as e: - verbose_proxy_logger.error(f"Error getting LiteLLM_SpendLogs row count: {e}") + verbose_proxy_logger.error("Error getting LiteLLM_SpendLogs row count: %s", e) return 0 @backoff.on_exception( @@ -5095,7 +5096,7 @@ class PrismaClient: value = float(response_time_ms) return value if value == value and value not in (float("inf"), float("-inf")) else None except (ValueError, TypeError): - verbose_proxy_logger.warning(f"Invalid response_time_ms value: {response_time_ms}") + verbose_proxy_logger.warning("Invalid response_time_ms value: %s", response_time_ms) return None def _clean_details(self, details: dict | None) -> dict | None: @@ -5105,7 +5106,7 @@ class PrismaClient: try: return safe_json_loads(safe_dumps(details)) except Exception as e: - verbose_proxy_logger.warning(f"Failed to clean details JSON: {e}") + verbose_proxy_logger.warning("Failed to clean details JSON: %s", e) return None async def save_health_check_result( @@ -5142,11 +5143,11 @@ class PrismaClient: # Add only non-None optional fields health_check_data.update({k: v for k, v in optional_fields.items() if v is not None}) - verbose_proxy_logger.debug(f"Saving health check data: {health_check_data}") + verbose_proxy_logger.debug("Saving health check data: %s", health_check_data) return await HealthCheckRepository(self).table.create(data=health_check_data) except Exception as e: - verbose_proxy_logger.error(f"Error saving health check result for model {model_name}: {e}") + verbose_proxy_logger.error("Error saving health check result for model %s: %s", model_name, e) return None async def get_health_check_history( @@ -5174,7 +5175,7 @@ class PrismaClient: ) return results except Exception as e: - verbose_proxy_logger.error(f"Error getting health check history: {e}") + verbose_proxy_logger.error("Error getting health check history: %s", e) return [] async def get_all_latest_health_checks(self): @@ -5194,7 +5195,7 @@ class PrismaClient: ], ) except Exception as e: - verbose_proxy_logger.error(f"Error getting all latest health checks: {e}") + verbose_proxy_logger.error("Error getting all latest health checks: %s", e) return [] @@ -5451,7 +5452,7 @@ class ProxyUpdateSpend: if len(logs_to_process) > 0 and base_url is not None and db_writer_client is not None: if not base_url.endswith("/"): base_url += "/" - verbose_proxy_logger.debug(f"base_url: {base_url}") + verbose_proxy_logger.debug("base_url: %s", base_url) json_data = json.dumps(logs_to_process) response = await db_writer_client.post( url=base_url + "spend/update", @@ -5475,7 +5476,7 @@ class ProxyUpdateSpend: statement_rows, isolation_budget, ) - verbose_proxy_logger.debug(f"Flushed {len(batch)} logs to the DB.") + verbose_proxy_logger.debug("Flushed %s logs to the DB.", len(batch)) # Explicitly clear batch memory del batch, batch_with_dates @@ -5483,7 +5484,7 @@ class ProxyUpdateSpend: async with prisma_client._spend_log_transactions_lock: remaining_count = len(prisma_client.spend_log_transactions) verbose_proxy_logger.debug( - f"{len(logs_to_process)} logs processed. Remaining in queue: {remaining_count}" + "%s logs processed. Remaining in queue: %s", len(logs_to_process), remaining_count ) break except DB_CONNECTION_ERROR_TYPES as e: @@ -5551,7 +5552,7 @@ async def update_spend( # Check queue size with lock protection async with prisma_client._spend_log_transactions_lock: queue_size = len(prisma_client.spend_log_transactions) - verbose_proxy_logger.debug(f"Spend Logs transactions: {queue_size}") + verbose_proxy_logger.debug("Spend Logs transactions: %s", queue_size) async with prisma_client._tool_usage_transactions_lock: tool_usage_queue_size = len(prisma_client.tool_usage_transactions) @@ -5611,7 +5612,7 @@ async def update_daily_tag_spend( # the active exception's traceback whenever the suppression env var # is unset, which would be a regression for operators who never saw # one here before. - verbose_proxy_logger.error(f"Error updating daily tag spend: {e}") + verbose_proxy_logger.error("Error updating daily tag spend: %s", e) async def update_spend_logs_job( @@ -5713,7 +5714,7 @@ async def _monitor_spend_logs_queue( current_interval = base_interval verbose_proxy_logger.info( - f"Starting spend logs queue monitor (threshold: {threshold}, poll_interval: {base_interval}s)" + "Starting spend logs queue monitor (threshold: %s, poll_interval: %ss)", threshold, base_interval ) while True: @@ -5729,13 +5730,17 @@ async def _monitor_spend_logs_queue( if queue_size > 0: if queue_size >= threshold: verbose_proxy_logger.debug( - f"Spend logs queue size ({queue_size}) reached threshold ({threshold}), triggering processing" + "Spend logs queue size (%s) reached threshold (%s), triggering processing", + queue_size, + threshold, ) # Reset to base interval when threshold is reached current_interval = base_interval else: verbose_proxy_logger.debug( - f"Spend logs queue size ({queue_size}) below threshold ({threshold}), processing with backoff" + "Spend logs queue size (%s) below threshold (%s), processing with backoff", + queue_size, + threshold, ) # Exponential backoff when below threshold but still processing current_interval = min(current_interval * backoff_multiplier, max_backoff) @@ -6121,7 +6126,7 @@ def handle_exception_on_proxy(e: Exception) -> ProxyException: """ from fastapi import status - verbose_proxy_logger.exception(f"Exception: {e}") + verbose_proxy_logger.exception("Exception: %s", e) if isinstance(e, HTTPException): return ProxyException( diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index 6176ae03d3d..d6d3392a7b4 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -207,11 +207,11 @@ def _resolve_embedding_config_from_router(embedding_model: str, llm_router) -> d # Only return config if we have at least api_key or api_base if embedding_config: verbose_proxy_logger.debug( - f"Resolved embedding config from router model {model_name}: {list(embedding_config.keys())}" + "Resolved embedding config from router model %s: %s", model_name, list(embedding_config.keys()) ) return embedding_config except Exception as e: - verbose_proxy_logger.debug(f"Error resolving embedding config from router for model {model_name}: {e}") + verbose_proxy_logger.debug("Error resolving embedding config from router for model %s: %s", model_name, e) continue return None @@ -295,11 +295,13 @@ async def _resolve_embedding_config_from_db(embedding_model: str, prisma_client) # Only return config if we have at least api_key or api_base if embedding_config: verbose_proxy_logger.debug( - f"Resolved embedding config from database model {model_name}: {list(embedding_config.keys())}" + "Resolved embedding config from database model %s: %s", + model_name, + list(embedding_config.keys()), ) return embedding_config except Exception as e: - verbose_proxy_logger.debug(f"Error resolving embedding config for model {model_name}: {e}") + verbose_proxy_logger.debug("Error resolving embedding config for model %s: %s", model_name, e) continue return None @@ -344,7 +346,7 @@ async def _resolve_embedding_config(embedding_model: str, prisma_client, llm_rou if llm_router is not None: router_config = _resolve_embedding_config_from_router(embedding_model=embedding_model, llm_router=llm_router) if router_config: - verbose_proxy_logger.debug(f"Resolved embedding config from router for model {embedding_model}") + verbose_proxy_logger.debug("Resolved embedding config from router for model %s", embedding_model) cache.set_cache(embedding_model, router_config) return router_config @@ -354,12 +356,12 @@ async def _resolve_embedding_config(embedding_model: str, prisma_client, llm_rou embedding_model=embedding_model, prisma_client=prisma_client ) if db_config: - verbose_proxy_logger.debug(f"Resolved embedding config from database for model {embedding_model}") + verbose_proxy_logger.debug("Resolved embedding config from database for model %s", embedding_model) cache.set_cache(embedding_model, db_config) return db_config verbose_proxy_logger.debug( - f"Could not resolve embedding config for model {embedding_model} from router or database" + "Could not resolve embedding config for model %s from router or database", embedding_model ) return None @@ -469,7 +471,7 @@ async def create_vector_store_in_db( if litellm.vector_store_registry is not None: litellm.vector_store_registry.add_vector_store_to_registry(vector_store=new_vector_store) - verbose_proxy_logger.info(f"Vector store {vector_store_id} created in database successfully") + verbose_proxy_logger.info("Vector store %s created in database successfully", vector_store_id) return new_vector_store @@ -542,7 +544,7 @@ async def new_vector_store( "vector_store": response_vs, } except Exception as e: - verbose_proxy_logger.exception(f"Error creating vector store: {e}") + verbose_proxy_logger.exception("Error creating vector store: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -604,7 +606,8 @@ async def list_vector_stores( # If vector store is in memory but NOT in database, it was deleted if vector_store_id not in db_vector_store_ids: verbose_proxy_logger.info( - f"Vector store {vector_store_id} exists in memory but not in database - marking for deletion from cache" + "Vector store %s exists in memory but not in database - marking for deletion from cache", + vector_store_id, ) vector_stores_to_delete_from_memory.append(vector_store_id) # If not in our map yet, add it (only in-memory, not in DB) @@ -615,7 +618,7 @@ async def list_vector_stores( # 1. Remove deleted vector stores from memory for vs_id in vector_stores_to_delete_from_memory: litellm.vector_store_registry.delete_vector_store_from_registry(vector_store_id=vs_id) - verbose_proxy_logger.debug(f"Removed deleted vector store {vs_id} from in-memory registry") + verbose_proxy_logger.debug("Removed deleted vector store %s from in-memory registry", vs_id) # 2. Update in-memory registry with database versions (for updates) for vector_store in vector_stores_from_db: @@ -647,7 +650,7 @@ async def list_vector_stores( return response except Exception as e: - verbose_proxy_logger.exception(f"Error listing vector stores: {e}") + verbose_proxy_logger.exception("Error listing vector stores: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -727,7 +730,7 @@ async def delete_vector_store( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error deleting vector store: {e}") + verbose_proxy_logger.exception("Error deleting vector store: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -799,7 +802,7 @@ async def get_vector_store_info( # the catch-all below would otherwise rewrite them as 500. raise except Exception as e: - verbose_proxy_logger.exception(f"Error getting vector store info: {e}") + verbose_proxy_logger.exception("Error getting vector store info: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -868,7 +871,7 @@ async def update_vector_store( updated_data=updated_vs, ) verbose_proxy_logger.debug( - f"Updated vector store {vector_store_id} in both database and in-memory registry" + "Updated vector store %s in both database and in-memory registry", vector_store_id ) # The DB row is returned in full, so the response would otherwise @@ -888,5 +891,5 @@ async def update_vector_store( # as 500 with the original status code embedded in the detail. raise except Exception as e: - verbose_proxy_logger.exception(f"Error updating vector store: {e}") + verbose_proxy_logger.exception("Error updating vector store: %s", e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/litellm/proxy/vector_store_files_endpoints/endpoints.py b/litellm/proxy/vector_store_files_endpoints/endpoints.py index 06bcc524ea7..feaefece266 100644 --- a/litellm/proxy/vector_store_files_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_files_endpoints/endpoints.py @@ -71,7 +71,7 @@ def _update_request_data_with_managed_file_id( if decoded_id: # This is a unified managed file ID - verbose_logger.debug(f"Processing unified managed file ID: {file_id}") + verbose_logger.debug("Processing unified managed file ID: %s", file_id) # Parse the unified ID to extract components parsed_id = parse_unified_id(file_id) @@ -90,7 +90,9 @@ def _update_request_data_with_managed_file_id( pass verbose_logger.debug( - f"Decoded unified file ID - target_model_names: {target_model_names}, llm_output_file_id: {llm_output_file_id}" + "Decoded unified file ID - target_model_names: %s, llm_output_file_id: %s", + target_model_names, + llm_output_file_id, ) # Set the model for routing @@ -108,14 +110,17 @@ def _update_request_data_with_managed_file_id( file_id=llm_output_file_id, # Use the actual provider file ID ) verbose_logger.info( - f"Routing vector store file operation to model: {routing_model}, file_id: {file_id} -> {llm_output_file_id}" + "Routing vector store file operation to model: %s, file_id: %s -> %s", + routing_model, + file_id, + llm_output_file_id, ) return data, file_id # Return original managed file ID # If we extracted the provider file ID but no routing, still use it if llm_output_file_id: data["file_id"] = llm_output_file_id - verbose_logger.debug(f"Replaced unified file ID with provider file ID: {llm_output_file_id}") + verbose_logger.debug("Replaced unified file ID with provider file ID: %s", llm_output_file_id) return data, file_id # Return original managed file ID return data, file_id if decoded_id else None @@ -361,7 +366,7 @@ def _update_request_data_with_litellm_managed_vector_store_registry( if decoded_id: # This is a managed vector store - decode and extract routing information - verbose_logger.debug(f"Processing managed vector store ID: {vector_store_id}") + verbose_logger.debug("Processing managed vector store ID: %s", vector_store_id) parsed_id = parse_unified_id(vector_store_id) @@ -371,7 +376,10 @@ def _update_request_data_with_litellm_managed_vector_store_registry( target_model_names = parsed_id.get("target_model_names", []) verbose_logger.debug( - f"Decoded vector store - model_id: {model_id}, provider_resource_id: {provider_resource_id}, target_model_names: {target_model_names}" + "Decoded vector store - model_id: %s, provider_resource_id: %s, target_model_names: %s", + model_id, + provider_resource_id, + target_model_names, ) # Set the model for routing - this tells the router which deployment to use @@ -384,13 +392,13 @@ def _update_request_data_with_litellm_managed_vector_store_registry( if routing_model: data["model"] = routing_model - verbose_logger.info(f"Routing vector store files operation to model: {routing_model}") + verbose_logger.info("Routing vector store files operation to model: %s", routing_model) # Replace unified vector store ID with provider resource ID if provider_resource_id: data["vector_store_id"] = provider_resource_id verbose_logger.debug( - f"Replaced unified vector store ID with provider resource ID: {provider_resource_id}" + "Replaced unified vector store ID with provider resource ID: %s", provider_resource_id ) return data diff --git a/litellm/rag/ingestion/base_ingestion.py b/litellm/rag/ingestion/base_ingestion.py index 76e6a4c574c..201baa2f471 100644 --- a/litellm/rag/ingestion/base_ingestion.py +++ b/litellm/rag/ingestion/base_ingestion.py @@ -365,7 +365,7 @@ class BaseRAGIngestion(ABC): ) except Exception as e: - verbose_logger.exception(f"RAG Pipeline failed: {e}") + verbose_logger.exception("RAG Pipeline failed: %s", e) return RAGIngestResponse( id=self.ingest_id, status="failed", diff --git a/litellm/rag/ingestion/bedrock_ingestion.py b/litellm/rag/ingestion/bedrock_ingestion.py index 10dc3af4319..45a3b64810f 100644 --- a/litellm/rag/ingestion/bedrock_ingestion.py +++ b/litellm/rag/ingestion/bedrock_ingestion.py @@ -138,7 +138,7 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): def _auto_detect_config(self): """Auto-detect data source ID and S3 bucket from existing Knowledge Base.""" - verbose_logger.debug(f"Auto-detecting data source and S3 bucket for KB={self.knowledge_base_id}") + verbose_logger.debug("Auto-detecting data source and S3 bucket for KB=%s", self.knowledge_base_id) bedrock_agent = self._get_boto3_client("bedrock-agent") @@ -157,7 +157,7 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): self.data_source_id = self._data_source_id else: self.data_source_id = data_sources[0]["dataSourceId"] - verbose_logger.info(f"Auto-detected data source: {self.data_source_id}") + verbose_logger.info("Auto-detected data source: %s", self.data_source_id) # Get data source details for S3 bucket ds_details = bedrock_agent.get_data_source( @@ -171,7 +171,7 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): if bucket_arn: # Extract bucket name from ARN: arn:aws:s3:::bucket-name self.s3_bucket = self._s3_bucket or bucket_arn.split(":")[-1] - verbose_logger.info(f"Auto-detected S3 bucket: {self.s3_bucket}") + verbose_logger.info("Auto-detected S3 bucket: %s", self.s3_bucket) # Use inclusion prefix if available prefixes = s3_config.get("inclusionPrefixes", []) @@ -218,8 +218,10 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): self.data_source_id = self._create_data_source(kb_name) verbose_logger.info( - f"Created KB infrastructure: kb_id={self.knowledge_base_id}, " - f"ds_id={self.data_source_id}, bucket={self.s3_bucket}" + "Created KB infrastructure: kb_id=%s, ds_id=%s, bucket=%s", + self.knowledge_base_id, + self.data_source_id, + self.s3_bucket, ) def _create_s3_bucket(self, unique_id: str) -> str: @@ -227,7 +229,7 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): s3 = self._get_boto3_client("s3") bucket_name = f"litellm-kb-{unique_id}" - verbose_logger.debug(f"Creating S3 bucket: {bucket_name}") + verbose_logger.debug("Creating S3 bucket: %s", bucket_name) create_params: dict[str, Any] = {"Bucket": bucket_name} if self.aws_region_name != "us-east-1": @@ -236,7 +238,7 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): s3.create_bucket(**create_params) self._created_resources["s3_bucket"] = bucket_name - verbose_logger.info(f"Created S3 bucket: {bucket_name}") + verbose_logger.info("Created S3 bucket: %s", bucket_name) return bucket_name async def _create_opensearch_collection(self, unique_id: str, account_id: str, caller_arn: str) -> tuple[str, str]: @@ -244,7 +246,7 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): oss = self._get_boto3_client("opensearchserverless") collection_name = f"litellm-kb-{unique_id}" - verbose_logger.debug(f"Creating OpenSearch Serverless collection: {collection_name}") + verbose_logger.debug("Creating OpenSearch Serverless collection: %s", collection_name) # Create encryption policy oss.create_security_policy( @@ -290,7 +292,7 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): # This ensures the credentials being used have access to the collection # Normalize the caller ARN (convert assumed-role ARN to IAM role ARN if needed) normalized_caller_arn = _normalize_principal_arn(caller_arn, account_id) - verbose_logger.debug(f"Caller ARN: {caller_arn}, Normalized: {normalized_caller_arn}") + verbose_logger.debug("Caller ARN: %s, Normalized: %s", caller_arn, normalized_caller_arn) principals = [f"arn:aws:iam::{account_id}:root", normalized_caller_arn] # Deduplicate in case caller is root @@ -340,7 +342,7 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): raise TimeoutError("OpenSearch collection did not become active in time") collection_arn = status_response["collectionDetails"][0]["arn"] - verbose_logger.info(f"Created OpenSearch collection: {collection_name}") + verbose_logger.info("Created OpenSearch collection: %s", collection_name) # Wait for data access policy to propagate before returning # AWS recommends waiting 60+ seconds for policy propagation @@ -412,15 +414,17 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): for attempt in range(max_retries): try: client.indices.create(index=index_name, body=index_body) - verbose_logger.info(f"Created OpenSearch index: {index_name}") + verbose_logger.info("Created OpenSearch index: %s", index_name) return except Exception as e: last_error = e error_str = str(e) if "authorization_exception" in error_str.lower() or "security_exception" in error_str.lower(): verbose_logger.warning( - f"OpenSearch index creation attempt {attempt + 1}/{max_retries} failed due to authorization. " - f"Waiting {retry_delay}s for policy propagation..." + "OpenSearch index creation attempt %s/%s failed due to authorization. Waiting %ss for policy propagation...", + attempt + 1, + max_retries, + retry_delay, ) await asyncio.sleep(retry_delay) else: @@ -438,7 +442,7 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): iam = self._get_boto3_client("iam") role_name = f"litellm-bedrock-kb-{unique_id}" - verbose_logger.debug(f"Creating IAM role: {role_name}") + verbose_logger.debug("Creating IAM role: %s", role_name) trust_policy = { "Version": "2012-10-17", @@ -498,14 +502,14 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): # Wait for role to propagate (use asyncio.sleep to avoid blocking) await asyncio.sleep(10) - verbose_logger.info(f"Created IAM role: {role_arn}") + verbose_logger.info("Created IAM role: %s", role_arn) return role_arn async def _create_knowledge_base(self, kb_name: str, role_arn: str, collection_arn: str) -> str: """Create Bedrock Knowledge Base.""" bedrock_agent = self._get_boto3_client("bedrock-agent") - verbose_logger.debug(f"Creating Knowledge Base: {kb_name}") + verbose_logger.debug("Creating Knowledge Base: %s", kb_name) response = bedrock_agent.create_knowledge_base( name=kb_name, @@ -543,14 +547,14 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): else: raise TimeoutError("Knowledge Base did not become active in time") - verbose_logger.info(f"Created Knowledge Base: {kb_id}") + verbose_logger.info("Created Knowledge Base: %s", kb_id) return kb_id def _create_data_source(self, kb_name: str) -> str: """Create Data Source for the Knowledge Base.""" bedrock_agent = self._get_boto3_client("bedrock-agent") - verbose_logger.debug(f"Creating Data Source for KB: {self.knowledge_base_id}") + verbose_logger.debug("Creating Data Source for KB: %s", self.knowledge_base_id) response = bedrock_agent.create_data_source( knowledgeBaseId=self.knowledge_base_id, @@ -566,7 +570,7 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): ds_id = response["dataSource"]["dataSourceId"] self._created_resources["data_source"] = ds_id - verbose_logger.info(f"Created Data Source: {ds_id}") + verbose_logger.info("Created Data Source: %s", ds_id) return ds_id def _get_boto3_client(self, service_name: str): @@ -652,25 +656,25 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): s3_client = self._get_boto3_client("s3") s3_key = f"{self.s3_prefix.rstrip('/')}/{filename}" - verbose_logger.debug(f"Uploading file to s3://{self.s3_bucket}/{s3_key}") + verbose_logger.debug("Uploading file to s3://%s/%s", self.s3_bucket, s3_key) s3_client.put_object( Bucket=self.s3_bucket, Key=s3_key, Body=file_content, ContentType=content_type or "application/octet-stream", ) - verbose_logger.info(f"Uploaded file to s3://{self.s3_bucket}/{s3_key}") + verbose_logger.info("Uploaded file to s3://%s/%s", self.s3_bucket, s3_key) # Step 2: Start ingestion job bedrock_agent = self._get_boto3_client("bedrock-agent") - verbose_logger.debug(f"Starting ingestion job for KB={self.knowledge_base_id}, DS={self.data_source_id}") + verbose_logger.debug("Starting ingestion job for KB=%s, DS=%s", self.knowledge_base_id, self.data_source_id) ingestion_response = bedrock_agent.start_ingestion_job( knowledgeBaseId=self.knowledge_base_id, dataSourceId=self.data_source_id, ) job_id = ingestion_response["ingestionJob"]["ingestionJobId"] - verbose_logger.info(f"Started ingestion job: {job_id}") + verbose_logger.info("Started ingestion job: %s", job_id) # Step 3: Wait for ingestion (optional) - use asyncio.sleep to avoid blocking if self.wait_for_ingestion: @@ -684,22 +688,22 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): ingestionJobId=job_id, ) status = job_status["ingestionJob"]["status"] - verbose_logger.debug(f"Ingestion job {job_id} status: {status}") + verbose_logger.debug("Ingestion job %s status: %s", job_id, status) if status == "COMPLETE": stats = job_status["ingestionJob"].get("statistics", {}) verbose_logger.info( - f"Ingestion complete: {stats.get('numberOfNewDocumentsIndexed', 0)} docs indexed" + "Ingestion complete: %s docs indexed", stats.get("numberOfNewDocumentsIndexed", 0) ) break elif status == "FAILED": failure_reasons = job_status["ingestionJob"].get("failureReasons", []) - verbose_logger.error(f"Ingestion failed: {failure_reasons}") + verbose_logger.error("Ingestion failed: %s", failure_reasons) break elif status in ("STARTING", "IN_PROGRESS"): await asyncio.sleep(2) else: - verbose_logger.warning(f"Unknown ingestion status: {status}") + verbose_logger.warning("Unknown ingestion status: %s", status) break return str(self.knowledge_base_id) if self.knowledge_base_id else None, s3_key diff --git a/litellm/rag/ingestion/file_parsers/pdf_parser.py b/litellm/rag/ingestion/file_parsers/pdf_parser.py index 2b4e07b224f..0301c5d6e03 100644 --- a/litellm/rag/ingestion/file_parsers/pdf_parser.py +++ b/litellm/rag/ingestion/file_parsers/pdf_parser.py @@ -35,7 +35,7 @@ def extract_text_from_pdf(file_content: bytes) -> str | None: if text_parts: extracted_text = "\n\n".join(text_parts) - verbose_logger.debug(f"Extracted {len(extracted_text)} characters from PDF using pypdf") + verbose_logger.debug("Extracted %s characters from PDF using pypdf", len(extracted_text)) return extracted_text except ImportError: @@ -56,13 +56,13 @@ def extract_text_from_pdf(file_content: bytes) -> str | None: if text_parts: extracted_text = "\n\n".join(text_parts) - verbose_logger.debug(f"Extracted {len(extracted_text)} characters from PDF using PyPDF2") + verbose_logger.debug("Extracted %s characters from PDF using PyPDF2", len(extracted_text)) return extracted_text except ImportError: verbose_logger.debug("PyPDF2 not available, PDF extraction requires OCR or pypdf/PyPDF2 library") except Exception as e: - verbose_logger.debug(f"PDF text extraction failed: {e}") + verbose_logger.debug("PDF text extraction failed: %s", e) return None diff --git a/litellm/rag/ingestion/gemini_ingestion.py b/litellm/rag/ingestion/gemini_ingestion.py index 5722936b742..f007e6282b9 100644 --- a/litellm/rag/ingestion/gemini_ingestion.py +++ b/litellm/rag/ingestion/gemini_ingestion.py @@ -162,7 +162,7 @@ class GeminiRAGIngestion(BaseRAGIngestion): response_data = response.json() store_name = response_data.get("name", "") - verbose_logger.debug(f"Created File Search store: {store_name}") + verbose_logger.debug("Created File Search store: %s", store_name) return store_name async def _upload_to_file_search_store( @@ -259,7 +259,7 @@ class GeminiRAGIngestion(BaseRAGIngestion): "x-goog-api-key": api_key, } - verbose_logger.debug(f"Initiating resumable upload: {url}") + verbose_logger.debug("Initiating resumable upload: %s", url) client = get_async_httpx_client( llm_provider=httpxSpecialProvider.RAG, @@ -275,13 +275,13 @@ class GeminiRAGIngestion(BaseRAGIngestion): error_msg = f"Failed to initiate upload: {response.text}" verbose_logger.error(error_msg) raise Exception(error_msg) - verbose_logger.debug(f"Initiate resumable upload response: {response.headers}") + verbose_logger.debug("Initiate resumable upload response: %s", response.headers) # Extract upload URL from response headers upload_url = response.headers.get("x-goog-upload-url") if not upload_url: raise Exception("No upload URL returned in response headers") - verbose_logger.debug(f"Got upload URL: {upload_url}") + verbose_logger.debug("Got upload URL: %s", upload_url) return upload_url async def _upload_file_content( @@ -301,7 +301,7 @@ class GeminiRAGIngestion(BaseRAGIngestion): "X-Goog-Upload-Command": "upload, finalize", } - verbose_logger.debug(f"Uploading file content ({len(file_content)} bytes)") + verbose_logger.debug("Uploading file content (%s bytes)", len(file_content)) client = get_async_httpx_client( llm_provider=httpxSpecialProvider.RAG, @@ -323,9 +323,9 @@ class GeminiRAGIngestion(BaseRAGIngestion): response_data = response.json() # The response should contain the document name or file reference file_id = response_data.get("name", "") or response_data.get("file", {}).get("name", "") - verbose_logger.debug(f"Upload complete. File ID: {file_id}") + verbose_logger.debug("Upload complete. File ID: %s", file_id) return file_id except Exception as e: - verbose_logger.warning(f"Could not parse upload response: {e}") + verbose_logger.warning("Could not parse upload response: %s", e) # Return a placeholder if we can't get the ID return "uploaded" diff --git a/litellm/rag/ingestion/s3_vectors_ingestion.py b/litellm/rag/ingestion/s3_vectors_ingestion.py index 6abd0737ba6..b50e798a7c9 100644 --- a/litellm/rag/ingestion/s3_vectors_ingestion.py +++ b/litellm/rag/ingestion/s3_vectors_ingestion.py @@ -105,7 +105,7 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): try: model_name = self.embedding_config["model"] - verbose_logger.debug(f"Auto-detecting dimension by making test embedding request to {model_name}") + verbose_logger.debug("Auto-detecting dimension by making test embedding request to %s", model_name) # Make a test embedding request test_input = "test" @@ -117,12 +117,13 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): # Get dimension from the response if response.data and len(response.data) > 0: dimension = len(response.data[0]["embedding"]) - verbose_logger.debug(f"Auto-detected dimension {dimension} for embedding model {model_name}") + verbose_logger.debug("Auto-detected dimension %s for embedding model %s", dimension, model_name) return dimension except Exception as e: verbose_logger.warning( - f"Could not auto-detect dimension from embedding model: {e}. " - f"Using default dimension of {S3_VECTORS_DEFAULT_DIMENSION}." + "Could not auto-detect dimension from embedding model: %s. Using default dimension of %s.", + e, + S3_VECTORS_DEFAULT_DIMENSION, ) return S3_VECTORS_DEFAULT_DIMENSION @@ -236,7 +237,7 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): async def _ensure_vector_bucket_exists(self): """Create vector bucket if it doesn't exist using GetVectorBucket and CreateVectorBucket APIs.""" - verbose_logger.debug(f"Ensuring S3 vector bucket exists: {self.vector_bucket_name}") + verbose_logger.debug("Ensuring S3 vector bucket exists: %s", self.vector_bucket_name) # Validate bucket name (AWS S3 naming rules) if len(self.vector_bucket_name) < 3: @@ -259,34 +260,34 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): try: response = await self._sign_and_execute_request("POST", get_url, data=get_body) if response.status_code == 200: - verbose_logger.debug(f"Vector bucket {self.vector_bucket_name} exists") + verbose_logger.debug("Vector bucket %s exists", self.vector_bucket_name) return except Exception as e: - verbose_logger.debug(f"Bucket check failed (may not exist): {e}, attempting to create") + verbose_logger.debug("Bucket check failed (may not exist): %s, attempting to create", e) # Create vector bucket using CreateVectorBucket API try: - verbose_logger.debug(f"Creating vector bucket: {self.vector_bucket_name}") + verbose_logger.debug("Creating vector bucket: %s", self.vector_bucket_name) create_url = f"https://s3vectors.{self.aws_region_name}.api.aws/CreateVectorBucket" create_body = safe_dumps({"vectorBucketName": self.vector_bucket_name}) response = await self._sign_and_execute_request("POST", create_url, data=create_body) if response.status_code in (200, 201): - verbose_logger.info(f"Created vector bucket: {self.vector_bucket_name}") + verbose_logger.info("Created vector bucket: %s", self.vector_bucket_name) elif response.status_code == 409: # Bucket already exists (ConflictException) - verbose_logger.debug(f"Vector bucket {self.vector_bucket_name} already exists") + verbose_logger.debug("Vector bucket %s already exists", self.vector_bucket_name) else: - verbose_logger.error(f"CreateVectorBucket failed: {response.status_code} - {response.text}") + verbose_logger.error("CreateVectorBucket failed: %s - %s", response.status_code, response.text) response.raise_for_status() except Exception as e: - verbose_logger.exception(f"Error creating vector bucket: {e}") + verbose_logger.exception("Error creating vector bucket: %s", e) raise async def _ensure_vector_index_exists(self): """Create vector index if it doesn't exist using GetIndex and CreateIndex APIs.""" - verbose_logger.debug(f"Ensuring vector index exists: {self.vector_bucket_name}/{self.index_name}") + verbose_logger.debug("Ensuring vector index exists: %s/%s", self.vector_bucket_name, self.index_name) # Try to get index info using GetIndex API get_url = f"https://s3vectors.{self.aws_region_name}.api.aws/GetIndex" @@ -295,15 +296,18 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): try: response = await self._sign_and_execute_request("POST", get_url, data=get_body) if response.status_code == 200: - verbose_logger.debug(f"Vector index {self.index_name} exists") + verbose_logger.debug("Vector index %s exists", self.index_name) return except Exception as e: - verbose_logger.debug(f"Index check failed (may not exist): {e}, attempting to create") + verbose_logger.debug("Index check failed (may not exist): %s, attempting to create", e) # Create vector index using CreateIndex API try: verbose_logger.debug( - f"Creating vector index: {self.index_name} with dimension={self.dimension}, metric={self.distance_metric}" + "Creating vector index: %s with dimension=%s, metric=%s", + self.index_name, + self.dimension, + self.distance_metric, ) # Prepare index configuration per AWS API docs @@ -322,14 +326,14 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): response = await self._sign_and_execute_request("POST", create_url, data=safe_dumps(index_config)) if response.status_code in (200, 201): - verbose_logger.info(f"Created vector index: {self.index_name}") + verbose_logger.info("Created vector index: %s", self.index_name) elif response.status_code == 409: - verbose_logger.debug(f"Vector index {self.index_name} already exists") + verbose_logger.debug("Vector index %s already exists", self.index_name) else: - verbose_logger.error(f"CreateIndex failed: {response.status_code} - {response.text}") + verbose_logger.error("CreateIndex failed: %s - %s", response.status_code, response.text) response.raise_for_status() except Exception as e: - verbose_logger.exception(f"Error creating vector index: {e}") + verbose_logger.exception("Error creating vector index: %s", e) raise async def _put_vectors(self, vectors: list[dict[str, Any]]): @@ -339,7 +343,7 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): Args: vectors: List of vector objects with keys: "key", "data", "metadata" """ - verbose_logger.debug(f"Storing {len(vectors)} vectors in {self.vector_bucket_name}/{self.index_name}") + verbose_logger.debug("Storing %s vectors in %s/%s", len(vectors), self.vector_bucket_name, self.index_name) url = f"https://s3vectors.{self.aws_region_name}.api.aws/PutVectors" @@ -354,12 +358,12 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): response = await self._sign_and_execute_request("POST", url, data=safe_dumps(request_body)) if response.status_code in (200, 201): - verbose_logger.info(f"Successfully stored {len(vectors)} vectors in index {self.index_name}") + verbose_logger.info("Successfully stored %s vectors in index %s", len(vectors), self.index_name) else: - verbose_logger.error(f"PutVectors failed with status {response.status_code}: {response.text}") + verbose_logger.error("PutVectors failed with status %s: %s", response.status_code, response.text) response.raise_for_status() except Exception as e: - verbose_logger.exception(f"Error storing vectors: {e}") + verbose_logger.exception("Error storing vectors: %s", e) raise async def embed( @@ -381,7 +385,7 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): embedding_model = self.embedding_config.get("model", "text-embedding-3-small") - verbose_logger.debug(f"Generating embeddings for {len(chunks)} chunks using {embedding_model}") + verbose_logger.debug("Generating embeddings for %s chunks using %s", len(chunks), embedding_model) # Convert to list to ensure type compatibility input_chunks: list[str] = list(chunks) @@ -476,7 +480,7 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): Returns: Query results with vectors and metadata """ - verbose_logger.debug(f"Querying index {vector_store_id} with query: {query}") + verbose_logger.debug("Querying index %s with query: %s", vector_store_id, query) # Generate query embedding if not self.embedding_config: @@ -504,7 +508,7 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): if response.status_code == 200: results = response.json() - verbose_logger.debug(f"Query returned {len(results.get('vectors', []))} results") + verbose_logger.debug("Query returned %s results", len(results.get("vectors", []))) # Check if query terms appear in results if results.get("vectors"): @@ -517,8 +521,8 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): # Return results even if exact match not found return results else: - verbose_logger.error(f"QueryVectors failed with status {response.status_code}: {response.text}") + verbose_logger.error("QueryVectors failed with status %s: %s", response.status_code, response.text) return None except Exception as e: - verbose_logger.exception(f"Error querying vectors: {e}") + verbose_logger.exception("Error querying vectors: %s", e) return None diff --git a/litellm/rag/ingestion/vertex_ai_ingestion.py b/litellm/rag/ingestion/vertex_ai_ingestion.py index ababca1a955..43c3ed1ac29 100644 --- a/litellm/rag/ingestion/vertex_ai_ingestion.py +++ b/litellm/rag/ingestion/vertex_ai_ingestion.py @@ -169,8 +169,8 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase): "vertexPredictionEndpoint": {"endpoint": embedding_model} } - verbose_logger.debug(f"Creating RAG corpus: {url}") - verbose_logger.debug(f"Request body: {json.dumps(request_body, indent=2)}") + verbose_logger.debug("Creating RAG corpus: %s", url) + verbose_logger.debug("Request body: %s", json.dumps(request_body, indent=2)) client = get_async_httpx_client( llm_provider=httpxSpecialProvider.RAG, @@ -191,7 +191,7 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase): raise Exception(error_msg) response_data = response.json() - verbose_logger.debug(f"Create corpus response: {json.dumps(response_data, indent=2)}") + verbose_logger.debug("Create corpus response: %s", json.dumps(response_data, indent=2)) # The response is a long-running operation # Check if it's already done or if we need to poll @@ -201,13 +201,13 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase): else: # Need to poll the operation operation_name = response_data.get("name", "") - verbose_logger.debug(f"Polling operation: {operation_name}") + verbose_logger.debug("Polling operation: %s", operation_name) corpus_name = await self._poll_operation( operation_name=operation_name, access_token=access_token, ) - verbose_logger.debug(f"Created RAG corpus: {corpus_name}") + verbose_logger.debug("Created RAG corpus: %s", corpus_name) return corpus_name async def _poll_operation( @@ -272,7 +272,7 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase): else: raise Exception(f"No corpus name in operation response: {operation_data}") - verbose_logger.debug(f"Operation not done yet, attempt {attempt + 1}/{max_retries}") + verbose_logger.debug("Operation not done yet, attempt %s/%s", attempt + 1, max_retries) await asyncio.sleep(retry_delay) raise Exception(f"Operation timed out after {max_retries} attempts") @@ -342,8 +342,8 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase): if chunk_overlap: chunking_config["chunk_overlap"] = chunk_overlap - verbose_logger.debug(f"Uploading file to RAG corpus: {url}") - verbose_logger.debug(f"Metadata: {json.dumps(metadata, indent=2)}") + verbose_logger.debug("Uploading file to RAG corpus: %s", url) + verbose_logger.debug("Metadata: %s", json.dumps(metadata, indent=2)) # Prepare multipart form data files = { @@ -381,10 +381,10 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase): if not file_id: file_id = response_data.get("name", "") - verbose_logger.debug(f"Upload complete. File ID: {file_id}") + verbose_logger.debug("Upload complete. File ID: %s", file_id) return file_id except Exception as e: - verbose_logger.warning(f"Could not parse upload response: {e}") + verbose_logger.warning("Could not parse upload response: %s", e) return "uploaded" async def _import_files_from_gcs( @@ -433,8 +433,8 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase): if max_embedding_qpm: request_body["importRagFilesConfig"]["maxEmbeddingRequestsPerMin"] = max_embedding_qpm - verbose_logger.debug(f"Importing files from GCS: {url}") - verbose_logger.debug(f"Request body: {json.dumps(request_body, indent=2)}") + verbose_logger.debug("Importing files from GCS: %s", url) + verbose_logger.debug("Request body: %s", json.dumps(request_body, indent=2)) client = get_async_httpx_client( llm_provider=httpxSpecialProvider.RAG, @@ -458,5 +458,5 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase): response_data = response.json() operation_name = response_data.get("name", "") - verbose_logger.debug(f"Import operation started: {operation_name}") + verbose_logger.debug("Import operation started: %s", operation_name) return operation_name diff --git a/litellm/repositories/config_repository.py b/litellm/repositories/config_repository.py index bc3e7fbaf9f..5d2a6b1421d 100644 --- a/litellm/repositories/config_repository.py +++ b/litellm/repositories/config_repository.py @@ -197,7 +197,7 @@ class ConfigRepository: param_name = response.param_name param_value = response.param_value - verbose_proxy_logger.debug(f"param_name={param_name}, param_value={param_value}") + verbose_proxy_logger.debug("param_name=%s, param_value=%s", param_name, param_value) if param_name is not None and param_value is not None: config = self._update_config_fields( diff --git a/litellm/rerank_api/main.py b/litellm/rerank_api/main.py index 2733fed744a..aa7ade5fdd9 100644 --- a/litellm/rerank_api/main.py +++ b/litellm/rerank_api/main.py @@ -159,7 +159,7 @@ def rerank( instruction=instruction, non_default_params=kwargs, ) - verbose_logger.debug(f"optional_rerank_params: {optional_rerank_params}") + verbose_logger.debug("optional_rerank_params: %s", optional_rerank_params) if isinstance(optional_params.timeout, str): optional_params.timeout = float(optional_params.timeout) @@ -534,5 +534,5 @@ def rerank( # Placeholder return return response except Exception as e: - verbose_logger.error(f"Error in rerank: {e}") + verbose_logger.error("Error in rerank: %s", e) raise exception_type(model=model, custom_llm_provider=custom_llm_provider, original_exception=e) diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index 8c0328fe5f0..357c2d25c22 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -376,7 +376,7 @@ async def acompletion_with_mcp( chunk = await self.follow_up_iterator.__anext__() from litellm._logging import verbose_logger - verbose_logger.debug(f"Follow-up chunk yielded: {chunk}") + verbose_logger.debug("Follow-up chunk yielded: %s", chunk) return chunk except StopAsyncIteration: self.follow_up_exhausted = True @@ -476,7 +476,7 @@ async def acompletion_with_mcp( from litellm._logging import verbose_logger verbose_logger.warning( - f"Follow-up response is not a CustomStreamWrapper: {type(follow_up_response)}" + "Follow-up response is not a CustomStreamWrapper: %s", type(follow_up_response) ) self.follow_up_stream = None diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 39881277a10..0777f86d52b 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -171,7 +171,7 @@ class LiteLLM_Proxy_MCP_Handler: ) return user_api_key_auth.model_copy(update={"object_permission": updated_op}) except Exception as _e: - verbose_logger.debug(f"Could not apply toolset permissions: {_e}") + verbose_logger.debug("Could not apply toolset permissions: %s", _e) return user_api_key_auth @staticmethod @@ -238,14 +238,16 @@ class LiteLLM_Proxy_MCP_Handler: # None means no grants configured → deny (consistent with # fetch_mcp_toolsets which returns [] for unconfigured keys) if granted is None or toolset.toolset_id not in granted: - verbose_logger.debug(f"Key does not have access to toolset '{name}', skipping.") + verbose_logger.debug( + "Key does not have access to toolset '%s', skipping.", name + ) continue resolved_toolset_ids.append(toolset.toolset_id) # Don't add to resolved_mcp_servers — toolset scope # restricts via object_permission, not server name filter. continue except Exception as _e: - verbose_logger.debug(f"Could not resolve '{name}' as toolset: {_e}") + verbose_logger.debug("Could not resolve '%s' as toolset: %s", name, _e) resolved_mcp_servers.append(name) # Apply all resolved toolsets at once (union), avoiding permission overwrite. @@ -664,7 +666,7 @@ class LiteLLM_Proxy_MCP_Handler: ) = LiteLLM_Proxy_MCP_Handler._extract_tool_call_details(tool_call) if not tool_name: - verbose_logger.warning(f"Tool call missing name: {tool_call}") + verbose_logger.warning("Tool call missing name: %s", tool_call) continue parsed_arguments = LiteLLM_Proxy_MCP_Handler._parse_tool_arguments(tool_arguments) @@ -844,7 +846,7 @@ class LiteLLM_Proxy_MCP_Handler: request_data=logging_request_data, error=e, ) - verbose_logger.error(f"BlockedPiiEntityError in MCP tool call: {e}") + verbose_logger.error("BlockedPiiEntityError in MCP tool call: %s", e) error_message = f"Tool call blocked: PII entity '{getattr(e, 'entity_type', 'unknown')}' detected by guardrail '{getattr(e, 'guardrail_name', 'unknown')}'. {e}" tool_results.append( { @@ -860,7 +862,7 @@ class LiteLLM_Proxy_MCP_Handler: request_data=logging_request_data, error=e, ) - verbose_logger.error(f"GuardrailRaisedException in MCP tool call: {e}") + verbose_logger.error("GuardrailRaisedException in MCP tool call: %s", e) error_message = ( f"Tool call blocked: Guardrail '{getattr(e, 'guardrail_name', 'unknown')}' violation. {e}" ) @@ -878,7 +880,7 @@ class LiteLLM_Proxy_MCP_Handler: request_data=logging_request_data, error=e, ) - verbose_logger.error(f"HTTPException in MCP tool call: {e}") + verbose_logger.error("HTTPException in MCP tool call: %s", e) error_message = f"Tool call failed: {str(e.detail) if hasattr(e, 'detail') else str(e)}" tool_results.append( { @@ -894,7 +896,7 @@ class LiteLLM_Proxy_MCP_Handler: request_data=logging_request_data, error=e, ) - verbose_logger.exception(f"Error executing MCP tool call: {e}") + verbose_logger.exception("Error executing MCP tool call: %s", e) tool_results.append( { "tool_call_id": tool_call_id, diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index c68628429da..c384dd86f5e 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -124,10 +124,10 @@ async def create_mcp_list_tools_events( ) events.append(output_item_done_event) - verbose_logger.debug(f"Created {len(events)} MCP discovery events") + verbose_logger.debug("Created %s MCP discovery events", len(events)) except Exception as e: - verbose_logger.error(f"Error creating MCP list tools events: {e}") + verbose_logger.error("Error creating MCP list tools events: %s", e) import traceback traceback.print_exc() @@ -513,7 +513,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): response_obj = getattr(chunk, "response", None) if response_obj and hasattr(response_obj, "id"): self._cached_response_id = response_obj.id - verbose_logger.debug(f"Cached response ID: {self._cached_response_id}") + verbose_logger.debug("Cached response ID: %s", self._cached_response_id) # After emitting response.output_item.added, transition to MCP discovery if not self.initial_events_emitted and hasattr(chunk, "type"): @@ -576,7 +576,9 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): response_obj = getattr(chunk, "response", None) if response_obj and hasattr(response_obj, "id"): if response_obj.id != self._cached_response_id: - verbose_logger.debug(f"Updating response ID from {response_obj.id} to {self._cached_response_id}") + verbose_logger.debug( + "Updating response ID from %s to %s", response_obj.id, self._cached_response_id + ) response_obj.id = self._cached_response_id # If auto-execution is enabled, check for completed responses @@ -607,7 +609,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): params_for_llm[key] = value # Copy all params as-is since tools are already processed tools_count = len(params_for_llm.get("tools", [])) if params_for_llm.get("tools") else 0 - verbose_logger.debug(f"Making LLM call with {tools_count} tools") + verbose_logger.debug("Making LLM call with %s tools", tools_count) response = await aresponses(**params_for_llm) # Set the base iterator @@ -617,15 +619,15 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): self.model = getattr(response, "model", self.model) self.litellm_metadata = getattr(response, "litellm_metadata", {}) self.custom_llm_provider = getattr(response, "custom_llm_provider", self.custom_llm_provider) - verbose_logger.debug(f"Created base iterator: {type(self.base_iterator)}") + verbose_logger.debug("Created base iterator: %s", type(self.base_iterator)) else: # Non-streaming response - this shouldn't happen but handle it - verbose_logger.warning(f"Got non-streaming response: {type(response)}") + verbose_logger.warning("Got non-streaming response: %s", type(response)) self.base_iterator = None self.phase = "finished" except Exception as e: - verbose_logger.error(f"Error creating initial response iterator: {e}") + verbose_logger.error("Error creating initial response iterator: %s", e) import traceback traceback.print_exc() @@ -742,7 +744,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): self._tool_results_for_response = self.collected_response except Exception as e: - verbose_logger.error(f"Error in tool execution: {e}") + verbose_logger.error("Error in tool execution: %s", e) import traceback traceback.print_exc() @@ -807,7 +809,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): self._cached_response_id = None except Exception as e: - verbose_logger.error(f"Error creating follow-up iterator: {e}") + verbose_logger.error("Error creating follow-up iterator: %s", e) import traceback traceback.print_exc() diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 6e951367995..3cd118fb1a4 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -557,7 +557,7 @@ class ResponsesAPIRequestUtils: response_id=decoded_response_id, ) except Exception as e: - verbose_logger.debug(f"Error decoding response_id '{response_id}': {e}") + verbose_logger.debug("Error decoding response_id '%s': %s", response_id, e) return DecodedResponseId( custom_llm_provider=None, model_id=None, @@ -670,7 +670,7 @@ class ResponsesAPIRequestUtils: response_id=original_container_id, ) except Exception as e: - verbose_logger.debug(f"Error decoding container_id '{container_id}': {e}") + verbose_logger.debug("Error decoding container_id '%s': %s", container_id, e) return DecodedResponseId( custom_llm_provider=None, model_id=None, diff --git a/litellm/router.py b/litellm/router.py index 6bf1bdfc670..7d6499cf7d2 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -721,7 +721,8 @@ class Router: self.retry_policy = retry_policy if self.retry_policy is not None: verbose_router_logger.info( - f"\033[32mRouter Custom Retry Policy Set:\n{self.retry_policy.model_dump(exclude_none=True)}\033[0m" + "\x1b[32mRouter Custom Retry Policy Set:\n%s\x1b[0m", + self.retry_policy.model_dump(exclude_none=True), ) self.model_group_retry_policy: dict[str, RetryPolicy] | None = model_group_retry_policy @@ -736,7 +737,8 @@ class Router: if self.allowed_fails_policy is not None: verbose_router_logger.info( - f"\033[32mRouter Custom Allowed Fails Policy Set:\n{self.allowed_fails_policy.model_dump(exclude_none=True)}\033[0m" + "\x1b[32mRouter Custom Allowed Fails Policy Set:\n%s\x1b[0m", + self.allowed_fails_policy.model_dump(exclude_none=True), ) self.alerting_config: AlertingConfig | None = alerting_config @@ -941,7 +943,7 @@ class Router: litellm.input_callback = [c for c in litellm.input_callback if id(c) not in selector_ids] def routing_strategy_init(self, routing_strategy: RoutingStrategy | str, routing_strategy_args: dict): - verbose_router_logger.info(f"Routing strategy: {routing_strategy}") + verbose_router_logger.info("Routing strategy: %s", routing_strategy) self._validate_routing_strategy(routing_strategy) self._reset_custom_routing_strategy() @@ -1721,7 +1723,7 @@ class Router: return _deployment_copy except Exception as e: - verbose_router_logger.debug(f"Error occurred while printing deployment - {e}") + verbose_router_logger.debug("Error occurred while printing deployment - %s", e) raise e ### COMPLETION, EMBEDDING, IMG GENERATION FUNCTIONS @@ -1732,7 +1734,7 @@ class Router: response = router.completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hey, how's it going?"}] """ try: - verbose_router_logger.debug(f"router.completion(model={model},..)") + verbose_router_logger.debug("router.completion(model=%s,..)", model) kwargs["model"] = model kwargs["messages"] = messages kwargs["original_function"] = self._completion @@ -1806,7 +1808,7 @@ class Router: **kwargs, } response = litellm.completion(**input_kwargs) - verbose_router_logger.info(f"litellm.completion(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) ## CHECK CONTENT FILTER ERROR ## if isinstance(response, ModelResponse): @@ -1829,7 +1831,7 @@ class Router: return response except Exception as e: - verbose_router_logger.info(f"litellm.completion(model={model_name})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.completion(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) # Set per-deployment num_retries on exception for retry logic if deployment is not None: self._set_deployment_num_retries_on_exception(e, deployment) @@ -1892,7 +1894,7 @@ class Router: messages = copy.deepcopy(messages) - verbose_router_logger.info(f"Starting silent experiment for model {silent_model}") + verbose_router_logger.info("Starting silent experiment for model %s", silent_model) silent_kwargs = self._get_silent_experiment_kwargs(**kwargs) @@ -1924,7 +1926,7 @@ class Router: finally: loop.close() except Exception as e: - verbose_router_logger.error(f"Silent experiment failed for model {silent_model}: {e}") + verbose_router_logger.error("Silent experiment failed for model %s: %s", silent_model, e) # fmt: off @@ -2160,7 +2162,7 @@ class Router: except Exception as fallback_error: # If fallback also fails, log and re-raise original error - verbose_router_logger.error(f"Fallback also failed: {fallback_error}") + verbose_router_logger.error("Fallback also failed: %s", fallback_error) # No fallback handled the mid-stream error, so surface the # real provider exception (e.g. RateLimitError) instead of # leaking the internal MidStreamFallbackError to the client @@ -2579,7 +2581,7 @@ class Router: else: yield fallback_response except Exception as fallback_error: - verbose_router_logger.error(f"Responses streaming fallback also failed: {fallback_error}") + verbose_router_logger.error("Responses streaming fallback also failed: %s", fallback_error) if ( isinstance(fallback_error, MidStreamFallbackError) and fallback_error.original_exception is not None @@ -2704,7 +2706,7 @@ class Router: yield None except Exception as fallback_error: - verbose_router_logger.error(f"Fallback also failed: {fallback_error}") + verbose_router_logger.error("Fallback also failed: %s", fallback_error) if ( isinstance(fallback_error, MidStreamFallbackError) and fallback_error.original_exception is not None @@ -2742,7 +2744,7 @@ class Router: messages = copy.deepcopy(messages) - verbose_router_logger.info(f"Starting silent experiment for model {silent_model}") + verbose_router_logger.info("Starting silent experiment for model %s", silent_model) silent_kwargs = self._get_silent_experiment_kwargs(**kwargs) # Override model_group to correctly attribute metrics to the silent model @@ -2755,7 +2757,7 @@ class Router: **silent_kwargs, ) except Exception as e: - verbose_router_logger.error(f"Silent experiment failed for model {silent_model}: {e}") + verbose_router_logger.error("Silent experiment failed for model %s: %s", silent_model, e) async def _acompletion( self, model: str, messages: list[dict[str, str]], **kwargs @@ -2879,7 +2881,7 @@ class Router: ) self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.acompletion(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.acompletion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) # debug how often this deployment picked self._track_deployment_metrics( deployment=deployment, @@ -2908,7 +2910,7 @@ class Router: self._set_failed_deployment_id_on_exception(e, deployment) raise e except Exception as e: - verbose_router_logger.info(f"litellm.acompletion(model={model_name})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.acompletion(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) if model_name is not None: self.fail_calls[model_name] += 1 # Set per-deployment num_retries on exception for retry logic @@ -3363,7 +3365,7 @@ class Router: result = await self.acompletion(model=model, messages=messages, stream=stream, **kwargs) # type: ignore return result except asyncio.CancelledError: - verbose_router_logger.debug(f"Received 'task.cancel'. Cancelling call w/ model={model}.") + verbose_router_logger.debug("Received 'task.cancel'. Cancelling call w/ model=%s.", model) raise except Exception as e: return e @@ -3665,7 +3667,7 @@ class Router: def _image_generation(self, prompt: str, model: str, **kwargs): model_name = "" try: - verbose_router_logger.debug(f"Inside _image_generation()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _image_generation()- model: %s; kwargs: %s", model, kwargs) deployment = self.get_available_deployment( model=model, messages=[{"role": "user", "content": "prompt"}], @@ -3694,10 +3696,10 @@ class Router: } ) self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.image_generation(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.info(f"litellm.image_generation(model={model_name})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -3725,7 +3727,7 @@ class Router: async def _aimage_generation(self, prompt: str, model: str, **kwargs): model_name = model try: - verbose_router_logger.debug(f"Inside _image_generation()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _image_generation()- model: %s; kwargs: %s", model, kwargs) parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) deployment = await self.async_get_available_deployment( model=model, @@ -3778,10 +3780,10 @@ class Router: response = await response self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.aimage_generation(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.aimage_generation(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.info(f"litellm.aimage_generation(model={model_name})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.aimage_generation(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -3831,7 +3833,7 @@ class Router: async def _atranscription(self, file: FileTypes, model: str, **kwargs): model_name = model try: - verbose_router_logger.debug(f"Inside _atranscription()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _atranscription()- model: %s; kwargs: %s", model, kwargs) parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) deployment = await self.async_get_available_deployment( model=model, @@ -3882,10 +3884,10 @@ class Router: response = await response self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.atranscription(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.atranscription(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.info(f"litellm.atranscription(model={model_name})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.atranscription(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -3945,7 +3947,7 @@ class Router: async def _aspeech(self, model: str, input: str, voice: str, **kwargs): model_name = model try: - verbose_router_logger.debug(f"Inside _aspeech()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _aspeech()- model: %s; kwargs: %s", model, kwargs) parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) deployment = await self.async_get_available_deployment( model=model, @@ -3996,10 +3998,10 @@ class Router: response = await response self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.aspeech(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.aspeech(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.info(f"litellm.aspeech(model={model_name})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.aspeech(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -4028,7 +4030,7 @@ class Router: async def _arerank(self, model: str, **kwargs): model_name = None try: - verbose_router_logger.debug(f"Inside _rerank()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _rerank()- model: %s; kwargs: %s", model, kwargs) deployment = await self.async_get_available_deployment( model=model, specific_deployment=kwargs.pop("specific_deployment", None), @@ -4054,10 +4056,10 @@ class Router: ) self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.arerank(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.arerank(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.info(f"litellm.arerank(model={model_name})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.arerank(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -4136,7 +4138,7 @@ class Router: async def _atext_completion(self, model: str, prompt: str, **kwargs): try: - verbose_router_logger.debug(f"Inside _atext_completion()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _atext_completion()- model: %s; kwargs: %s", model, kwargs) parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) deployment = await self.async_get_available_deployment( model=model, @@ -4188,10 +4190,10 @@ class Router: response = await response self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.atext_completion(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.atext_completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.info(f"litellm.atext_completion(model={model})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.atext_completion(model=%s)\x1b[31m Exception %s\x1b[0m", model, e) if model is not None: self.fail_calls[model] += 1 raise e @@ -4226,7 +4228,7 @@ class Router: async def _aadapter_completion(self, adapter_id: str, model: str, **kwargs): try: - verbose_router_logger.debug(f"Inside _aadapter_completion()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _aadapter_completion()- model: %s; kwargs: %s", model, kwargs) parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) deployment = await self.async_get_available_deployment( model=model, @@ -4278,10 +4280,10 @@ class Router: response = await response # type: ignore self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.aadapter_completion(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.aadapter_completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.info(f"litellm.aadapter_completion(model={model})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.aadapter_completion(model=%s)\x1b[31m Exception %s\x1b[0m", model, e) if model is not None: self.fail_calls[model] += 1 raise e @@ -4338,7 +4340,7 @@ class Router: kwargs=kwargs, metadata_variable_name="litellm_metadata", ) - verbose_router_logger.debug(f"Inside aguardrail() - guardrail_name: {guardrail_name}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside aguardrail() - guardrail_name: %s; kwargs: %s", guardrail_name, kwargs) response = await self.async_function_with_fallbacks(**kwargs) return response @@ -4363,7 +4365,7 @@ class Router: ) verbose_router_logger.debug( - f"Selected guardrail deployment: {selected_guardrail.get('litellm_params', {}).get('guardrail')}" + "Selected guardrail deployment: %s", selected_guardrail.get("litellm_params", {}).get("guardrail") ) # Pass the selected guardrail config to the original function @@ -4413,7 +4415,9 @@ class Router: kwargs["original_generic_function"] = original_function kwargs["original_function"] = self._ageneric_api_call_with_fallbacks_helper self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs, metadata_variable_name="litellm_metadata") - verbose_router_logger.debug(f"Inside ageneric_api_call_with_fallbacks() - model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug( + "Inside ageneric_api_call_with_fallbacks() - model: %s; kwargs: %s", model, kwargs + ) response = await self.async_function_with_fallbacks(**kwargs) return response @@ -4536,11 +4540,13 @@ class Router: response = await response # type: ignore self.success_calls[model_name] += 1 - verbose_router_logger.info(f"ageneric_api_call_with_fallbacks(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("ageneric_api_call_with_fallbacks(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.info(f"ageneric_api_call_with_fallbacks(model={model})\033[31m Exception {e}\033[0m") + verbose_router_logger.info( + "ageneric_api_call_with_fallbacks(model=%s)\x1b[31m Exception %s\x1b[0m", model, e + ) if model is not None: self.fail_calls[model] += 1 raise e @@ -4607,7 +4613,7 @@ class Router: metadata_variable_name = _get_router_metadata_variable_name(function_name="generic_api_call") try: verbose_router_logger.debug( - f"Inside _generic_api_call() - handler: {handler_name}, model: {model}; kwargs: {kwargs}" + "Inside _generic_api_call() - handler: %s, model: %s; kwargs: %s", handler_name, model, kwargs ) self._update_kwargs_before_fallbacks( model=model, @@ -4657,10 +4663,10 @@ class Router: ) self.success_calls[model_name] += 1 - verbose_router_logger.info(f"{handler_name}(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("%s(model=%s)\x1b[32m 200 OK\x1b[0m", handler_name, model_name) return response except Exception as e: - verbose_router_logger.info(f"{handler_name}(model={model})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("%s(model=%s)\x1b[31m Exception %s\x1b[0m", handler_name, model, e) if model is not None: self.fail_calls[model] += 1 raise e @@ -4685,7 +4691,7 @@ class Router: def _embedding(self, input: str | list, model: str, **kwargs): model_name = None try: - verbose_router_logger.debug(f"Inside embedding()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside embedding()- model: %s; kwargs: %s", model, kwargs) deployment = self.get_available_deployment( model=model, input=input, @@ -4722,10 +4728,10 @@ class Router: } ) self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.embedding(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.embedding(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.info(f"litellm.embedding(model={model_name})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.embedding(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -4758,7 +4764,7 @@ class Router: async def _aembedding(self, input: str | list, model: str, **kwargs): model_name = None try: - verbose_router_logger.debug(f"Inside _aembedding()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _aembedding()- model: %s; kwargs: %s", model, kwargs) parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) deployment = await self.async_get_available_deployment( model=model, @@ -4809,10 +4815,10 @@ class Router: response = await response self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.aembedding(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.aembedding(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.info(f"litellm.aembedding(model={model_name})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.aembedding(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -4849,7 +4855,7 @@ class Router: try: from litellm.router_utils.common_utils import add_model_file_id_mappings - verbose_router_logger.debug(f"Inside _atext_completion()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _atext_completion()- model: %s; kwargs: %s", model, kwargs) parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) healthy_deployments = await self.async_get_healthy_deployments( model=model, @@ -4940,7 +4946,7 @@ class Router: response = await response # type: ignore self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.acreate_file(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.acreate_file(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response @@ -4965,7 +4971,7 @@ class Router: return returned_response except Exception as e: verbose_router_logger.exception( - f"litellm.acreate_file(model={model}, {kwargs})\033[31m Exception {e}\033[0m" + "litellm.acreate_file(model=%s, %s)\x1b[31m Exception %s\x1b[0m", model, kwargs, e ) if model is not None: self.fail_calls[model] += 1 @@ -5056,11 +5062,13 @@ class Router: response = await response self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.avector_store_create(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.avector_store_create(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.exception(f"litellm.avector_store_create(model={model})\033[31m Exception {e}\033[0m") + verbose_router_logger.exception( + "litellm.avector_store_create(model=%s)\x1b[31m Exception %s\x1b[0m", model, e + ) if model is not None: self.fail_calls[model] += 1 raise e @@ -5110,7 +5118,7 @@ class Router: **kwargs, ) -> LiteLLMBatch: try: - verbose_router_logger.debug(f"Inside _acreate_batch()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _acreate_batch()- model: %s; kwargs: %s", model, kwargs) parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) deployment = await self.async_get_available_deployment( model=model, @@ -5170,12 +5178,12 @@ class Router: response = await response # type: ignore self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.acreate_batch(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.acreate_batch(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response # type: ignore except Exception as e: verbose_router_logger.exception( - f"litellm._acreate_batch(model={model}, {kwargs})\033[31m Exception {e}\033[0m" + "litellm._acreate_batch(model=%s, %s)\x1b[31m Exception %s\x1b[0m", model, kwargs, e ) if model is not None: self.fail_calls[model] += 1 @@ -5325,7 +5333,7 @@ class Router: **kwargs, ) -> LiteLLMBatch: try: - verbose_router_logger.debug(f"Inside _acancel_batch()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _acancel_batch()- model: %s; kwargs: %s", model, kwargs) parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) deployment = await self.async_get_available_deployment( model=model, @@ -5392,12 +5400,12 @@ class Router: response = await response # type: ignore self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.acancel_batch(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.acancel_batch(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response # type: ignore except Exception as e: verbose_router_logger.exception( - f"litellm._acancel_batch(model={model}, {kwargs})\033[31m Exception {e}\033[0m" + "litellm._acancel_batch(model=%s, %s)\x1b[31m Exception %s\x1b[0m", model, kwargs, e ) if model is not None: self.fail_calls[model] += 1 @@ -6061,8 +6069,10 @@ class Router: return None verbose_router_logger.debug( - f"Weighted failover: exclude={excluded!r}, remaining={len(remaining)} " - f"for model_group={original_model_group!r}" + "Weighted failover: exclude=%r, remaining=%s for model_group=%r", + excluded, + len(remaining), + original_model_group, ) meta["_failover_excluded_ids"] = list(excluded) @@ -6107,7 +6117,7 @@ class Router: """ Common utilities for async_function_with_fallbacks """ - verbose_router_logger.debug(f"Traceback{traceback.format_exc()}") + verbose_router_logger.debug("Traceback%s", traceback.format_exc()) original_exception = e fallback_model_group = None original_model_group: str | None = kwargs.get("model") # type: ignore @@ -6284,7 +6294,7 @@ class Router: if litellm.expose_router_debug_in_errors: e.message += f"\n{error_message}" if fallbacks is not None and model_group is not None: - verbose_router_logger.debug(f"inside model fallbacks: {mask_sensitive_structure(fallbacks)}") + verbose_router_logger.debug("inside model fallbacks: %s", mask_sensitive_structure(fallbacks)) ( fallback_model_group, generic_fallback_idx, @@ -6299,7 +6309,9 @@ class Router: if fallback_model_group is None: masked_fallbacks = mask_sensitive_structure(fallbacks) verbose_router_logger.info( - f"No fallback model group found for original model_group={model_group}. Fallbacks={masked_fallbacks}" + "No fallback model group found for original model_group=%s. Fallbacks=%s", + model_group, + masked_fallbacks, ) if hasattr(original_exception, "message") and litellm.expose_router_debug_in_errors: original_exception.message += f"No fallback model group found for original model_group={model_group}. Fallbacks={masked_fallbacks}" # type: ignore @@ -6374,7 +6386,7 @@ class Router: else: response = await self.async_function_with_retries(*args, **kwargs) if verbose_router_logger.isEnabledFor(logging.DEBUG): - verbose_router_logger.debug(f"Async Response: {response}") + verbose_router_logger.debug("Async Response: %s", response) response = add_fallback_headers_to_response( response=response, attempted_fallbacks=0, @@ -6466,7 +6478,7 @@ class Router: _metadata.update({"model_group_size": len(model_list)}) verbose_router_logger.debug( - f"async function w/ retries: original_function - {original_function}, num_retries - {num_retries}" + "async function w/ retries: original_function - %s, num_retries - %s", original_function, num_retries ) ## ADD RETRY TRACKING TO METADATA - used for spend logs retry tracking _metadata["attempted_retries"] = 0 @@ -6538,7 +6550,7 @@ class Router: else: raise - verbose_router_logger.debug(f"Retrying request with num_retries: {num_retries}") + verbose_router_logger.debug("Retrying request with num_retries: %s", num_retries) # decides how long to sleep before retry retry_after = self._time_to_sleep_before_retry( e=original_exception, @@ -6655,7 +6667,7 @@ class Router: if mock_testing_rate_limit_error is not None and mock_testing_rate_limit_error is True: verbose_router_logger.info( - f"litellm.router.py::_mock_rate_limit_error() - Raising mock RateLimitError for model={model_group}" + "litellm.router.py::_mock_rate_limit_error() - Raising mock RateLimitError for model=%s", model_group ) raise litellm.RateLimitError( model=model_group, @@ -6945,7 +6957,7 @@ class Router: except Exception as e: verbose_router_logger.debug( - f"litellm.router.Router::deployment_callback_on_success(): Exception occured - {e}" + "litellm.router.Router::deployment_callback_on_success(): Exception occured - %s", e ) def sync_deployment_callback_on_success( @@ -7198,7 +7210,9 @@ class Router: return True verbose_router_logger.debug( - f"Content Policy Error occurred. No available fallbacks. Returning original response. model={model}, content_policy_fallbacks={content_policy_fallbacks}" + "Content Policy Error occurred. No available fallbacks. Returning original response. model=%s, content_policy_fallbacks=%s", + model, + content_policy_fallbacks, ) return False @@ -7537,7 +7551,9 @@ class Router: ## Check if LLM Deployment is allowed for this deployment if self.deployment_is_active_for_environment(deployment=deployment) is not True: verbose_router_logger.warning( - f"Ignoring deployment {deployment.model_name} as it is not active for environment {deployment.model_info['supported_environments']}" + "Ignoring deployment %s as it is not active for environment %s", + deployment.model_name, + deployment.model_info["supported_environments"], ) return None @@ -7561,7 +7577,7 @@ class Router: except Exception as e: if self.ignore_invalid_deployments: verbose_router_logger.exception( - f"Error creating deployment: {e}, ignoring and continuing with other deployments." + "Error creating deployment: %s, ignoring and continuing with other deployments.", e ) return None else: @@ -8014,7 +8030,7 @@ class Router: _model_info=_model_info, ) - verbose_router_logger.debug(f"\nInitialized Model List {self.get_model_names()}") + verbose_router_logger.debug("\nInitialized Model List %s", self.get_model_names()) self.model_names = {m["model_name"] for m in model_list} # Note: model_name_to_deployment_indices is already built incrementally @@ -8436,8 +8452,10 @@ class Router: except Exception as e: if self.ignore_invalid_deployments: verbose_router_logger.warning( - f"Error upserting deployment {deployment.model_name} (id={deployment.model_info.id}): {e}. " - "Dropping it and continuing with other deployments." + "Error upserting deployment %s (id=%s): %s. Dropping it and continuing with other deployments.", + deployment.model_name, + deployment.model_info.id, + e, ) return None else: @@ -8710,7 +8728,7 @@ class Router: ) if not credential_values: verbose_router_logger.warning( - f"Credential '{deployment.litellm_params.litellm_credential_name}' not found in credential_list" + "Credential '%s' not found in credential_list", deployment.litellm_params.litellm_credential_name ) credentials.update(credential_values) # Remove the credential name since we've resolved it @@ -8795,7 +8813,8 @@ class Router: ## SET MODEL TO 'model=' - if base_model is None + not azure if custom_llm_provider == "azure" and base_model is None: verbose_router_logger.error( - f"Could not identify azure model '{_model}'. Set azure 'base_model' for accurate max tokens, cost tracking, etc.- https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models" + "Could not identify azure model '%s'. Set azure 'base_model' for accurate max tokens, cost tracking, etc.- https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models", + _model, ) elif custom_llm_provider != "azure": model = _model @@ -9011,7 +9030,7 @@ class Router: custom_llm_provider=litellm_params.custom_llm_provider, ) except litellm.exceptions.BadRequestError as e: - verbose_router_logger.error(f"litellm.router.py::get_model_group_info() - {e}") + verbose_router_logger.error("litellm.router.py::get_model_group_info() - %s", e) if model_info is None: supported_openai_params = litellm.get_supported_openai_params( @@ -10073,7 +10092,7 @@ class Router: relink_lar1_from_args = True setattr(self, var, value) else: - verbose_router_logger.debug(f"Setting {var} is not allowed") + verbose_router_logger.debug("Setting %s is not allowed", var) if relink_lar1_from_args and self._normalize_strategy(self.routing_strategy) == "lar1": from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy @@ -10082,7 +10101,7 @@ class Router: if rebuild_routing_groups: self._init_routing_groups(self._routing_groups_input) - verbose_router_logger.debug(f"Updated Router settings: {self.get_settings()}") + verbose_router_logger.debug("Updated Router settings: %s", self.get_settings()) def _get_client(self, deployment, kwargs, client_type=None): """ @@ -10173,7 +10192,7 @@ class Router: - [TODO] function call and model doesn't support function calling """ - verbose_router_logger.debug(f"Starting Pre-call checks for deployments in model={model}") + verbose_router_logger.debug("Starting Pre-call checks for deployments in model=%s", model) # Optimized: Use list() shallow copy instead of deepcopy # We only pop from the list, not modify deployment dicts - 100x+ faster on hot path (every request) @@ -10225,7 +10244,8 @@ class Router: ) except Exception as e: verbose_router_logger.error( - f"litellm.router.py::_pre_call_checks: failed to count tokens. Returning initial list of deployments. Got - {e}" + "litellm.router.py::_pre_call_checks: failed to count tokens. Returning initial list of deployments. Got - %s", + e, ) return _returned_deployments if input_tokens > max_input_tokens: @@ -10236,7 +10256,7 @@ class Router: ) continue except Exception as e: - verbose_router_logger.exception(f"An error occurs - {e}") + verbose_router_logger.exception("An error occurs - %s", e) model_id = _model_info.get("id", "") ## RPM CHECK ## @@ -10297,7 +10317,7 @@ class Router: for k, v in non_default_params.items(): if k not in supported_openai_params and k in special_params: # if not -> invalid model - verbose_router_logger.debug(f"INVALID MODEL INDEX @ REQUEST KWARG FILTERING, k={k}") + verbose_router_logger.debug("INVALID MODEL INDEX @ REQUEST KWARG FILTERING, k=%s", k) invalid_model_indices.add(idx) if len(invalid_model_indices) == len(_returned_deployments): @@ -10490,7 +10510,7 @@ class Router: _access_group_filter_emptied_candidates = True if verbose_router_logger.isEnabledFor(logging.DEBUG): - verbose_router_logger.debug(f"initial list of deployments: {healthy_deployments}") + verbose_router_logger.debug("initial list of deployments: %s", healthy_deployments) if len(healthy_deployments) == 0: # Check for default fallbacks if no deployments are found for the requested model @@ -10501,7 +10521,7 @@ class Router: fallback_model = self._get_first_default_fallback() if fallback_model: verbose_router_logger.info( - f"Model '{model}' not found. Attempting to use default fallback model '{fallback_model}'." + "Model '%s' not found. Attempting to use default fallback model '%s'.", model, fallback_model ) # Re-assign model to the fallback and try to get deployments again model = fallback_model @@ -10618,7 +10638,7 @@ class Router: ) if verbose_router_logger.isEnabledFor(logging.DEBUG): - verbose_router_logger.debug(f"healthy_deployments after team filter: {healthy_deployments}") + verbose_router_logger.debug("healthy_deployments after team filter: %s", healthy_deployments) healthy_deployments = filter_web_search_deployments( healthy_deployments=healthy_deployments, @@ -10626,7 +10646,7 @@ class Router: ) if verbose_router_logger.isEnabledFor(logging.DEBUG): - verbose_router_logger.debug(f"healthy_deployments after web search filter: {healthy_deployments}") + verbose_router_logger.debug("healthy_deployments after web search filter: %s", healthy_deployments) if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: @@ -10647,7 +10667,7 @@ class Router: litellm_router_instance=self, parent_otel_span=parent_otel_span ) if verbose_router_logger.isEnabledFor(logging.DEBUG): - verbose_router_logger.debug(f"cooldown deployments: {cooldown_deployments}") + verbose_router_logger.debug("cooldown deployments: %s", cooldown_deployments) _pre_cooldown_deployments = healthy_deployments healthy_deployments = self._filter_cooldown_deployments( healthy_deployments=healthy_deployments, @@ -10810,7 +10830,10 @@ class Router: ) raise exception verbose_router_logger.info( - f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment)} for model: {model}" + "get_available_deployment for model: %s, Selected deployment: %s for model: %s", + model, + self.print_deployment(deployment), + model, ) end_time = time.time() @@ -10939,7 +10962,9 @@ class Router: raise exception verbose_router_logger.info( - f"async_get_available_deployment_for_pass_through model: {model}, selected deployment: {self.print_deployment(deployment)}" + "async_get_available_deployment_for_pass_through model: %s, selected deployment: %s", + model, + self.print_deployment(deployment), ) end_time = time.perf_counter() @@ -11312,7 +11337,7 @@ class Router: ) if deployment is None: - verbose_router_logger.info(f"get_available_deployment for model: {model}, No deployment available") + verbose_router_logger.info("get_available_deployment for model: %s, No deployment available", model) model_ids = self.get_model_ids(model_name=model) _cooldown_time = self.cooldown_cache.get_min_cooldown( model_ids=model_ids, parent_otel_span=parent_otel_span @@ -11325,7 +11350,10 @@ class Router: cooldown_list=_cooldown_list, ) verbose_router_logger.info( - f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment)} for model: {model}" + "get_available_deployment for model: %s, Selected deployment: %s for model: %s", + model, + self.print_deployment(deployment), + model, ) return deployment @@ -11452,7 +11480,7 @@ class Router: if deployment is None: verbose_router_logger.info( - f"get_available_deployment_for_pass_through model: {model}, no available deployments" + "get_available_deployment_for_pass_through model: %s, no available deployments", model ) model_ids = self.get_model_ids(model_name=model) _cooldown_time = self.cooldown_cache.get_min_cooldown( @@ -11467,7 +11495,9 @@ class Router: ) verbose_router_logger.info( - f"get_available_deployment_for_pass_through model: {model}, selected deployment: {self.print_deployment(deployment)}" + "get_available_deployment_for_pass_through model: %s, selected deployment: %s", + model, + self.print_deployment(deployment), ) return deployment @@ -11485,7 +11515,7 @@ class Router: List of healthy deployments """ if verbose_router_logger.isEnabledFor(logging.DEBUG): - verbose_router_logger.debug(f"cooldown deployments: {cooldown_deployments}") + verbose_router_logger.debug("cooldown deployments: %s", cooldown_deployments) # Convert to set for O(1) lookup and use list comprehension for O(n) filtering cooldown_set = set(cooldown_deployments) return [deployment for deployment in healthy_deployments if deployment["model_info"]["id"] not in cooldown_set] @@ -11587,7 +11617,7 @@ class Router: List[Dict]: Only includes a list of deployments that support pass-through """ verbose_router_logger.debug( - f"Filter pass-through deployments from {len(healthy_deployments)} healthy deployments" + "Filter pass-through deployments from %s healthy deployments", len(healthy_deployments) ) pass_through_deployments = [ @@ -11596,7 +11626,7 @@ class Router: if deployment.get("litellm_params", {}).get("use_in_pass_through", False) ] - verbose_router_logger.debug(f"Found {len(pass_through_deployments)} deployments with pass-through enabled") + verbose_router_logger.debug("Found %s deployments with pass-through enabled", len(pass_through_deployments)) return pass_through_deployments @@ -11611,7 +11641,7 @@ class Router: if model_id is not None: self._update_usage(model_id, parent_otel_span) # update in-memory cache for tracking except Exception as e: - verbose_router_logger.error(f"Error in _track_deployment_metrics: {e}") + verbose_router_logger.error("Error in _track_deployment_metrics: %s", e) def get_num_retries_from_retry_policy(self, exception: Exception, model_group: str | None = None): return _get_num_retries_from_retry_policy( diff --git a/litellm/router_strategy/auto_router/auto_router.py b/litellm/router_strategy/auto_router/auto_router.py index d959eb3ef73..4ea3389381c 100644 --- a/litellm/router_strategy/auto_router/auto_router.py +++ b/litellm/router_strategy/auto_router/auto_router.py @@ -147,7 +147,7 @@ class AutoRouter(CustomLogger): message_content = self._extract_text_from_messages(messages) route_choice: RouteChoice | list[RouteChoice] | None = routelayer(text=message_content) - verbose_router_logger.debug(f"route_choice: {route_choice}") + verbose_router_logger.debug("route_choice: %s", route_choice) if isinstance(route_choice, RouteChoice): model = route_choice.name or self.default_model elif isinstance(route_choice, list): diff --git a/litellm/router_strategy/base_routing_strategy.py b/litellm/router_strategy/base_routing_strategy.py index 70e1c12665d..9752c96fa5b 100644 --- a/litellm/router_strategy/base_routing_strategy.py +++ b/litellm/router_strategy/base_routing_strategy.py @@ -97,7 +97,7 @@ class BaseRoutingStrategy(ABC): default_sync_interval ) # Wait for DEFAULT_REDIS_SYNC_INTERVAL seconds before next sync except Exception as e: - verbose_router_logger.error(f"Error in periodic sync task: {e}") + verbose_router_logger.error("Error in periodic sync task: %s", e) await asyncio.sleep( default_sync_interval ) # Still wait DEFAULT_REDIS_SYNC_INTERVAL seconds on error before retrying @@ -146,7 +146,7 @@ class BaseRoutingStrategy(ABC): return return_result except Exception as e: - verbose_router_logger.error(f"Error syncing in-memory cache with Redis: {e}") + verbose_router_logger.error("Error syncing in-memory cache with Redis: %s", e) self.redis_increment_operation_queue = [] def add_to_in_memory_keys_to_update(self, key: str): @@ -226,4 +226,4 @@ class BaseRoutingStrategy(ABC): await self.dual_cache.in_memory_cache.async_set_cache(key=key, value=merged) except Exception as e: - verbose_router_logger.exception(f"Error syncing in-memory cache with Redis: {e}") + verbose_router_logger.exception("Error syncing in-memory cache with Redis: %s", e) diff --git a/litellm/router_strategy/budget_limiter.py b/litellm/router_strategy/budget_limiter.py index 3b8a75f4e49..0d8980b6f16 100644 --- a/litellm/router_strategy/budget_limiter.py +++ b/litellm/router_strategy/budget_limiter.py @@ -499,7 +499,7 @@ class RouterBudgetLimiting(CustomLogger): spend_key=spend_key, response_cost=response_cost, ttl=ttl_for_increment ) - verbose_router_logger.debug(f"Incremented spend for {spend_key} by {response_cost}") + verbose_router_logger.debug("Incremented spend for %s by %s", spend_key, response_cost) async def periodic_sync_in_memory_spend_with_redis(self): """ @@ -514,7 +514,7 @@ class RouterBudgetLimiting(CustomLogger): DEFAULT_REDIS_SYNC_INTERVAL ) # Wait for DEFAULT_REDIS_SYNC_INTERVAL seconds before next sync except Exception as e: - verbose_router_logger.error(f"Error in periodic sync task: {e}") + verbose_router_logger.error("Error in periodic sync task: %s", e) await asyncio.sleep( DEFAULT_REDIS_SYNC_INTERVAL ) # Still wait DEFAULT_REDIS_SYNC_INTERVAL seconds on error before retrying @@ -545,7 +545,7 @@ class RouterBudgetLimiting(CustomLogger): self.redis_increment_operation_queue = [] except Exception as e: - verbose_router_logger.error(f"Error syncing in-memory cache with Redis: {e}") + verbose_router_logger.error("Error syncing in-memory cache with Redis: %s", e) async def _sync_in_memory_spend_with_redis(self): """ @@ -597,10 +597,10 @@ class RouterBudgetLimiting(CustomLogger): for key, value in redis_values.items(): if value is not None: await self.dual_cache.in_memory_cache.async_set_cache(key=key, value=float(value)) - verbose_router_logger.debug(f"Updated in-memory cache for {key}: {value}") + verbose_router_logger.debug("Updated in-memory cache for %s: %s", key, value) except Exception as e: - verbose_router_logger.error(f"Error syncing in-memory cache with Redis: {e}") + verbose_router_logger.error("Error syncing in-memory cache with Redis: %s", e) def _get_budget_config_for_deployment( self, @@ -639,7 +639,7 @@ class RouterBudgetLimiting(CustomLogger): litellm_params=provider_resolution_params, ) except Exception: - verbose_router_logger.error(f"Error getting LLM provider for deployment: {deployment}") + verbose_router_logger.error("Error getting LLM provider for deployment: %s", deployment) return None return custom_llm_provider @@ -772,7 +772,7 @@ class RouterBudgetLimiting(CustomLogger): ) ) - verbose_router_logger.debug(f"Initalized Provider budget config: {self.provider_budget_config}") + verbose_router_logger.debug("Initalized Provider budget config: %s", self.provider_budget_config) def _init_deployment_budgets( self, @@ -788,7 +788,10 @@ class RouterBudgetLimiting(CustomLogger): _budget_duration = _litellm_params.get("budget_duration") verbose_router_logger.debug( - f"Init Deployment Budget: max_budget: {_max_budget}, budget_duration: {_budget_duration}, model_id: {_model_id}" + "Init Deployment Budget: max_budget: %s, budget_duration: %s, model_id: %s", + _max_budget, + _budget_duration, + _model_id, ) if _max_budget is not None and _budget_duration is not None and _model_id is not None: _budget_config = GenericBudgetInfo( @@ -799,7 +802,7 @@ class RouterBudgetLimiting(CustomLogger): self.deployment_budget_config = {} self.deployment_budget_config[_model_id] = _budget_config - verbose_router_logger.debug(f"Initialized Deployment Budget Config: {self.deployment_budget_config}") + verbose_router_logger.debug("Initialized Deployment Budget Config: %s", self.deployment_budget_config) def register_deployment_budget( self, @@ -837,4 +840,4 @@ class RouterBudgetLimiting(CustomLogger): ) self.tag_budget_config[_tag] = _generic_budget_config - verbose_router_logger.debug(f"Initialized Tag Budget Config: {self.tag_budget_config}") + verbose_router_logger.debug("Initialized Tag Budget Config: %s", self.tag_budget_config) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 11a1b686c2e..1d8176d1534 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -456,7 +456,7 @@ class ComplexityRouter(CustomLogger): self._model_tiers: dict[str, tuple[ComplexityTier, ...]] = {} self._adaptive_init_attempted = False - verbose_router_logger.debug(f"ComplexityRouter initialized for {model_name} with tiers: {self.config.tiers}") + verbose_router_logger.debug("ComplexityRouter initialized for %s with tiers: %s", model_name, self.config.tiers) def _estimate_tokens(self, text: str) -> int: """ @@ -746,7 +746,7 @@ class ComplexityRouter(CustomLogger): ) except Exception as e: # noqa: BLE001 -- external LLM call can fail in many distinct ways (timeout, provider error, validation, parse error); any failure must fall back to the heuristic scorer verbose_router_logger.warning( - f"ComplexityRouter: LLM classifier failed ({e}), falling back to heuristic scoring" + "ComplexityRouter: LLM classifier failed (%s), falling back to heuristic scoring", e ) tier, score, signals, cause = self._score_and_classify(prompt, system_prompt) return ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause) @@ -1324,7 +1324,7 @@ class ComplexityRouter(CustomLogger): semantic_tier = await self._semantic_tier_override(user_message, request_kwargs) except Exception as e: # noqa: BLE001 -- embedding call can fail many ways (timeout, provider/network/parse error); any failure must fall back to scoring, never fail the request verbose_router_logger.warning( - f"ComplexityRouter: semantic keyword matching failed ({e}), falling back to complexity scoring" + "ComplexityRouter: semantic keyword matching failed (%s), falling back to complexity scoring", e ) return None if semantic_tier is None: @@ -1468,7 +1468,7 @@ class ComplexityRouter(CustomLogger): escalated = routed_model != pinned_model cause: RoutingDecisionCause = "session_affinity_escalation" if escalated else "session_affinity_pin" verbose_router_logger.info( - f"ComplexityRouter: routing decision cause={cause}, routed_model={routed_model}" + "ComplexityRouter: routing decision cause=%s, routed_model=%s", cause, routed_model ) has_original_messages = messages is not None and len(messages) > 0 return PreRoutingHookResponse( @@ -1578,8 +1578,11 @@ class ComplexityRouter(CustomLogger): "semantic_keyword_match" if self.config.semantic_keyword_matching else "literal_keyword_match" ) verbose_router_logger.info( - f"ComplexityRouter: routing decision cause={keyword_cause}, escalated={keyword_escalated}, " - f"tier={routed_tier.value}, routed_model={routed_model}" + "ComplexityRouter: routing decision cause=%s, escalated=%s, tier=%s, routed_model=%s", + keyword_cause, + keyword_escalated, + routed_tier.value, + routed_model, ) return PreRoutingHookResponse( model=routed_model, @@ -1613,15 +1616,22 @@ class ComplexityRouter(CustomLogger): chosen_key = getattr(self, "_adaptive_chosen_model_key", "adaptive_router_chosen_model") kwargs_metadata[chosen_key] = routed_model verbose_router_logger.info( - f"ComplexityRouter[adaptive]: routing decision cause={outcome.cause}, " - f"tier={tier.value}, score={score_repr}, " - f"signals={signals}, routed_model={routed_model}" + "ComplexityRouter[adaptive]: routing decision cause=%s, tier=%s, score=%s, signals=%s, routed_model=%s", + outcome.cause, + tier.value, + score_repr, + signals, + routed_model, ) else: routed_model = await self._pick_model_for_tier(tier, messages, resolved_messages, request_kwargs) verbose_router_logger.info( - f"ComplexityRouter: routing decision cause={outcome.cause}, tier={tier.value}, " - f"score={score_repr}, signals={signals}, routed_model={routed_model}" + "ComplexityRouter: routing decision cause=%s, tier=%s, score=%s, signals=%s, routed_model=%s", + outcome.cause, + tier.value, + score_repr, + signals, + routed_model, ) classifier_model = ( diff --git a/litellm/router_strategy/lar1_routing.py b/litellm/router_strategy/lar1_routing.py index acd7ac63225..99a26926558 100644 --- a/litellm/router_strategy/lar1_routing.py +++ b/litellm/router_strategy/lar1_routing.py @@ -68,12 +68,12 @@ def _normalize_thresholds(thresholds: dict[str, float] | None) -> dict[str, floa def _parse_lar1_metadata(request_kwargs: dict) -> LAR1Metadata: lar1_raw = request_kwargs.get("metadata", {}).get("lar1", {}) if not isinstance(lar1_raw, dict): - verbose_router_logger.warning(f"[LAR-1] Invalid lar1 metadata type: {type(lar1_raw).__name__}. Using defaults") + verbose_router_logger.warning("[LAR-1] Invalid lar1 metadata type: %s. Using defaults", type(lar1_raw).__name__) return LAR1Metadata() try: return LAR1Metadata.model_validate(lar1_raw) except ValidationError as exc: - verbose_router_logger.warning(f"[LAR-1] Invalid lar1 metadata: {exc}. Using defaults") + verbose_router_logger.warning("[LAR-1] Invalid lar1 metadata: %s. Using defaults", exc) return LAR1Metadata() @@ -123,11 +123,11 @@ class LAR1RoutingStrategy(CustomRoutingStrategyBase): if selected is None: return None if exact_match: - verbose_router_logger.info(f"[LAR-1] confidence={confidence} -> {target}") + verbose_router_logger.info("[LAR-1] confidence=%s -> %s", confidence, target) else: actual_type = selected.get("model_info", {}).get("type", "unknown") verbose_router_logger.warning( - f"[LAR-1] No deployment for type '{target}', fallback to deployment type '{actual_type}'" + "[LAR-1] No deployment for type '%s', fallback to deployment type '%s'", target, actual_type ) return selected diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index ba7d32c42ad..7a2970d053a 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -91,7 +91,7 @@ class LowestCostLoggingHandler(CustomLogger): self.logged_success += 1 except Exception as e: verbose_logger.exception( - f"litellm.router_strategy.lowest_cost.py::log_success_event(): Exception occured - {e}" + "litellm.router_strategy.lowest_cost.py::log_success_event(): Exception occured - %s", e ) async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -170,7 +170,7 @@ class LowestCostLoggingHandler(CustomLogger): self.logged_success += 1 except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {e}" + "litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - %s", e ) async def async_get_available_deployments( @@ -269,7 +269,11 @@ class LowestCostLoggingHandler(CustomLogger): item_tpm = item_map.get(precise_minute, {}).get("tpm", 0) verbose_router_logger.debug( - f"item_cost: {item_cost}, item_tpm: {item_tpm}, item_rpm: {item_rpm}, model_id: {_deployment.get('model_info', {}).get('id')}" + "item_cost: %s, item_tpm: %s, item_rpm: %s, model_id: %s", + item_cost, + item_tpm, + item_rpm, + _deployment.get("model_info", {}).get("id"), ) # -------------- # diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index 0adcdebcbf2..294409d042c 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -160,7 +160,7 @@ class LowestLatencyLoggingHandler(CustomLogger): self.logged_success += 1 except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {e}" + "litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - %s", e ) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): @@ -217,7 +217,7 @@ class LowestLatencyLoggingHandler(CustomLogger): return except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {e}" + "litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - %s", e ) async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -350,7 +350,7 @@ class LowestLatencyLoggingHandler(CustomLogger): self.logged_success += 1 except Exception as e: verbose_logger.exception( - f"litellm.router_strategy.lowest_latency.py::async_log_success_event(): Exception occured - {e}" + "litellm.router_strategy.lowest_latency.py::async_log_success_event(): Exception occured - %s", e ) def _get_available_deployments( diff --git a/litellm/router_strategy/lowest_tpm_rpm.py b/litellm/router_strategy/lowest_tpm_rpm.py index f8e7e93eb54..7375658f982 100644 --- a/litellm/router_strategy/lowest_tpm_rpm.py +++ b/litellm/router_strategy/lowest_tpm_rpm.py @@ -73,7 +73,7 @@ class LowestTPMLoggingHandler(CustomLogger): self.logged_success += 1 except Exception as e: verbose_router_logger.error( - f"litellm.router_strategy.lowest_tpm_rpm.py::async_log_success_event(): Exception occured - {e}" + "litellm.router_strategy.lowest_tpm_rpm.py::async_log_success_event(): Exception occured - %s", e ) verbose_router_logger.debug(traceback.format_exc()) @@ -135,7 +135,7 @@ class LowestTPMLoggingHandler(CustomLogger): self.logged_success += 1 except Exception as e: verbose_router_logger.exception( - f"litellm.router_strategy.lowest_tpm_rpm.py::async_log_success_event(): Exception occured - {e}" + "litellm.router_strategy.lowest_tpm_rpm.py::async_log_success_event(): Exception occured - %s", e ) verbose_router_logger.debug(traceback.format_exc()) @@ -151,7 +151,9 @@ class LowestTPMLoggingHandler(CustomLogger): """ # get list of potential deployments verbose_router_logger.debug( - f"get_available_deployments - Usage Based. model_group: {model_group}, healthy_deployments: {healthy_deployments}" + "get_available_deployments - Usage Based. model_group: %s, healthy_deployments: %s", + model_group, + healthy_deployments, ) current_minute = datetime.now().strftime("%H-%M") tpm_key = f"{model_group}:tpm:{current_minute}" @@ -160,12 +162,12 @@ class LowestTPMLoggingHandler(CustomLogger): tpm_dict = self.router_cache.get_cache(key=tpm_key) rpm_dict = self.router_cache.get_cache(key=rpm_key) - verbose_router_logger.debug(f"tpm_key={tpm_key}, tpm_dict: {tpm_dict}, rpm_dict: {rpm_dict}") + verbose_router_logger.debug("tpm_key=%s, tpm_dict: %s, rpm_dict: %s", tpm_key, tpm_dict, rpm_dict) try: input_tokens = token_counter(messages=messages, text=input) except Exception: input_tokens = 0 - verbose_router_logger.debug(f"input_tokens={input_tokens}") + verbose_router_logger.debug("input_tokens=%s", input_tokens) # ----------------------- # Find lowest used model # ---------------------- diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py index a81428fd5fa..1d4703f6026 100644 --- a/litellm/router_strategy/lowest_tpm_rpm_v2.py +++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py @@ -245,7 +245,7 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): self.logged_success += 1 except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.lowest_tpm_rpm_v2.py::log_success_event(): Exception occured - {e}" + "litellm.proxy.hooks.lowest_tpm_rpm_v2.py::log_success_event(): Exception occured - %s", e ) async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -289,7 +289,7 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): self.logged_success += 1 except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.lowest_tpm_rpm_v2.py::async_log_success_event(): Exception occured - {e}" + "litellm.proxy.hooks.lowest_tpm_rpm_v2.py::async_log_success_event(): Exception occured - %s", e ) def _return_potential_deployments( @@ -375,7 +375,7 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): input_tokens = token_counter(messages=messages, text=input) except Exception: input_tokens = 0 - verbose_router_logger.debug(f"input_tokens={input_tokens}") + verbose_router_logger.debug("input_tokens=%s", input_tokens) # ----------------------- # Find lowest used model # ---------------------- @@ -420,7 +420,9 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): """ # get list of potential deployments verbose_router_logger.debug( - f"get_available_deployments - Usage Based. model_group: {model_group}, healthy_deployments: {healthy_deployments}" + "get_available_deployments - Usage Based. model_group: %s, healthy_deployments: %s", + model_group, + healthy_deployments, ) dt = get_utc_datetime() @@ -535,7 +537,9 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): """ # get list of potential deployments verbose_router_logger.debug( - f"get_available_deployments - Usage Based. model_group: {model_group}, healthy_deployments: {healthy_deployments}" + "get_available_deployments - Usage Based. model_group: %s, healthy_deployments: %s", + model_group, + healthy_deployments, ) dt = get_utc_datetime() diff --git a/litellm/router_strategy/quality_router/quality_router.py b/litellm/router_strategy/quality_router/quality_router.py index da6825a5741..a84ee70864a 100644 --- a/litellm/router_strategy/quality_router/quality_router.py +++ b/litellm/router_strategy/quality_router/quality_router.py @@ -85,9 +85,10 @@ class QualityRouter(CustomLogger): self._tier_to_models_cache: dict[int, list[str]] | None = None verbose_router_logger.debug( - f"QualityRouter initialized for {model_name} with " - f"available_models={self.config.available_models}, " - f"default_model={self.config.default_model}" + "QualityRouter initialized for %s with available_models=%s, default_model=%s", + model_name, + self.config.available_models, + self.config.default_model, ) @property @@ -371,10 +372,11 @@ class QualityRouter(CustomLogger): if keyword_match is not None: routed_model, matched_keyword = keyword_match verbose_router_logger.info( - f"QualityRouter: keyword override matched='{matched_keyword}' " - f"routed_model={routed_model} " - f"(quality_tier={self._model_quality.get(routed_model)}, " - f"input_cost_per_token={self._model_cost.get(routed_model)})" + "QualityRouter: keyword override matched='%s' routed_model=%s (quality_tier=%s, input_cost_per_token=%s)", + matched_keyword, + routed_model, + self._model_quality.get(routed_model), + self._model_cost.get(routed_model), ) self._stash_decision( request_kwargs, diff --git a/litellm/router_strategy/simple_shuffle.py b/litellm/router_strategy/simple_shuffle.py index ab2fab09a0d..1a2abac4b05 100644 --- a/litellm/router_strategy/simple_shuffle.py +++ b/litellm/router_strategy/simple_shuffle.py @@ -44,7 +44,7 @@ def simple_shuffle( weight = healthy_deployments[0].get("litellm_params").get(weight_by, None) if weight is not None: weights = [m["litellm_params"].get(weight_by, 0) for m in healthy_deployments] - verbose_router_logger.debug(f"\nweight {weights}") + verbose_router_logger.debug("\nweight %s", weights) total_weight = sum(weights) if total_weight <= 0: # All remaining candidates have weight 0 for this metric (e.g. @@ -54,13 +54,16 @@ def simple_shuffle( # through to the uniform random pick at the end. continue weights = [weight / total_weight for weight in weights] - verbose_router_logger.debug(f"\n weights {weights} by {weight_by}") + verbose_router_logger.debug("\n weights %s by %s", weights, weight_by) # Perform weighted random pick selected_index = random.choices(range(len(weights)), weights=weights)[0] - verbose_router_logger.debug(f"\n selected index, {selected_index}") + verbose_router_logger.debug("\n selected index, %s", selected_index) deployment = healthy_deployments[selected_index] verbose_router_logger.info( - f"get_available_deployment for model: {model}, Selected deployment: {llm_router_instance.print_deployment(deployment) or deployment[0]} for model: {model}" + "get_available_deployment for model: %s, Selected deployment: %s for model: %s", + model, + llm_router_instance.print_deployment(deployment) or deployment[0], + model, ) return deployment or deployment[0] diff --git a/litellm/router_utils/batch_utils.py b/litellm/router_utils/batch_utils.py index eefacb7c9e6..c968e8dfde8 100644 --- a/litellm/router_utils/batch_utils.py +++ b/litellm/router_utils/batch_utils.py @@ -52,7 +52,7 @@ def parse_jsonl_with_embedded_newlines(content: str) -> list[dict]: json_object = json.loads(buffer.strip()) json_objects.append(json_object) except json.JSONDecodeError as e: - verbose_logger.error(f"error parsing final buffer: {buffer[:100]}..., error: {e}") + verbose_logger.error("error parsing final buffer: %s..., error: %s", buffer[:100], e) raise e return json_objects @@ -128,7 +128,7 @@ def replace_model_in_jsonl(file_content: FileTypes, new_model_name: str) -> File # that followed it). Returning the partial `output` would silently # drop those rows; return the unchanged original so the provider # rejects the batch loudly instead of accepting a truncated one. - verbose_logger.error(f"error parsing trailing batch content: {buffer[:100]}...") + verbose_logger.error("error parsing trailing batch content: %s...", buffer[:100]) if hasattr(source, "seek"): try: source.seek(0) # type: ignore[attr-defined] diff --git a/litellm/router_utils/cooldown_cache.py b/litellm/router_utils/cooldown_cache.py index 4e9a11a4bfd..4681609b9d7 100644 --- a/litellm/router_utils/cooldown_cache.py +++ b/litellm/router_utils/cooldown_cache.py @@ -58,7 +58,7 @@ class CooldownCache: return cooldown_key, cooldown_data except Exception as e: - verbose_logger.error(f"CooldownCache::_common_add_cooldown_logic - Exception occurred - {e}") + verbose_logger.error("CooldownCache::_common_add_cooldown_logic - Exception occurred - %s", e) raise e def add_deployment_to_cooldown( @@ -92,7 +92,7 @@ class CooldownCache: ttl=_cooldown_time, ) except Exception as e: - verbose_logger.error(f"CooldownCache::add_deployment_to_cooldown - Exception occurred - {e}") + verbose_logger.error("CooldownCache::add_deployment_to_cooldown - Exception occurred - %s", e) raise e @staticmethod diff --git a/litellm/router_utils/cooldown_callbacks.py b/litellm/router_utils/cooldown_callbacks.py index acd1c5b47ad..94e2847121b 100644 --- a/litellm/router_utils/cooldown_callbacks.py +++ b/litellm/router_utils/cooldown_callbacks.py @@ -34,7 +34,8 @@ async def router_cooldown_event_callback( _deployment = litellm_router_instance.get_deployment(model_id=deployment_id) if _deployment is None: verbose_logger.warning( - f"in router_cooldown_event_callback but _deployment is None for deployment_id={deployment_id}. Doing nothing" + "in router_cooldown_event_callback but _deployment is None for deployment_id=%s. Doing nothing", + deployment_id, ) return _litellm_params = _deployment["litellm_params"] diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 380689e653b..ca5fe198abb 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -281,7 +281,7 @@ def _set_cooldown_deployments( return False exception_status_int = cast_exception_status_to_int(exception_status) - verbose_router_logger.debug(f"Attempting to add {deployment} to cooldown list") + verbose_router_logger.debug("Attempting to add %s to cooldown list", deployment) if _should_cooldown_deployment( litellm_router_instance=litellm_router_instance, @@ -331,7 +331,7 @@ async def _async_get_cooldown_deployments( ): cached_value_deployment_ids = [cv[0] for cv in cooldown_models] - verbose_router_logger.debug(f"retrieve cooldown models: {cooldown_models}") + verbose_router_logger.debug("retrieve cooldown models: %s", cooldown_models) return cached_value_deployment_ids @@ -347,7 +347,7 @@ async def _async_get_cooldown_deployments_with_debug_info( model_ids=model_ids, parent_otel_span=parent_otel_span ) - verbose_router_logger.debug(f"retrieve cooldown models: {cooldown_models}") + verbose_router_logger.debug("retrieve cooldown models: %s", cooldown_models) return cooldown_models @@ -432,7 +432,7 @@ def cast_exception_status_to_int(exception_status: str | int) -> int: exception_status = int(exception_status) except Exception: verbose_router_logger.debug( - f"Unable to cast exception status to int {exception_status}. Defaulting to status=500." + "Unable to cast exception status to int %s. Defaulting to status=500.", exception_status ) exception_status = 500 return exception_status diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 3fad860fa7d..e2fcd9109d2 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -127,7 +127,7 @@ async def run_async_fallback( try: # LOGGING kwargs = litellm_router.log_retry(kwargs=kwargs, e=original_exception) - verbose_router_logger.info(f"Falling back to model_group = {mask_sensitive_structure(mg)}") + verbose_router_logger.info("Falling back to model_group = %s", mask_sensitive_structure(mg)) if isinstance(mg, str): kwargs["model"] = mg elif isinstance(mg, dict): @@ -190,7 +190,7 @@ async def log_success_fallback_event(original_model_group: str, kwargs: dict, or original_exception=original_exception, ) except Exception as e: - verbose_router_logger.error(f"Error in log_success_fallback_event: {e}") + verbose_router_logger.error("Error in log_success_fallback_event: %s", e) async def log_failure_fallback_event(original_model_group: str, kwargs: dict, original_exception: Exception): @@ -218,7 +218,7 @@ async def log_failure_fallback_event(original_model_group: str, kwargs: dict, or original_exception=original_exception, ) except Exception as e: - verbose_router_logger.error(f"Error in log_failure_fallback_event: {e}") + verbose_router_logger.error("Error in log_failure_fallback_event: %s", e) def _check_non_standard_fallback_format(fallbacks: list[Any] | None) -> bool: diff --git a/litellm/router_utils/handle_error.py b/litellm/router_utils/handle_error.py index b38d6605ed2..c25238a89b3 100644 --- a/litellm/router_utils/handle_error.py +++ b/litellm/router_utils/handle_error.py @@ -74,7 +74,7 @@ async def async_raise_no_deployment_exception( """ Raises a RouterRateLimitError if no deployment is found for the given model. """ - verbose_router_logger.info(f"get_available_deployment for model: {model}, No deployment available") + verbose_router_logger.info("get_available_deployment for model: %s, No deployment available", model) model_ids = litellm_router_instance.get_model_ids(model_name=model) _cooldown_time = litellm_router_instance.cooldown_cache.get_min_cooldown( model_ids=model_ids, parent_otel_span=parent_otel_span @@ -84,7 +84,7 @@ async def async_raise_no_deployment_exception( parent_otel_span=parent_otel_span, ) verbose_router_logger.info( - f"No deployment found for model: {model}, cooldown_list with debug info: {_cooldown_list}" + "No deployment found for model: %s, cooldown_list with debug info: %s", model, _cooldown_list ) cooldown_list_ids = [cooldown_model[0] for cooldown_model in (_cooldown_list or [])] diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index 42704cea826..c8161529626 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -150,7 +150,7 @@ class PatternMatchRouter: matched_pattern=pattern_match, deployments=llm_deployments ) except Exception as e: - verbose_router_logger.debug(f"Error in PatternMatchRouter.route: {e}") + verbose_router_logger.debug("Error in PatternMatchRouter.route: %s", e) return None # No matching pattern found diff --git a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py index 9561eafa900..4dd6944d791 100644 --- a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py @@ -551,16 +551,20 @@ def io_token_reconcile_success( ) else: verbose_router_logger.debug( - "[IO TOKEN LIMIT] usage missing; keeping reservation " - f"(itpm_reserved={itpm_reserved}, otpm_reserved={otpm_reserved})" + "[IO TOKEN LIMIT] usage missing; keeping reservation (itpm_reserved=%s, otpm_reserved=%s)", + itpm_reserved, + otpm_reserved, ) finally: _clear_reservation_from_kwargs(kwargs) verbose_router_logger.debug( - f"[IO TOKEN LIMIT] reconciled " - f"(usage_resolved={usage_resolved}, itpm_reserved={itpm_reserved}, " - f"billable_input={billable_input}, otpm_reserved={otpm_reserved}, output={completion_tokens})" + "[IO TOKEN LIMIT] reconciled (usage_resolved=%s, itpm_reserved=%s, billable_input=%s, otpm_reserved=%s, output=%s)", + usage_resolved, + itpm_reserved, + billable_input, + otpm_reserved, + completion_tokens, ) @@ -606,16 +610,20 @@ async def async_io_token_reconcile_success( ) else: verbose_router_logger.debug( - "[IO TOKEN LIMIT] usage missing; keeping reservation " - f"(itpm_reserved={itpm_reserved}, otpm_reserved={otpm_reserved})" + "[IO TOKEN LIMIT] usage missing; keeping reservation (itpm_reserved=%s, otpm_reserved=%s)", + itpm_reserved, + otpm_reserved, ) finally: _clear_reservation_from_kwargs(kwargs) verbose_router_logger.debug( - f"[IO TOKEN LIMIT] reconciled " - f"(usage_resolved={usage_resolved}, itpm_reserved={itpm_reserved}, " - f"billable_input={billable_input}, otpm_reserved={otpm_reserved}, output={completion_tokens})" + "[IO TOKEN LIMIT] reconciled (usage_resolved=%s, itpm_reserved=%s, billable_input=%s, otpm_reserved=%s, output=%s)", + usage_resolved, + itpm_reserved, + billable_input, + otpm_reserved, + completion_tokens, ) @@ -639,7 +647,7 @@ def io_token_refund_failure( ttl=RoutingArgsTTL, ) _clear_reservation_from_kwargs(kwargs) - verbose_router_logger.debug(f"[IO TOKEN LIMIT] refunded ITPM={itpm_reserved} OTPM={otpm_reserved}") + verbose_router_logger.debug("[IO TOKEN LIMIT] refunded ITPM=%s OTPM=%s", itpm_reserved, otpm_reserved) def refund_stale_reservation_before_retry(dual_cache: DualCache, kwargs: dict[str, Any] | None) -> None: @@ -693,7 +701,7 @@ async def async_io_token_refund_failure( parent_otel_span=parent_otel_span, ) _clear_reservation_from_kwargs(kwargs) - verbose_router_logger.debug(f"[IO TOKEN LIMIT] refunded ITPM={itpm_reserved} OTPM={otpm_reserved}") + verbose_router_logger.debug("[IO TOKEN LIMIT] refunded ITPM=%s OTPM=%s", itpm_reserved, otpm_reserved) def build_io_token_rate_limit_headers( diff --git a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py index da8b452fa8a..416b53f3267 100644 --- a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py @@ -75,8 +75,8 @@ class ModelRateLimitingCheck(CustomLogger): return self._io_token_conflict_warned_ids.add(str(model_id)) verbose_router_logger.warning( - f"Deployment '{model_id}' configures itpm/otpm alongside tpm/rpm; " - "both limit types are enforced on this deployment" + "Deployment '%s' configures itpm/otpm alongside tpm/rpm; both limit types are enforced on this deployment", + model_id, ) def _refund_io_token_reservation_if_any(self) -> None: @@ -212,7 +212,7 @@ class ModelRateLimitingCheck(CustomLogger): self._refund_io_token_reservation_if_any() raise except Exception as e: - verbose_router_logger.debug(f"Error in ModelRateLimitingCheck.pre_call_check: {e}") + verbose_router_logger.debug("Error in ModelRateLimitingCheck.pre_call_check: %s", e) # Don't fail the request if rate limit check fails return deployment @@ -300,7 +300,7 @@ class ModelRateLimitingCheck(CustomLogger): await self._async_refund_io_token_reservation_if_any(parent_otel_span=parent_otel_span) raise except Exception as e: - verbose_router_logger.debug(f"Error in ModelRateLimitingCheck.async_pre_call_check: {e}") + verbose_router_logger.debug("Error in ModelRateLimitingCheck.async_pre_call_check: %s", e) # Don't fail the request if rate limit check fails return deployment @@ -341,7 +341,7 @@ class ModelRateLimitingCheck(CustomLogger): model = standard_logging_object.get("hidden_params", {}).get("litellm_model_name") verbose_router_logger.debug( - f"[TPM TRACKING] model_id={model_id}, total_tokens={total_tokens}, model={model}" + "[TPM TRACKING] model_id=%s, total_tokens=%s, model=%s", model_id, total_tokens, model ) if not model or not total_tokens: @@ -351,7 +351,7 @@ class ModelRateLimitingCheck(CustomLogger): current_minute = dt.strftime("%H-%M") tpm_key = f"{model_id}:{model}:tpm:{current_minute}" - verbose_router_logger.debug(f"[TPM TRACKING] Incrementing {tpm_key} by {total_tokens}") + verbose_router_logger.debug("[TPM TRACKING] Incrementing %s by %s", tpm_key, total_tokens) await self.dual_cache.async_increment_cache( key=tpm_key, @@ -360,7 +360,7 @@ class ModelRateLimitingCheck(CustomLogger): ) except Exception as e: - verbose_router_logger.debug(f"Error in ModelRateLimitingCheck.async_log_success_event: {e}") + verbose_router_logger.debug("Error in ModelRateLimitingCheck.async_log_success_event: %s", e) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): from litellm.litellm_core_utils.core_helpers import ( @@ -418,7 +418,7 @@ class ModelRateLimitingCheck(CustomLogger): ) except Exception as e: - verbose_router_logger.debug(f"Error in ModelRateLimitingCheck.log_success_event: {e}") + verbose_router_logger.debug("Error in ModelRateLimitingCheck.log_success_event: %s", e) def log_failure_event(self, kwargs, response_obj, start_time, end_time): with contextlib.suppress(Exception): diff --git a/litellm/router_utils/search_api_router.py b/litellm/router_utils/search_api_router.py index 0ce0d4229c1..5708b772970 100644 --- a/litellm/router_utils/search_api_router.py +++ b/litellm/router_utils/search_api_router.py @@ -57,7 +57,7 @@ class SearchAPIRouter: try: from litellm.types.router import SearchToolTypedDict - verbose_router_logger.debug(f"Adding {len(search_tools)} search tools to router") + verbose_router_logger.debug("Adding %s search tools to router", len(search_tools)) # Convert search tools to the format expected by the router router_search_tools: list = [] @@ -74,10 +74,10 @@ class SearchAPIRouter: # Update the router's search_tools list router_instance.search_tools = router_search_tools - verbose_router_logger.info(f"Successfully updated router with {len(router_search_tools)} search tool(s)") + verbose_router_logger.info("Successfully updated router with %s search tool(s)", len(router_search_tools)) except Exception as e: - verbose_router_logger.exception(f"Error updating router with search tools: {e}") + verbose_router_logger.exception("Error updating router with search tools: %s", e) raise e @staticmethod @@ -149,7 +149,10 @@ class SearchAPIRouter: available_search_tool_names = [tool.get("search_tool_name") for tool in router_instance.search_tools] verbose_router_logger.debug( - f"Inside SearchAPIRouter.async_search_with_fallbacks() - search_tool_name: {search_tool_name}, Available Search Tools: {available_search_tool_names}, kwargs: {kwargs}" + "Inside SearchAPIRouter.async_search_with_fallbacks() - search_tool_name: %s, Available Search Tools: %s, kwargs: %s", + search_tool_name, + available_search_tool_names, + kwargs, ) # Use the existing retry/fallback infrastructure @@ -212,7 +215,7 @@ class SearchAPIRouter: tool_litellm_params=litellm_params, ) - verbose_router_logger.debug(f"Selected search tool with provider: {search_provider}") + verbose_router_logger.debug("Selected search tool with provider: %s", search_provider) # Call the original search function with the provider config response = await original_generic_function( @@ -226,6 +229,6 @@ class SearchAPIRouter: except Exception as e: verbose_router_logger.error( - f"Error in SearchAPIRouter.async_search_with_fallbacks_helper for {search_tool_name}: {e}" + "Error in SearchAPIRouter.async_search_with_fallbacks_helper for %s: %s", search_tool_name, e ) raise e diff --git a/litellm/sandbox/main.py b/litellm/sandbox/main.py index eae7e5c097a..3210e327fef 100644 --- a/litellm/sandbox/main.py +++ b/litellm/sandbox/main.py @@ -141,4 +141,4 @@ async def acode_interpreter_tool( try: await config.adelete_sandbox(container=container, api_key=api_key, api_base=api_base, **forwarded) except Exception as e: - litellm._logging.verbose_logger.debug(f"sandbox: failed to delete ephemeral container: {e}") + litellm._logging.verbose_logger.debug("sandbox: failed to delete ephemeral container: %s", e) diff --git a/litellm/search/main.py b/litellm/search/main.py index 932a73c0955..af9b1f7a745 100644 --- a/litellm/search/main.py +++ b/litellm/search/main.py @@ -247,7 +247,7 @@ def search( if search_provider_config is None: raise ValueError(f"Search is not supported for provider: {search_provider}") - verbose_logger.debug(f"Search call - provider: {search_provider}") + verbose_logger.debug("Search call - provider: %s", search_provider) # Build optional_params from explicit parameters optional_params = _build_search_optional_params( @@ -265,7 +265,7 @@ def search( if key not in optional_params: optional_params[key] = value - verbose_logger.debug(f"Search optional_params: {optional_params}") + verbose_logger.debug("Search optional_params: %s", optional_params) # Validate environment and get headers headers = search_provider_config.validate_environment( diff --git a/litellm/secret_managers/cyberark_secret_manager.py b/litellm/secret_managers/cyberark_secret_manager.py index 6e7eb742088..0e8aa2a5b3a 100644 --- a/litellm/secret_managers/cyberark_secret_manager.py +++ b/litellm/secret_managers/cyberark_secret_manager.py @@ -143,17 +143,17 @@ class CyberArkSecretManager(BaseSecretManager): content=policy_yaml, ) resp.raise_for_status() - verbose_logger.debug(f"Created policy entry for variable: {secret_name}") + verbose_logger.debug("Created policy entry for variable: %s", secret_name) except httpx.HTTPStatusError as e: # Variable might already exist, which is fine if e.response.status_code in [409, 422]: - verbose_logger.debug(f"Variable {secret_name} already exists or policy conflict (expected)") + verbose_logger.debug("Variable %s already exists or policy conflict (expected)", secret_name) else: verbose_logger.warning( - f"Could not ensure variable exists: {e.response.status_code} - {e.response.text}" + "Could not ensure variable exists: %s - %s", e.response.status_code, e.response.text ) except Exception as e: - verbose_logger.warning(f"Error ensuring variable exists: {e}") + verbose_logger.warning("Error ensuring variable exists: %s", e) def get_url(self, secret_name: str) -> str: """ @@ -207,12 +207,12 @@ class CyberArkSecretManager(BaseSecretManager): except httpx.HTTPStatusError as e: if e.response.status_code == 404: - verbose_logger.debug(f"Secret {secret_name} not found in CyberArk Conjur") + verbose_logger.debug("Secret %s not found in CyberArk Conjur", secret_name) else: - verbose_logger.exception(f"Error reading secret from CyberArk Conjur: {e}") + verbose_logger.exception("Error reading secret from CyberArk Conjur: %s", e) return None except Exception as e: - verbose_logger.exception(f"Error reading secret from CyberArk Conjur: {e}") + verbose_logger.exception("Error reading secret from CyberArk Conjur: %s", e) return None def sync_read_secret( @@ -250,12 +250,12 @@ class CyberArkSecretManager(BaseSecretManager): except httpx.HTTPStatusError as e: if e.response.status_code == 404: - verbose_logger.debug(f"Secret {secret_name} not found in CyberArk Conjur") + verbose_logger.debug("Secret %s not found in CyberArk Conjur", secret_name) else: - verbose_logger.exception(f"Error reading secret from CyberArk Conjur: {e}") + verbose_logger.exception("Error reading secret from CyberArk Conjur: %s", e) return None except Exception as e: - verbose_logger.exception(f"Error reading secret from CyberArk Conjur: {e}") + verbose_logger.exception("Error reading secret from CyberArk Conjur: %s", e) return None async def async_write_secret( @@ -303,7 +303,7 @@ class CyberArkSecretManager(BaseSecretManager): "message": f"Secret {secret_name} written successfully", } except Exception as e: - verbose_logger.exception(f"Error writing secret to CyberArk Conjur: {e}") + verbose_logger.exception("Error writing secret to CyberArk Conjur: %s", e) return {"status": "error", "message": str(e)} async def async_delete_secret( diff --git a/litellm/secret_managers/get_azure_ad_token_provider.py b/litellm/secret_managers/get_azure_ad_token_provider.py index ed348865859..fc3841008f3 100644 --- a/litellm/secret_managers/get_azure_ad_token_provider.py +++ b/litellm/secret_managers/get_azure_ad_token_provider.py @@ -69,7 +69,7 @@ def get_azure_ad_token_provider( if azure_credential else None or os.environ.get("AZURE_CREDENTIAL") or infer_credential_type_from_environment() ) - verbose_logger.info(f"For Azure AD Token Provider, choosing credential type: {cred}") + verbose_logger.info("For Azure AD Token Provider, choosing credential type: %s", cred) credential: ( ClientSecretCredential | ManagedIdentityCredential | CertificateCredential | DefaultAzureCredential | Any | None ) = None diff --git a/litellm/secret_managers/hashicorp_secret_manager.py b/litellm/secret_managers/hashicorp_secret_manager.py index 3f15a4fe5f5..12dae2af706 100644 --- a/litellm/secret_managers/hashicorp_secret_manager.py +++ b/litellm/secret_managers/hashicorp_secret_manager.py @@ -135,7 +135,7 @@ class HashicorpSecretManager(BaseSecretManager): _lease_duration = auth_data["lease_duration"] verbose_logger.debug( - f"Successfully obtained Vault token via AppRole auth. Lease duration: {_lease_duration}s" + "Successfully obtained Vault token via AppRole auth. Lease duration: %ss", _lease_duration ) # Cache the token with its lease duration @@ -337,7 +337,7 @@ class HashicorpSecretManager(BaseSecretManager): return _value except Exception as e: - verbose_logger.exception(f"Error reading secret from Hashicorp Vault: {e}") + verbose_logger.exception("Error reading secret from Hashicorp Vault: %s", e) return None def sync_read_secret( @@ -368,7 +368,7 @@ class HashicorpSecretManager(BaseSecretManager): return _value except Exception as e: - verbose_logger.exception(f"Error reading secret from Hashicorp Vault: {e}") + verbose_logger.exception("Error reading secret from Hashicorp Vault: %s", e) return None async def async_write_secret( @@ -415,7 +415,7 @@ class HashicorpSecretManager(BaseSecretManager): response.raise_for_status() return response.json() except Exception as e: - verbose_logger.exception(f"Error writing secret to Hashicorp Vault: {e}") + verbose_logger.exception("Error writing secret to Hashicorp Vault: %s", e) return {"status": "error", "message": str(e)} async def async_rotate_secret( @@ -459,20 +459,20 @@ class HashicorpSecretManager(BaseSecretManager): # Secret exists, we can proceed except httpx.HTTPStatusError as e: if e.response.status_code == 404: - verbose_logger.exception(f"Current secret {current_secret_name} not found") + verbose_logger.exception("Current secret %s not found", current_secret_name) return { "status": "error", "message": f"Current secret {current_secret_name} not found", } verbose_logger.exception( - f"Error checking current secret existence: {e.response.text if hasattr(e, 'response') else str(e)}" + "Error checking current secret existence: %s", e.response.text if hasattr(e, "response") else str(e) ) return { "status": "error", "message": f"HTTP error occurred while checking current secret: {e.response.text if hasattr(e, 'response') else str(e)}", } except Exception as e: - verbose_logger.exception(f"Error checking current secret existence: {e}") + verbose_logger.exception("Error checking current secret existence: %s", e) return { "status": "error", "message": f"Error checking current secret: {e}", @@ -506,7 +506,9 @@ class HashicorpSecretManager(BaseSecretManager): new_secret_value_from_vault = json_resp.get("data", {}).get("data", {}).get(data_key, None) if new_secret_value_from_vault != new_secret_value: verbose_logger.exception( - f"New secret value mismatch. Expected: {new_secret_value}, Got: {new_secret_value_from_vault}" + "New secret value mismatch. Expected: %s, Got: %s", + new_secret_value, + new_secret_value_from_vault, ) return { "status": "error", @@ -514,20 +516,20 @@ class HashicorpSecretManager(BaseSecretManager): } except httpx.HTTPStatusError as e: if e.response.status_code == 404: - verbose_logger.exception(f"Failed to verify new secret {new_secret_name}") + verbose_logger.exception("Failed to verify new secret %s", new_secret_name) return { "status": "error", "message": f"Failed to verify new secret {new_secret_name}", } verbose_logger.exception( - f"Error verifying new secret: {e.response.text if hasattr(e, 'response') else str(e)}" + "Error verifying new secret: %s", e.response.text if hasattr(e, "response") else str(e) ) return { "status": "error", "message": f"HTTP error occurred while verifying new secret: {e.response.text if hasattr(e, 'response') else str(e)}", } except Exception as e: - verbose_logger.exception(f"Error verifying new secret: {e}") + verbose_logger.exception("Error verifying new secret: %s", e) return { "status": "error", "message": f"Error verifying new secret: {e}", @@ -546,7 +548,9 @@ class HashicorpSecretManager(BaseSecretManager): if isinstance(delete_response, dict) and delete_response.get("status") == "error": # Log the error but don't fail the rotation since new secret was created successfully verbose_logger.warning( - f"Failed to delete old secret {current_secret_name} after rotation: {delete_response.get('message')}" + "Failed to delete old secret %s after rotation: %s", + current_secret_name, + delete_response.get("message"), ) else: # Clear cache for the old secret only if deletion was successful @@ -561,7 +565,7 @@ class HashicorpSecretManager(BaseSecretManager): verbose_logger.exception("Timeout error occurred during secret rotation") return {"status": "error", "message": "Timeout error occurred"} except Exception as e: - verbose_logger.exception(f"Error rotating secret in Hashicorp Vault: {e}") + verbose_logger.exception("Error rotating secret in Hashicorp Vault: %s", e) return {"status": "error", "message": str(e)} async def async_delete_secret( @@ -604,7 +608,7 @@ class HashicorpSecretManager(BaseSecretManager): "message": f"Secret {target['secret_name']} deleted successfully", } except Exception as e: - verbose_logger.exception(f"Error deleting secret from Hashicorp Vault: {e}") + verbose_logger.exception("Error deleting secret from Hashicorp Vault: %s", e) return {"status": "error", "message": str(e)} def _get_secret_value_from_json_response(self, json_resp: dict | None) -> str | None: diff --git a/litellm/secret_managers/main.py b/litellm/secret_managers/main.py index 2982d30274b..a3094bc06a1 100644 --- a/litellm/secret_managers/main.py +++ b/litellm/secret_managers/main.py @@ -335,7 +335,10 @@ def get_secret( ) except Exception as e: # check if it's in os.environ verbose_logger.error( - f"Defaulting to os.environ value for key={secret_name}. An exception occurred - {e}.\n\n{traceback.format_exc()}" + "Defaulting to os.environ value for key=%s. An exception occurred - %s.\n\n%s", + secret_name, + e, + traceback.format_exc(), ) secret = os.getenv(secret_name) try: diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/zscaler_ai_guard.py b/litellm/types/proxy/guardrails/guardrail_hooks/zscaler_ai_guard.py index f522f5b470a..c4eb5c39f1c 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/zscaler_ai_guard.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/zscaler_ai_guard.py @@ -102,7 +102,7 @@ class ZscalerAIGuardConfigModel(GuardrailConfigModel): policy_id = int(env_policy) except ValueError: verbose_proxy_logger.warning( - f"ZSCALER_AI_GUARD_POLICY_ID env var is not a valid integer: {env_policy}" + "ZSCALER_AI_GUARD_POLICY_ID env var is not a valid integer: %s", env_policy ) # Check for configuration issues diff --git a/litellm/types/videos/utils.py b/litellm/types/videos/utils.py index f1b20618a74..1b701260f2f 100644 --- a/litellm/types/videos/utils.py +++ b/litellm/types/videos/utils.py @@ -105,7 +105,7 @@ def decode_video_id_with_provider(encoded_video_id: str) -> DecodedVideoId: video_id=decoded_video_id, ) except Exception as e: - verbose_logger.debug(f"Error decoding video_id '{encoded_video_id}': {e}") + verbose_logger.debug("Error decoding video_id '%s': %s", encoded_video_id, e) return DecodedVideoId( custom_llm_provider=None, model_id=None, @@ -182,7 +182,7 @@ def decode_character_id_with_provider(encoded_character_id: str) -> DecodedChara character_id=decoded_character_id, ) except Exception as e: - verbose_logger.debug(f"Error decoding character_id '{encoded_character_id}': {e}") + verbose_logger.debug("Error decoding character_id '%s': %s", encoded_character_id, e) return DecodedCharacterId( custom_llm_provider=None, model_id=None, diff --git a/litellm/utils.py b/litellm/utils.py index eb3e578b7e8..5dbff5070aa 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -513,7 +513,9 @@ def _add_custom_logger_callback_to_specific_event(callback: str, logging_event: if callback not in litellm._known_custom_logger_compatible_callbacks: verbose_logger.debug( - f"Callback {callback} is not a valid custom logger compatible callback. Known list - {litellm._known_custom_logger_compatible_callbacks}" + "Callback %s is not a valid custom logger compatible callback. Known list - %s", + callback, + litellm._known_custom_logger_compatible_callbacks, ) return @@ -947,7 +949,7 @@ def function_setup( except Exception as e: # Log the error but don't fail the request - verbose_logger.warning(f"Error removing thought signatures from tool call IDs: {e}") + verbose_logger.warning("Error removing thought signatures from tool call IDs: %s", e) elif call_type == CallTypes.embedding.value or call_type == CallTypes.aembedding.value: messages = args[1] if len(args) > 1 else kwargs.get("input", None) elif call_type == CallTypes.image_generation.value or call_type == CallTypes.aimage_generation.value: @@ -1004,7 +1006,7 @@ def function_setup( else: messages = "default-message-value" except Exception as e: - verbose_logger.debug(f"Error extracting messages from Google contents: {e}") + verbose_logger.debug("Error extracting messages from Google contents: %s", e) messages = "default-message-value" else: messages = "default-message-value" @@ -1951,7 +1953,7 @@ def _select_tokenizer_helper(model: str) -> SelectTokenizerResponse: if result is not None: return result except Exception as e: - verbose_logger.debug(f"Error selecting tokenizer: {e}") + verbose_logger.debug("Error selecting tokenizer: %s", e) # default - tiktoken return _return_openai_tokenizer(model) @@ -2064,7 +2066,7 @@ def create_pretrained_tokenizer(identifier: str, revision="main", auth_token: st auth_token=auth_token, # type: ignore ) except Exception as e: - verbose_logger.error(f"Error creating pretrained tokenizer: {e}. Defaulting to version without 'auth_token'.") + verbose_logger.error("Error creating pretrained tokenizer: %s. Defaulting to version without 'auth_token'.", e) tokenizer = Tokenizer.from_pretrained(identifier, revision=revision) return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} @@ -2224,7 +2226,10 @@ def supports_native_streaming(model: str, custom_llm_provider: str | None) -> bo return supports_native_streaming except Exception as e: verbose_logger.debug( - f"Model not found or error in checking supports_native_streaming support. You passed model={model}, custom_llm_provider={custom_llm_provider}. Error: {e}" + "Model not found or error in checking supports_native_streaming support. You passed model=%s, custom_llm_provider=%s. Error: %s", + model, + custom_llm_provider, + e, ) return False @@ -2248,7 +2253,10 @@ def supports_response_schema(model: str, custom_llm_provider: str | None = None) model, custom_llm_provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) except Exception as e: verbose_logger.debug( - f"Model not found or error in checking response schema support. You passed model={model}, custom_llm_provider={custom_llm_provider}. Error: {e}" + "Model not found or error in checking response schema support. You passed model=%s, custom_llm_provider=%s. Error: %s", + model, + custom_llm_provider, + e, ) return False @@ -2362,7 +2370,11 @@ def _supports_factory(model: str, custom_llm_provider: str | None, key: str) -> return False except Exception as e: verbose_logger.debug( - f"Model not found or error in checking {key} support. You passed model={model}, custom_llm_provider={custom_llm_provider}. Error: {e}" + "Model not found or error in checking %s support. You passed model=%s, custom_llm_provider=%s. Error: %s", + key, + model, + custom_llm_provider, + e, ) supported_by_provider = _supports_provider_info_factory(model, custom_llm_provider, key) @@ -2402,9 +2414,11 @@ def _is_explicitly_disabled_factory(model: str, custom_llm_provider: str | None, return False except Exception as e: verbose_logger.debug( - f"Model not found or error in checking {key} disabled state. " - f"You passed model={model}, custom_llm_provider={custom_llm_provider}. " - f"Error: {e}" + "Model not found or error in checking %s disabled state. You passed model=%s, custom_llm_provider=%s. Error: %s", + key, + model, + custom_llm_provider, + e, ) return False @@ -2537,7 +2551,10 @@ def get_supported_regions(model: str, custom_llm_provider: str | None = None) -> return None except Exception as e: verbose_logger.debug( - f"Model not found or error in checking supported_regions support. You passed model={model}, custom_llm_provider={custom_llm_provider}. Error: {e}" + "Model not found or error in checking supported_regions support. You passed model=%s, custom_llm_provider=%s. Error: %s", + model, + custom_llm_provider, + e, ) return None @@ -2720,10 +2737,8 @@ def register_model(model_cost: str | dict): and value.get("cache_read_input_token_cost") is None ): verbose_logger.warning( - f"register_model: model={key} not in built-in cost map and no " - "prefix/region variant matched; cache cost fields will default " - "to 0. To track cache cost, add cache_creation_input_token_cost " - "and cache_read_input_token_cost to model_info" + "register_model: model=%s not in built-in cost map and no prefix/region variant matched; cache cost fields will default to 0. To track cache cost, add cache_creation_input_token_cost and cache_read_input_token_cost to model_info", + key, ) # ``get_model_info`` returns ``litellm_provider: None`` when the # provider is unknown (e.g. custom deployments registered via @@ -2754,7 +2769,7 @@ def register_model(model_cost: str | dict): # Invalidate case-insensitive lookup map since model_cost was modified _invalidate_model_cost_lowercase_map() - verbose_logger.debug(f"added/updated model={model_cost_key} in litellm.model_cost: {model_cost_key}") + verbose_logger.debug("added/updated model=%s in litellm.model_cost: %s", model_cost_key, model_cost_key) # add new model names to provider lists if value.get("litellm_provider") == "openai": if key not in litellm.open_ai_chat_completion_models: @@ -3828,9 +3843,9 @@ def get_optional_params( Args: supported_params: List[str] - supported params from the litellm config """ - verbose_logger.info(f"\nLiteLLM completion() model= {model}; provider = {custom_llm_provider}") - verbose_logger.debug(f"\nLiteLLM: Params passed to completion() {passed_params}") - verbose_logger.debug(f"\nLiteLLM: Non-Default params passed to completion() {non_default_params}") + verbose_logger.info("\nLiteLLM completion() model= %s; provider = %s", model, custom_llm_provider) + verbose_logger.debug("\nLiteLLM: Params passed to completion() %s", passed_params) + verbose_logger.debug("\nLiteLLM: Non-Default params passed to completion() %s", non_default_params) unsupported_params = {} for k in non_default_params.keys(): if k not in supported_params: @@ -4571,7 +4586,7 @@ def _infer_model_region(litellm_params: LiteLLM_Params) -> AllowedModelRegion | model_region = _get_model_region(custom_llm_provider=custom_llm_provider, litellm_params=litellm_params) if model_region is None: - verbose_logger.debug(f"Cannot infer model region for model: {litellm_params.model}") + verbose_logger.debug("Cannot infer model region for model: %s", litellm_params.model) return None if custom_llm_provider == "azure": @@ -5238,7 +5253,7 @@ def _get_model_info_helper( ########################## potential_model_names = _get_potential_model_names(model=model, custom_llm_provider=custom_llm_provider) - verbose_logger.debug(f"checking potential_model_names in litellm.model_cost: {potential_model_names}") + verbose_logger.debug("checking potential_model_names in litellm.model_cost: %s", potential_model_names) combined_model_name = potential_model_names["combined_model_name"] stripped_model_name = potential_model_names["stripped_model_name"] @@ -5373,7 +5388,9 @@ def _get_model_info_helper( if _input_cost_per_token is None: # default value to 0, be noisy about this verbose_logger.debug( - f"model={model}, custom_llm_provider={custom_llm_provider} has no input_cost_per_token in model_cost_map. Defaulting to 0." + "model=%s, custom_llm_provider=%s has no input_cost_per_token in model_cost_map. Defaulting to 0.", + model, + custom_llm_provider, ) _input_cost_per_token = 0 @@ -5381,7 +5398,9 @@ def _get_model_info_helper( if _output_cost_per_token is None: # default value to 0, be noisy about this verbose_logger.debug( - f"model={model}, custom_llm_provider={custom_llm_provider} has no output_cost_per_token in model_cost_map. Defaulting to 0." + "model=%s, custom_llm_provider=%s has no output_cost_per_token in model_cost_map. Defaulting to 0.", + model, + custom_llm_provider, ) _output_cost_per_token = 0 @@ -5548,7 +5567,7 @@ def _get_model_info_helper( returned_model_info[cost_key] = cost_value # type: ignore[literal-required] return returned_model_info except Exception as e: - verbose_logger.debug(f"Error getting model info: {e}") + verbose_logger.debug("Error getting model info: %s", e) raise Exception( f"This model isn't mapped yet. model={model}, custom_llm_provider={custom_llm_provider}. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json." ) @@ -6663,13 +6682,13 @@ def process_messages(messages, max_tokens, model): messages = messages[::-1] final_messages = [] verbose_logger.debug( - f"calling process_messages with messages: {messages}, max_tokens: {max_tokens}, model: {model}" + "calling process_messages with messages: %s, max_tokens: %s, model: %s", messages, max_tokens, model ) for message in messages: - verbose_logger.debug(f"processing final_messages: {final_messages}") + verbose_logger.debug("processing final_messages: %s", final_messages) used_tokens = get_token_count(final_messages, model) available_tokens = max_tokens - used_tokens - verbose_logger.debug(f"used_tokens: {used_tokens}, available_tokens: {available_tokens}") + verbose_logger.debug("used_tokens: %s, available_tokens: %s", used_tokens, available_tokens) if available_tokens <= 3: break @@ -6680,15 +6699,15 @@ def process_messages(messages, max_tokens, model): max_tokens=max_tokens, model=model, ) - verbose_logger.debug(f"final_messages after attempt_message_addition: {final_messages}") - verbose_logger.debug(f"Final messages: {final_messages}") + verbose_logger.debug("final_messages after attempt_message_addition: %s", final_messages) + verbose_logger.debug("Final messages: %s", final_messages) return final_messages def attempt_message_addition(final_messages, message, available_tokens, max_tokens, model): temp_messages = [message] + final_messages temp_message_tokens = get_token_count(messages=temp_messages, model=model) - verbose_logger.debug(f"temp_message_tokens: {temp_message_tokens}, max_tokens: {max_tokens}") + verbose_logger.debug("temp_message_tokens: %s, max_tokens: %s", temp_message_tokens, max_tokens) if temp_message_tokens <= max_tokens: return temp_messages @@ -6735,12 +6754,12 @@ def shorten_message_to_fit_limit(message, tokens_needed, model: str | None, rais content = message["content"] attempts = 0 - verbose_logger.debug(f"content: {content}") + verbose_logger.debug("content: %s", content) while attempts < MAX_TOKEN_TRIMMING_ATTEMPTS: - verbose_logger.debug(f"getting token count for message: {message}") + verbose_logger.debug("getting token count for message: %s", message) total_tokens = get_token_count([message], model) - verbose_logger.debug(f"total_tokens: {total_tokens}, tokens_needed: {tokens_needed}") + verbose_logger.debug("total_tokens: %s, tokens_needed: %s", total_tokens, tokens_needed) if total_tokens <= tokens_needed: break @@ -6756,7 +6775,7 @@ def shorten_message_to_fit_limit(message, tokens_needed, model: str | None, rais trimmed_content = left_half + ".." + right_half message["content"] = trimmed_content - verbose_logger.debug(f"trimmed_content: {trimmed_content}") + verbose_logger.debug("trimmed_content: %s", trimmed_content) content = trimmed_content attempts += 1 @@ -6851,9 +6870,9 @@ def trim_messages( # we remove all system messages from the messages list messages = [message for message in messages if message["role"] != "system"] - verbose_logger.debug(f"Processed system message: {system_message_event}") + verbose_logger.debug("Processed system message: %s", system_message_event) final_messages = process_messages(messages=messages, max_tokens=max_tokens, model=model) - verbose_logger.debug(f"Processed messages: {final_messages}") + verbose_logger.debug("Processed messages: %s", final_messages) # Add system message to the beginning of the final messages if system_message_event: @@ -6862,13 +6881,13 @@ def trim_messages( if len(tool_messages) > 0: final_messages.extend(tool_messages) - verbose_logger.debug(f"Final messages: {final_messages}, return_response_tokens: {return_response_tokens}") + verbose_logger.debug("Final messages: %s, return_response_tokens: %s", final_messages, return_response_tokens) if return_response_tokens: # if user wants token count with new trimmed messages response_tokens = max_tokens - get_token_count(final_messages, model) return final_messages, response_tokens return final_messages except Exception as e: # [NON-Blocking, if error occurs just return final_messages - verbose_logger.exception(f"Got exception while token trimming - {e}") + verbose_logger.exception("Got exception while token trimming - %s", e) return original_messages @@ -6982,7 +7001,7 @@ def _get_valid_models_from_provider_api( _model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) return models except Exception as e: - verbose_logger.warning(f"Error getting valid models: {e}") + verbose_logger.warning("Error getting valid models: %s", e) return [] @@ -7056,7 +7075,7 @@ def get_valid_models( return valid_models except Exception as e: - verbose_logger.warning(f"Error getting valid models: {e}") + verbose_logger.warning("Error getting valid models: %s", e) return [] # NON-Blocking @@ -9112,7 +9131,7 @@ def is_prompt_caching_valid_prompt( min_token_count = get_prompt_cache_min_tokens(model=model) return token_count >= min_token_count except Exception as e: - verbose_logger.error(f"Error in is_prompt_caching_valid_prompt: {e}") + verbose_logger.error("Error in is_prompt_caching_valid_prompt: %s", e) return False diff --git a/litellm/vector_stores/vector_store_registry.py b/litellm/vector_stores/vector_store_registry.py index 1350e2b187e..37866a76797 100644 --- a/litellm/vector_stores/vector_store_registry.py +++ b/litellm/vector_stores/vector_store_registry.py @@ -235,7 +235,7 @@ class VectorStoreRegistry: self.add_vector_store_to_registry(vector_store=db_vector_store) return db_vector_store except Exception as e: - verbose_logger.debug(f"Error fetching vector store from database: {e}") + verbose_logger.debug("Error fetching vector store from database: %s", e) return None @@ -341,12 +341,13 @@ class VectorStoreRegistry: if db_vector_store is None: # Vector store was deleted from database, remove from cache verbose_logger.debug( - f"Vector store {vector_store_id} found in memory but deleted from database, removing from cache" + "Vector store %s found in memory but deleted from database, removing from cache", + vector_store_id, ) self.delete_vector_store_from_registry(vector_store_id=vector_store_id) vector_store = None except Exception as e: - verbose_logger.debug(f"Error verifying vector store {vector_store_id} in database: {e}") + verbose_logger.debug("Error verifying vector store %s in database: %s", vector_store_id, e) # Fall back to database if not found in memory (or was deleted) if vector_store is None and prisma_client is not None: @@ -355,7 +356,7 @@ class VectorStoreRegistry: vector_store_id=vector_store_id, prisma_client=prisma_client ) except Exception as e: - verbose_logger.debug(f"Error fetching vector store {vector_store_id} from database: {e}") + verbose_logger.debug("Error fetching vector store %s from database: %s", vector_store_id, e) if vector_store is not None: # Create a copy to avoid modifying the registry diff --git a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py index c97856598f1..8e6fa35b452 100644 --- a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py +++ b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py @@ -552,9 +552,15 @@ class TestMCPClientResolvedAuth: await http_client.aclose() +def _rendered_log_message(call): + message = str(call.args[0]) + values = call.args[1:] + return message % values if values else message + + def _all_logged_messages(mock_logger): return " ".join( - str(call.args[0]) + _rendered_log_message(call) for level in ("info", "debug", "warning", "error", "exception") for call in getattr(mock_logger, level).call_args_list if call.args diff --git a/tests/test_litellm/integrations/newrelic/test_newrelic.py b/tests/test_litellm/integrations/newrelic/test_newrelic.py index 541b271cb77..00a115dd8f2 100644 --- a/tests/test_litellm/integrations/newrelic/test_newrelic.py +++ b/tests/test_litellm/integrations/newrelic/test_newrelic.py @@ -22,6 +22,13 @@ import litellm import litellm.integrations.newrelic.newrelic as nr_module from litellm.integrations.newrelic.newrelic import NewRelicLogger + +def _rendered_log_message(call): + message = str(call.args[0]) + values = call.args[1:] + return message % values if values else message + + # The module may have been imported before sys.modules was patched (e.g. via # litellm's own startup imports), leaving _newrelic_agent=None. Point it at # the mock agent so all tests see a non-None agent. @@ -268,8 +275,9 @@ class TestParseBoolEnv: assert mock_warn.call_count == 2 # Warning should mention the variable name and the raw value for call in mock_warn.call_args_list: - assert "MY_VAR" in call.args[0] - assert repr(raw) in call.args[0] + rendered = _rendered_log_message(call) + assert "MY_VAR" in rendered + assert repr(raw) in rendered # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index d94f0d5f47e..47baacd61d7 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -20,6 +20,12 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import StandardCallbackDynamicParams +def _rendered_log_message(call): + message = str(call.args[0]) + values = call.args[1:] + return message % values if values else message + + @pytest.mark.asyncio async def test_anthropic_cache_control_hook_system_message(): # Use patch.dict to mock environment variables instead of setting them directly @@ -339,7 +345,7 @@ async def test_anthropic_cache_control_hook_out_of_bounds_logging(): # Verify that warning was called with the expected message mock_logger.warning.assert_called_once() - warning_call = mock_logger.warning.call_args[0][0] + warning_call = _rendered_log_message(mock_logger.warning.call_args) # Check that the warning message contains the expected information assert "AnthropicCacheControlHook: Provided index 10 is out of bounds" in warning_call @@ -405,7 +411,7 @@ async def test_anthropic_cache_control_hook_negative_out_of_bounds_logging(): # Verify that warning was called with the expected message mock_logger.warning.assert_called_once() - warning_call = mock_logger.warning.call_args[0][0] + warning_call = _rendered_log_message(mock_logger.warning.call_args) # Check that the warning message contains the original negative index assert "AnthropicCacheControlHook: Provided index -5 is out of bounds" in warning_call diff --git a/tests/test_litellm/llms/azure/test_azure_common_utils.py b/tests/test_litellm/llms/azure/test_azure_common_utils.py index c5d4bd044cc..85db11fdb24 100644 --- a/tests/test_litellm/llms/azure/test_azure_common_utils.py +++ b/tests/test_litellm/llms/azure/test_azure_common_utils.py @@ -1430,7 +1430,7 @@ def test_token_provider_returns_non_string(setup_mocks): # Verify the error was logged setup_mocks["logger"].error.assert_any_call( - "Azure AD token provider returned non-string value: " + "Azure AD token provider returned non-string value: %s", int ) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index 3a84428add1..850d01c6e34 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -24,6 +24,12 @@ from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer +def _rendered_log_message(call): + message = str(call.args[0]) + values = call.args[1:] + return message % values if values else message + + @pytest.fixture(autouse=True) def cleanup_mcp_global_state(): """Clean up MCP global state before and after each test. @@ -1106,12 +1112,12 @@ async def test_get_tools_from_mcp_servers_continues_when_one_server_fails(): assert result.outcomes["failing"].tag == "internal" # Verify failure logging - mock_logger.exception.assert_any_call( - "Error getting tools from server failing_server: Server connection failed" - ) + assert "Error getting tools from server failing_server: Server connection failed" in [ + _rendered_log_message(c) for c in mock_logger.exception.call_args_list if c.args + ] # Verify success logging - mock_logger.info.assert_any_call("Successfully fetched 1 tools total from all MCP servers") + mock_logger.info.assert_any_call("Successfully fetched %s tools total from all MCP servers", 1) @pytest.mark.asyncio @@ -1201,15 +1207,20 @@ async def test_get_tools_from_mcp_servers_handles_all_servers_failing(): assert result.outcomes["failing2"].tag == "internal" # Verify failure logging for both servers - mock_logger.exception.assert_any_call( + rendered_exceptions = [ + _rendered_log_message(c) for c in mock_logger.exception.call_args_list if c.args + ] + assert ( "Error getting tools from server failing_server1: Server failing_server1 connection failed" + in rendered_exceptions ) - mock_logger.exception.assert_any_call( + assert ( "Error getting tools from server failing_server2: Server failing_server2 connection failed" + in rendered_exceptions ) # Verify total logging - mock_logger.info.assert_any_call("Successfully fetched 0 tools total from all MCP servers") + mock_logger.info.assert_any_call("Successfully fetched %s tools total from all MCP servers", 0) @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index 329b6d5c45d..7ba9f463197 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -22,6 +22,12 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.types.mcp import MCPAuth +def _rendered_log_message(call): + message = str(call.args[0]) + values = call.args[1:] + return message % values if values else message + + def _build_request( headers: Optional[Dict[str, str]] = None, *, @@ -2048,9 +2054,9 @@ class TestCallToolRestAPI: assert exc_info.value.headers.get("www-authenticate") == challenge # The expected caller-must-reauth signal is logged once, at info, and never at error, so # error-rate alerts do not fire on normal pass-through re-authentication. - error_messages = [str(c.args[0]) for c in mock_logger.error.call_args_list if c.args] + error_messages = [_rendered_log_message(c) for c in mock_logger.error.call_args_list if c.args] assert not any("MCP tool call" in m for m in error_messages) - info_messages = [str(c.args[0]) for c in mock_logger.info.call_args_list if c.args] + info_messages = [_rendered_log_message(c) for c in mock_logger.info.call_args_list if c.args] assert sum(str(upstream_status) in m for m in info_messages) == 1 async def test_local_permission_denial_keeps_error_level_logging(self, monkeypatch): @@ -2112,9 +2118,9 @@ class TestCallToolRestAPI: await rest_endpoints.call_tool_rest_api(request, user_api_key_dict=UserAPIKeyAuth()) assert exc_info.value.status_code == 403 - error_messages = [str(c.args[0]) for c in mock_logger.error.call_args_list if c.args] + error_messages = [_rendered_log_message(c) for c in mock_logger.error.call_args_list if c.args] assert any("HTTPException in MCP tool call" in m for m in error_messages) - info_messages = [str(c.args[0]) for c in mock_logger.info.call_args_list if c.args] + info_messages = [_rendered_log_message(c) for c in mock_logger.info.call_args_list if c.args] assert not any("relaying upstream" in m for m in info_messages) async def test_success_logging_cancellation_propagates(self, monkeypatch): diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index f5aa695cb78..d1b5395c73d 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -57,6 +57,12 @@ from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.utils import get_utc_datetime +def _rendered_log_message(call): + message = str(call.args[0]) + values = call.args[1:] + return message % values if values else message + + @pytest.fixture(autouse=True) def set_salt_key(monkeypatch): """Automatically set LITELLM_SALT_KEY for all tests""" @@ -1167,7 +1173,7 @@ def test_log_budget_lookup_failure_dry_run(): err = Exception("column 'policies' does not exist in prisma schema") _log_budget_lookup_failure("user", err) mock_logger.error.assert_called_once() - call_msg = mock_logger.error.call_args[0][0] + call_msg = _rendered_log_message(mock_logger.error.call_args) assert "user" in call_msg assert "cache will not be populated" in call_msg assert "policies" in call_msg or "prisma" in call_msg diff --git a/tests/test_litellm/proxy/auth/test_model_checks_fallbacks.py b/tests/test_litellm/proxy/auth/test_model_checks_fallbacks.py index f3bd1742827..c65f36d1ef0 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks_fallbacks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks_fallbacks.py @@ -2,6 +2,12 @@ import pytest from unittest.mock import Mock, patch +def _rendered_log_message(call): + message = str(call.args[0]) + values = call.args[1:] + return message % values if values else message + + def create_mock_router( fallbacks=None, context_window_fallbacks=None, content_policy_fallbacks=None ): @@ -150,7 +156,7 @@ def test_invalid_fallback_type_returns_empty_list(): ) assert result == [] - mock_logger.warning.assert_called_once_with("Unknown fallback_type: invalid") + mock_logger.warning.assert_called_once_with("Unknown fallback_type: %s", "invalid") def test_exception_handling_returns_empty_list(): @@ -171,7 +177,7 @@ def test_exception_handling_returns_empty_list(): assert result == [] mock_logger.error.assert_called_once() - error_call_args = mock_logger.error.call_args[0][0] + error_call_args = _rendered_log_message(mock_logger.error.call_args) assert ( "Error getting fallbacks for model claude-4-sonnet" in error_call_args ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index fc018a5333e..eedffa1ea5f 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -984,7 +984,8 @@ async def test_add_team_member_budget_table_exception_handling(): # Verify the error was logged mock_logger.info.assert_called_once_with( - "Team member budget table not found, passed team_member_budget_id=nonexistent-budget-456" + "Team member budget table not found, passed team_member_budget_id=%s", + "nonexistent-budget-456", ) # Verify database call was attempted diff --git a/tests/test_litellm/test_logging.py b/tests/test_litellm/test_logging.py index fbed044445b..beba5794444 100644 --- a/tests/test_litellm/test_logging.py +++ b/tests/test_litellm/test_logging.py @@ -1,7 +1,9 @@ +import ast import asyncio import json import os import sys +from pathlib import Path from typing import List import pytest @@ -328,3 +330,66 @@ async def test_cache_hit_includes_custom_llm_provider(): # Clean up litellm.callbacks = original_callbacks litellm.cache = None + + +LITELLM_LOGGER_NAMES = frozenset( + {"verbose_logger", "verbose_proxy_logger", "verbose_router_logger", "logger", "logging"} +) +LOG_LEVEL_METHODS = frozenset({"debug", "info", "warning", "error", "exception", "critical"}) +LITELLM_PACKAGE_ROOT = Path(__file__).resolve().parents[2] / "litellm" + + +def _receiver_name(node: ast.expr) -> str: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + return node.attr + return "" + + +def _is_logging_call(node: ast.AST) -> bool: + return ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr in LOG_LEVEL_METHODS + and _receiver_name(node.func.value) in LITELLM_LOGGER_NAMES + ) + + +def _has_format_spec(message: ast.JoinedStr) -> bool: + return any(isinstance(value, ast.FormattedValue) and value.format_spec is not None for value in message.values) + + +def _eager_logging_calls(source: str, path: Path) -> tuple[str, ...]: + return tuple( + f"{path}:{node.lineno}" + for node in ast.walk(ast.parse(source)) + if _is_logging_call(node) + and node.args + and isinstance(node.args[0], ast.JoinedStr) + and not _has_format_spec(node.args[0]) + ) + + +def test_logging_calls_do_not_build_their_message_eagerly(): + """A discarded log record must not have cost anything to build. + + `log.debug(f"payload: {body}")` interpolates before the call runs, so the message is + built and thrown away on every request the level filters out; `log.debug("payload: %s", body)` + defers that to `record.getMessage()`, which only runs once the record passes the level check. + + f-strings carrying a format spec are exempt: `%`-style has no faithful equivalent for + specs like `{ratio:.1%}`, and those sites interpolate scalars rather than payloads. + """ + offenders = tuple( + offender + for path in sorted(LITELLM_PACKAGE_ROOT.rglob("*.py")) + for offender in _eager_logging_calls( + path.read_text(encoding="utf-8"), path.relative_to(LITELLM_PACKAGE_ROOT.parent) + ) + ) + + assert offenders == (), ( + "these logging calls build their message eagerly; pass the values as %-style arguments instead:\n" + + "\n".join(offenders) + ) From abe3289398bba544193366d9777b6953e2b3ed9e Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 3 Aug 2026 22:08:58 -0700 Subject: [PATCH 072/124] fix(proxy): retry model cost map fetch with Retry-After-aware backoff and keep current map on reload failure (#35739) * fix(proxy): retry model cost map fetch with Retry-After-aware backoff and stop downgrading to the packaged backup on reload failure A 429 or transient network error during a manual or scheduled model cost map reload used to silently replace litellm.model_cost with the stale backup JSON bundled in the installed wheel, stamp the reload as successful, and clear the force_reload flag, so a fleet could serve months-old pricing until the next interval. Runtime reloads now go through refetch_model_cost_map, which retries 429/5xx/transport errors up to 3 times honoring Retry-After (capped at 30s, exponential backoff with jitter otherwise) and returns a failure value instead of the backup when the fetch or integrity validation fails. On failure the pod keeps its currently loaded map, the periodic job leaves last_run and force_reload untouched so it retries on the next config poll, and the manual endpoint returns 502 with the reason instead of reporting a fake success. Startup behavior is unchanged: boot still falls back to the packaged backup since there is no previously loaded map to keep. * fix(proxy): use shared async httpx client for cost map reload and make retry tests CI-env-proof The reload fetch now goes through get_async_httpx_client with a dedicated httpxSpecialProvider.ModelCostMap pool instead of constructing a raw httpx.AsyncClient, so it inherits deployment-level TLS and transport settings and passes the ensure_async_clients gate. Tests inject a MockTransport-backed client through the same seam. An autouse fixture clears LITELLM_LOCAL_MODEL_COST_MAP, which CI exports and which short-circuited the retry tests; the two TestPriceDataReloadAPI tests and the config sync pubsub reload test that still patched get_model_cost_map now patch refetch_model_cost_map instead. --- .../litellm_core_utils/get_model_cost_map.py | 157 +++++++++++++++ litellm/proxy/proxy_server.py | 26 ++- litellm/types/llms/custom_http.py | 1 + .../test_get_model_cost_map.py | 183 ++++++++++++++++++ .../common_utils/test_config_sync_pubsub.py | 10 +- .../test_routes_model_cost_map.py | 45 ++++- tests/test_litellm/proxy/test_proxy_server.py | 116 ++++++++--- 7 files changed, 502 insertions(+), 36 deletions(-) diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index e87e3d8aca2..5d36e391fbc 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -8,9 +8,14 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True ``` """ +import asyncio import json import os +import random +from collections.abc import Awaitable, Callable +from dataclasses import dataclass from importlib.resources import files +from typing import Protocol import httpx @@ -161,6 +166,158 @@ class GetModelCostMap: return response.json() +RETRYABLE_FETCH_STATUS_CODES = frozenset({429, 500, 502, 503, 504}) +MODEL_COST_MAP_FETCH_MAX_ATTEMPTS = 3 +MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS = 30.0 + + +@dataclass(frozen=True, slots=True) +class ModelCostMapReloaded: + model_cost_map: dict # mutable-ok: adopted as litellm.model_cost, whose consumer contract is a plain mutable dict + + +@dataclass(frozen=True, slots=True) +class ModelCostMapReloadUnavailable: + reason: str + + +ModelCostMapReloadResult = ModelCostMapReloaded | ModelCostMapReloadUnavailable + + +@dataclass(frozen=True, slots=True) +class _FetchAttemptRetryable: + reason: str + retry_after_seconds: float | None + + +def _parse_retry_after_seconds(response: httpx.Response) -> float | None: + header = response.headers.get("Retry-After") + if header is None: + return None + try: + seconds = float(header) + except ValueError: + return None + return seconds if seconds >= 0 else None + + +def _retry_wait_seconds(outcome: _FetchAttemptRetryable, attempt: int, rng: random.Random) -> float: + if outcome.retry_after_seconds is not None: + return min(outcome.retry_after_seconds, MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS) + return min(float(2**attempt) + rng.uniform(0.0, 1.0), MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS) + + +class _AsyncGetClient(Protocol): + def get(self, url: str, *, timeout: float | None = None) -> Awaitable[httpx.Response]: ... + + +def _default_reload_client() -> _AsyncGetClient: + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.llms.custom_http import httpxSpecialProvider + + return get_async_httpx_client(llm_provider=httpxSpecialProvider.ModelCostMap) + + +async def _attempt_fetch( + client: _AsyncGetClient, url: str, timeout: int +) -> ModelCostMapReloaded | ModelCostMapReloadUnavailable | _FetchAttemptRetryable: + try: + response = await client.get(url, timeout=timeout) + except httpx.HTTPError as e: + return _FetchAttemptRetryable(reason=f"{type(e).__name__} fetching {url}: {e}", retry_after_seconds=None) + if response.status_code in RETRYABLE_FETCH_STATUS_CODES: + return _FetchAttemptRetryable( + reason=f"HTTP {response.status_code} from {url}", + retry_after_seconds=_parse_retry_after_seconds(response), + ) + if response.is_error: + return ModelCostMapReloadUnavailable(reason=f"HTTP {response.status_code} from {url}") + try: + parsed = response.json() + except ValueError as e: + return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}") + if not isinstance(parsed, dict): + return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}") + return ModelCostMapReloaded(model_cost_map=parsed) + + +async def _fetch_remote_model_cost_map_with_retry( + url: str, + timeout: int, + max_attempts: int, + sleep: Callable[[float], Awaitable[None]], + rng: random.Random, + client: _AsyncGetClient, +) -> ModelCostMapReloadResult: + for attempt in range(1, max_attempts + 1): + outcome = await _attempt_fetch(client=client, url=url, timeout=timeout) + if not isinstance(outcome, _FetchAttemptRetryable): + return outcome + if attempt == max_attempts: + return ModelCostMapReloadUnavailable(reason=f"{outcome.reason} (after {max_attempts} attempts)") + wait_seconds = _retry_wait_seconds(outcome=outcome, attempt=attempt, rng=rng) + verbose_logger.warning( + "LiteLLM: model cost map fetch attempt %d/%d failed (%s); retrying in %.1fs", + attempt, + max_attempts, + outcome.reason, + wait_seconds, + ) + await sleep(wait_seconds) + return ModelCostMapReloadUnavailable(reason="model cost map fetch failed") + + +async def refetch_model_cost_map( + url: str, + timeout: int = 5, + max_attempts: int = MODEL_COST_MAP_FETCH_MAX_ATTEMPTS, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + rng: random.Random | None = None, + client: "_AsyncGetClient | None" = None, +) -> ModelCostMapReloadResult: + """ + Re-fetch the model cost map for a runtime reload, retrying transient HTTP + errors (429/5xx/transport) with Retry-After-aware backoff. + + Unlike ``get_model_cost_map`` this never falls back to the packaged backup: + on failure it returns ``ModelCostMapReloadUnavailable`` so callers keep the + map they already have. + """ + if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true": + _cost_map_source_info.source = "local" + _cost_map_source_info.url = None + _cost_map_source_info.is_env_forced = True + _cost_map_source_info.fallback_reason = None + return ModelCostMapReloaded( + model_cost_map=_finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()) + ) + + result = await _fetch_remote_model_cost_map_with_retry( + url=url, + timeout=timeout, + max_attempts=max_attempts, + sleep=sleep, + rng=rng if rng is not None else random.Random(), + client=client if client is not None else _default_reload_client(), + ) + if isinstance(result, ModelCostMapReloadUnavailable): + verbose_logger.warning( + "LiteLLM: model cost map reload failed: %s. Keeping the currently loaded map.", + result.reason, + ) + return result + if not GetModelCostMap.validate_model_cost_map( + fetched_map=result.model_cost_map, + backup_model_count=GetModelCostMap._get_backup_model_count(), + ): + return ModelCostMapReloadUnavailable(reason=f"model cost map from {url} failed integrity validation") + _cost_map_source_info.source = "remote" + _cost_map_source_info.url = url + _cost_map_source_info.is_env_forced = False + _cost_map_source_info.fallback_reason = None + return ModelCostMapReloaded(model_cost_map=_finalize_model_cost_map(result.model_cost_map)) + + class ModelCostMapSourceInfo: """Tracks the source of the currently loaded model cost map.""" diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 16a63540d4d..e8299be8f07 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6538,11 +6538,18 @@ class ProxyConfig: if should_reload: # Perform the reload from litellm.litellm_core_utils.get_model_cost_map import ( - get_model_cost_map, + ModelCostMapReloadUnavailable, + refetch_model_cost_map, ) model_cost_map_url = litellm.model_cost_map_url - new_model_cost_map = get_model_cost_map(url=model_cost_map_url) + reload_result = await refetch_model_cost_map(url=model_cost_map_url) + if isinstance(reload_result, ModelCostMapReloadUnavailable): + verbose_proxy_logger.warning( + f"Model cost map reload failed ({reload_result.reason}); keeping current pricing data, will retry on the next config poll" + ) + return + new_model_cost_map = reload_result.model_cost_map litellm.model_cost = new_model_cost_map # Invalidate case-insensitive lookup map since model_cost was replaced _invalidate_model_cost_lowercase_map() @@ -15836,10 +15843,19 @@ async def reload_model_cost_map( raise HTTPException(status_code=500, detail="Database connection not available") # Immediately reload the model cost map in the current pod - from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map + from litellm.litellm_core_utils.get_model_cost_map import ( + ModelCostMapReloadUnavailable, + refetch_model_cost_map, + ) model_cost_map_url = litellm.model_cost_map_url - new_model_cost_map = get_model_cost_map(url=model_cost_map_url) + reload_result = await refetch_model_cost_map(url=model_cost_map_url) + if isinstance(reload_result, ModelCostMapReloadUnavailable): + raise HTTPException( + status_code=502, + detail=f"Failed to reload model cost map: {reload_result.reason}. Current pricing data was kept.", + ) + new_model_cost_map = reload_result.model_cost_map litellm.model_cost = new_model_cost_map # Invalidate case-insensitive lookup map since model_cost was replaced _invalidate_model_cost_lowercase_map() @@ -15881,6 +15897,8 @@ async def reload_model_cost_map( "models_count": models_count, "timestamp": current_time.isoformat(), } + except HTTPException: + raise except Exception as e: verbose_proxy_logger.exception("Failed to reload model cost map: %s", e) raise HTTPException(status_code=500, detail=f"Failed to reload model cost map: {e}") diff --git a/litellm/types/llms/custom_http.py b/litellm/types/llms/custom_http.py index d7dfc0e486b..548dde4c04d 100644 --- a/litellm/types/llms/custom_http.py +++ b/litellm/types/llms/custom_http.py @@ -30,6 +30,7 @@ class httpxSpecialProvider(str, Enum): PromptManagement = "prompt_management" UI = "ui" Sandbox = "sandbox" + ModelCostMap = "model_cost_map" VerifyTypes = Union[str, bool, ssl.SSLContext] diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index f21c667276e..463bdc6161f 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -248,3 +248,186 @@ def test_azure_ai_claude_1m_context_entries(cost_map: dict): "azure_ai/claude-haiku-4-5", ]: assert cost_map[model]["max_input_tokens"] == 200000, model + + +# --------------------------------------------------------------------------- +# refetch_model_cost_map: retry/backoff behavior for runtime reloads +# --------------------------------------------------------------------------- + +import functools +import random + +import httpx + +from litellm.litellm_core_utils.get_model_cost_map import ( + ModelCostMapReloaded, + ModelCostMapReloadUnavailable, + refetch_model_cost_map, +) + +_URL = "https://example.invalid/model_prices.json" + + +@functools.lru_cache(maxsize=1) +def _real_map_bytes() -> bytes: + return json.dumps(_load_root_cost_map()).encode() + + +class _SleepRecorder: + """Injected in place of asyncio.sleep so tests assert waits without real delay.""" + + def __init__(self): + self.waits = [] + + async def __call__(self, seconds: float) -> None: + self.waits.append(seconds) + + +@pytest.fixture(autouse=True) +def _unset_local_cost_map_env(monkeypatch): + """CI exports LITELLM_LOCAL_MODEL_COST_MAP=True; clear it so fetch behavior is deterministic.""" + monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) + + +def _mock_client(outcomes): + """httpx client over a MockTransport serving one outcome per request; an exception instance is raised.""" + calls = {"count": 0} + + def handler(request): + idx = min(calls["count"], len(outcomes) - 1) + calls["count"] += 1 + outcome = outcomes[idx] + if isinstance(outcome, Exception): + raise outcome + return outcome + + return httpx.AsyncClient(transport=httpx.MockTransport(handler)), calls + + +@pytest.mark.asyncio +async def test_refetch_retries_429_honoring_retry_after(): + """Two 429s with Retry-After then success: waits follow the header, not backoff.""" + client, calls = _mock_client( + [ + httpx.Response(429, headers={"Retry-After": "7"}), + httpx.Response(429, headers={"Retry-After": "7"}), + httpx.Response(200, content=_real_map_bytes()), + ] + ) + sleeper = _SleepRecorder() + result = await refetch_model_cost_map( + url=_URL, sleep=sleeper, rng=random.Random(0), client=client + ) + assert isinstance(result, ModelCostMapReloaded) + assert len(result.model_cost_map) > 100 + assert calls["count"] == 3 + assert sleeper.waits == [7.0, 7.0] + + +@pytest.mark.asyncio +async def test_refetch_gives_up_after_max_attempts_with_exponential_backoff(): + """All 429 without Retry-After: exponential backoff waits, then a failure value.""" + client, calls = _mock_client([httpx.Response(429)]) + sleeper = _SleepRecorder() + result = await refetch_model_cost_map( + url=_URL, sleep=sleeper, rng=random.Random(0), client=client + ) + assert isinstance(result, ModelCostMapReloadUnavailable) + assert "429" in result.reason + assert "after 3 attempts" in result.reason + assert calls["count"] == 3 + assert len(sleeper.waits) == 2 + assert 2.0 <= sleeper.waits[0] < 3.0 + assert 4.0 <= sleeper.waits[1] < 5.0 + + +@pytest.mark.asyncio +async def test_refetch_caps_retry_after_wait(): + """A hostile/huge Retry-After is capped so reloads never sleep unbounded.""" + client, _calls = _mock_client( + [ + httpx.Response(429, headers={"Retry-After": "9999"}), + httpx.Response(200, content=_real_map_bytes()), + ] + ) + sleeper = _SleepRecorder() + result = await refetch_model_cost_map( + url=_URL, sleep=sleeper, rng=random.Random(0), client=client + ) + assert isinstance(result, ModelCostMapReloaded) + assert sleeper.waits == [30.0] + + +@pytest.mark.asyncio +async def test_refetch_retries_transport_errors(): + """Connection failures are transient: retried like 5xx, succeeding when the network heals.""" + client, calls = _mock_client( + [ + httpx.ConnectError("connection refused"), + httpx.Response(200, content=_real_map_bytes()), + ] + ) + sleeper = _SleepRecorder() + result = await refetch_model_cost_map( + url=_URL, sleep=sleeper, rng=random.Random(0), client=client + ) + assert isinstance(result, ModelCostMapReloaded) + assert calls["count"] == 2 + assert len(sleeper.waits) == 1 + + +@pytest.mark.asyncio +async def test_refetch_non_retryable_status_fails_immediately(): + """A 404 is permanent: one attempt, no sleeps, failure value.""" + client, calls = _mock_client([httpx.Response(404)]) + sleeper = _SleepRecorder() + result = await refetch_model_cost_map( + url=_URL, sleep=sleeper, rng=random.Random(0), client=client + ) + assert isinstance(result, ModelCostMapReloadUnavailable) + assert "404" in result.reason + assert calls["count"] == 1 + assert sleeper.waits == [] + + +@pytest.mark.asyncio +async def test_refetch_invalid_json_fails_immediately(): + client, calls = _mock_client([httpx.Response(200, content=b"not json")]) + sleeper = _SleepRecorder() + result = await refetch_model_cost_map( + url=_URL, sleep=sleeper, rng=random.Random(0), client=client + ) + assert isinstance(result, ModelCostMapReloadUnavailable) + assert "invalid JSON" in result.reason + assert calls["count"] == 1 + assert sleeper.waits == [] + + +@pytest.mark.asyncio +async def test_refetch_shrunk_map_fails_integrity_not_swapped_in(): + """A drastically shrunk upstream file is rejected instead of being adopted.""" + tiny = json.dumps(_make_models(60)).encode() + client, _calls = _mock_client([httpx.Response(200, content=tiny)]) + result = await refetch_model_cost_map( + url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client + ) + assert isinstance(result, ModelCostMapReloadUnavailable) + assert "integrity validation" in result.reason + + +@pytest.mark.asyncio +async def test_refetch_respects_local_env_override(monkeypatch): + """LITELLM_LOCAL_MODEL_COST_MAP=True short-circuits to the bundled backup, zero HTTP.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + + def _fail(request): + raise AssertionError("no HTTP request should be made when local map is forced") + + result = await refetch_model_cost_map( + url=_URL, + sleep=_SleepRecorder(), + rng=random.Random(0), + client=httpx.AsyncClient(transport=httpx.MockTransport(_fail)), + ) + assert isinstance(result, ModelCostMapReloaded) + assert len(result.model_cost_map) > 100 diff --git a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py index 6872407808c..e357f25123b 100644 --- a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py +++ b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py @@ -776,8 +776,14 @@ async def test_model_cost_map_reload_does_not_publish_config_change() -> None: original_model_cost = litellm.model_cost.copy() _set_redis_usage_cache(_FakeRedisCache(client)) try: - with patch("litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map") as mock_get_map: - mock_get_map.return_value = {"gpt-5.2": {"input_cost_per_token": 0.001}} + from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded + + with patch( + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + new=AsyncMock( + return_value=ModelCostMapReloaded(model_cost_map={"gpt-5.2": {"input_cost_per_token": 0.001}}) + ), + ): await ProxyConfig()._check_and_reload_model_cost_map(prisma_client=prisma_client) finally: litellm.model_cost = original_model_cost diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py b/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py index 16e410f1b1e..df1d096b3e2 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py @@ -48,6 +48,7 @@ def _attach_litellm_config(mock_prisma): def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma): """Admin can trigger a manual reload; handler returns model count + status.""" + from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -56,8 +57,8 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma): fake_cost_map = {"gpt-4": {"input_cost": 0.03}, "gpt-3.5": {"input_cost": 0.002}} monkeypatch.setattr( - "litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map", - lambda url=None: fake_cost_map, + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + AsyncMock(return_value=ModelCostMapReloaded(model_cost_map=fake_cost_map)), ) monkeypatch.setattr("litellm.add_known_models", lambda model_cost_map=None: None) monkeypatch.setattr("litellm.model_cost", {}, raising=False) @@ -85,6 +86,46 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma): assert table.upsert.await_count == 1 +def test_reload_model_cost_map_fetch_failure_502_keeps_map( + client, auth_as, monkeypatch, mock_prisma +): + """Fetch failure returns 502 with the reason; the pod's map and DB flag are untouched. + + Regression: the endpoint used to report success after silently swapping in + the stale packaged backup. + """ + from litellm.litellm_core_utils.get_model_cost_map import ( + ModelCostMapReloadUnavailable, + ) + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _attach_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + sentinel_map = {"existing-model": {"input_cost": 0.01}} + monkeypatch.setattr("litellm.model_cost", sentinel_map, raising=False) + monkeypatch.setattr( + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + AsyncMock( + return_value=ModelCostMapReloadUnavailable( + reason="HTTP 429 from upstream (after 3 attempts)" + ) + ), + ) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post("/reload/model_cost_map") + assert response.status_code == 502 + detail = response.json().get("detail", "") + assert "HTTP 429 from upstream" in detail + assert "Current pricing data was kept" in detail + import litellm as litellm_module + + assert litellm_module.model_cost is sentinel_map + assert table.upsert.await_count == 0 + + def test_reload_model_cost_map_not_admin_forbidden(client, auth_as): """Non-admin caller gets 403 with a role-specific detail.""" from litellm.proxy._types import LitellmUserRoles diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index ab64224fdc3..67e76585485 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -3622,14 +3622,18 @@ class TestPriceDataReloadAPI: # Save the original model_cost so the endpoint's direct assignment # (litellm.model_cost = new_model_cost_map) does not contaminate # subsequent tests running in the same worker process. + from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded + original_model_cost = litellm.model_cost.copy() try: with patch( - "litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map" - ) as mock_get_map: - mock_get_map.return_value = { - "gpt-3.5-turbo": {"input_cost_per_token": 0.001} - } + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + new=AsyncMock( + return_value=ModelCostMapReloaded( + model_cost_map={"gpt-3.5-turbo": {"input_cost_per_token": 0.001}} + ) + ), + ): # Mock the database connection with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: mock_prisma.db.litellm_config.find_unique = AsyncMock( @@ -3684,10 +3688,9 @@ class TestPriceDataReloadAPI: def test_reload_model_cost_map_error_handling(self, client_with_auth): """Test error handling in the reload endpoint""" with patch( - "litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map" - ) as mock_get_map: - mock_get_map.side_effect = Exception("Network error") - + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + new=AsyncMock(side_effect=Exception("Network error")), + ): # Mock the database connection with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) @@ -3697,7 +3700,7 @@ class TestPriceDataReloadAPI: assert ( response.status_code == 500 - ) # The new implementation immediately reloads and fails on error + ) # An unexpected exception still maps to 500 data = response.json() assert "Failed to reload model cost map" in data["detail"] @@ -3885,13 +3888,16 @@ class TestPriceDataReloadIntegration: "gpt-4": {"input_cost_per_token": 0.03, "output_cost_per_token": 0.06}, } + from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded + original_model_cost = litellm.model_cost.copy() try: with patch( - "litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map" - ) as mock_get_map: - mock_get_map.return_value = mock_cost_map - + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + new=AsyncMock( + return_value=ModelCostMapReloaded(model_cost_map=mock_cost_map) + ), + ): # Mock the database connection with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: mock_prisma.db.litellm_config.find_unique = AsyncMock( @@ -3956,15 +3962,18 @@ class TestPriceDataReloadIntegration: mock_prisma.get_generic_data = AsyncMock(return_value=mock_config) mock_prisma.db.litellm_config.upsert = AsyncMock(return_value=None) + from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded + original_model_cost = litellm.model_cost.copy() try: with patch( - "litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map" - ) as mock_get_map: - mock_get_map.return_value = { - "gpt-3.5-turbo": {"input_cost_per_token": 0.001} - } - + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + new=AsyncMock( + return_value=ModelCostMapReloaded( + model_cost_map={"gpt-3.5-turbo": {"input_cost_per_token": 0.001}} + ) + ), + ): # Should reload due to force flag asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma)) @@ -3999,13 +4008,18 @@ class TestPriceDataReloadIntegration: mock_prisma.get_generic_data = AsyncMock(return_value=mock_config) mock_prisma.db.litellm_config.upsert = AsyncMock(return_value=None) + from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded + original_model_cost = litellm.model_cost.copy() try: with patch( - "litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map" - ) as mock_get_map: - mock_get_map.return_value = {"gpt-4": {"input_cost_per_token": 0.001}} - + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + new=AsyncMock( + return_value=ModelCostMapReloaded( + model_cost_map={"gpt-4": {"input_cost_per_token": 0.001}} + ) + ), + ): asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma)) # Verify the upsert update branch preserves interval_hours @@ -4022,6 +4036,47 @@ class TestPriceDataReloadIntegration: litellm.model_cost = original_model_cost _invalidate_model_cost_lowercase_map() + def test_distributed_reload_keeps_current_map_when_fetch_fails(self): + """Fetch failure during a periodic/forced reload must not downgrade the pod. + + Regression: a 429/network failure used to silently replace litellm.model_cost + with the stale packaged backup, stamp last_run, and clear force_reload. + """ + from litellm.litellm_core_utils.get_model_cost_map import ( + ModelCostMapReloadUnavailable, + ) + from litellm.proxy import proxy_server as ps + from litellm.proxy.proxy_server import ProxyConfig + + proxy_config = ProxyConfig() + mock_prisma = MagicMock() + + mock_config = MagicMock() + mock_config.param_value = {"interval_hours": 6, "force_reload": True} + mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=mock_config) + mock_prisma.get_generic_data = AsyncMock(return_value=mock_config) + mock_prisma.db.litellm_config.upsert = AsyncMock(return_value=None) + + original_model_cost = litellm.model_cost + with patch( + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + new=AsyncMock( + return_value=ModelCostMapReloadUnavailable(reason="HTTP 429 from upstream") + ), + ): + with patch("litellm.proxy.proxy_server.last_model_cost_map_reload", None): + asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma)) + assert ps.last_model_cost_map_reload is None, ( + "a failed reload must not stamp the pod's last reload time, " + "otherwise the retry waits a full interval" + ) + + assert litellm.model_cost is original_model_cost, ( + "a failed reload must keep the currently loaded cost map, " + "not swap in the packaged backup" + ) + mock_prisma.db.litellm_config.upsert.assert_not_called() + def test_manual_reload_preserves_interval_hours(self): """Test that manual reload via /reload/model_cost_map preserves existing interval_hours. @@ -4041,13 +4096,18 @@ class TestPriceDataReloadIntegration: app.dependency_overrides[user_api_key_auth] = lambda: mock_auth client = TestClient(app) + from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded + original_model_cost = litellm.model_cost.copy() try: with patch( - "litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map" - ) as mock_get_map: - mock_get_map.return_value = {"gpt-4": {"input_cost_per_token": 0.001}} - + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + new=AsyncMock( + return_value=ModelCostMapReloaded( + model_cost_map={"gpt-4": {"input_cost_per_token": 0.001}} + ) + ), + ): with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: # Simulate existing config with a schedule mock_existing = MagicMock() From 903c0d82aafb2d75d7b75be2557f06d822b1b4d1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 3 Aug 2026 22:19:54 -0700 Subject: [PATCH 073/124] refactor(repositories): add prisma protocol seams and a spend-reset unit of work Moves reset_budget_job's hand-rolled private Prisma protocols into litellm/repositories as shared seams, and replaces its three ad-hoc db.batch_() write helpers with a composed unit of work that binds typed per-table write repositories to a single batch, committing on clean exit and writing nothing when the block raises. --- .../proxy/common_utils/reset_budget_job.py | 81 ++++--------------- litellm/repositories/__init__.py | 24 ++++++ litellm/repositories/prisma_protocols.py | 43 ++++++++++ litellm/repositories/unit_of_work.py | 61 ++++++++++++++ .../common_utils/test_reset_budget_job.py | 26 ++++++ .../repositories/test_unit_of_work.py | 66 +++++++++++++++ 6 files changed, 236 insertions(+), 65 deletions(-) create mode 100644 litellm/repositories/prisma_protocols.py create mode 100644 litellm/repositories/unit_of_work.py create mode 100644 tests/test_litellm/repositories/test_unit_of_work.py diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 6ec441a0e06..1537958063a 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -1,7 +1,7 @@ import asyncio import json import time -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Sequence from datetime import datetime, timezone from typing import Literal, Protocol, TypeVar @@ -23,50 +23,20 @@ from litellm.proxy.common_utils.timezone_utils import ( ) from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.repositories.organization_repository import OrganizationRepository +from litellm.repositories.prisma_protocols import ReadOnlyTable, SpendLinkedTable from litellm.repositories.table_repositories import ( EndUserRepository, TagRepository, TeamMembershipRepository, ) from litellm.repositories.team_repository import TeamRepository +from litellm.repositories.unit_of_work import spend_reset_unit_of_work from litellm.repositories.verification_token_repository import ( VerificationTokenRepository, ) from litellm.types.services import ServiceTypes _RowT = TypeVar("_RowT") -_RowT_co = TypeVar("_RowT_co", covariant=True) - - -class _PrismaRecord(Protocol): - def dict(self) -> Mapping[str, object]: ... - - -class _BatchTable(Protocol): - def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> None: ... - - -class _ResetBatcher(Protocol): - @property - def litellm_verificationtoken(self) -> _BatchTable: ... - - @property - def litellm_usertable(self) -> _BatchTable: ... - - @property - def litellm_teamtable(self) -> _BatchTable: ... - - async def commit(self) -> None: ... - - -class _EndUserTable(Protocol): - async def find_many(self, where: Mapping[str, object]) -> Sequence[_PrismaRecord]: ... - - -class _SpendLinkedTable(Protocol[_RowT_co]): - async def find_many(self, where: Mapping[str, object]) -> Sequence[_RowT_co]: ... - - async def update_many(self, where: Mapping[str, object], data: Mapping[str, object]) -> int: ... class _TeamMembershipRow(Protocol): @@ -227,7 +197,7 @@ class ResetBudgetJob: async def _cascade_reset_spend_for_budget_link( self, budgets_to_reset: list[LiteLLM_BudgetTableFull], - table: "_SpendLinkedTable[_RowT]", + table: SpendLinkedTable[_RowT], counter_key_fn: Callable[[_RowT], str], log_subject: str, extra_where: dict[str, object] | None = None, @@ -466,7 +436,7 @@ class ResetBudgetJob: rely on the default budget (litellm.max_end_user_budget_id) applied in-memory during auth checks. """ - table: _EndUserTable = EndUserRepository(self.prisma_client).table + table: ReadOnlyTable = EndUserRepository(self.prisma_client).table rows = await table.find_many( where={ "budget_id": None, @@ -486,16 +456,11 @@ class ResetBudgetJob: aborts the entire batch — silently leaving spend over the cap and budget_reset_at unchanged forever. """ - batcher: _ResetBatcher = self.prisma_client.db.batch_() - for k in updated_keys: - token = getattr(k, "token", None) - if token is None: - continue - batcher.litellm_verificationtoken.update( - where={"token": token}, - data={"spend": 0, "budget_reset_at": k.budget_reset_at}, - ) - await batcher.commit() + async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow: + for k in updated_keys: + if k.token is None: + continue + uow.keys.queue_spend_reset(token=k.token, budget_reset_at=k.budget_reset_at) async def _write_user_reset_updates(self, updated_users: list[LiteLLM_UserTable]) -> None: """ @@ -505,16 +470,9 @@ class ResetBudgetJob: that trips Prisma's DataError on rows carrying unrecognised fields (see #27730). """ - batcher: _ResetBatcher = self.prisma_client.db.batch_() - for u in updated_users: - user_id = getattr(u, "user_id", None) - if user_id is None: - continue - batcher.litellm_usertable.update( - where={"user_id": user_id}, - data={"spend": 0, "budget_reset_at": u.budget_reset_at}, - ) - await batcher.commit() + async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow: + for u in updated_users: + uow.users.queue_spend_reset(user_id=u.user_id, budget_reset_at=u.budget_reset_at) async def _write_team_reset_updates(self, updated_teams: list[LiteLLM_TeamTable]) -> None: """ @@ -524,16 +482,9 @@ class ResetBudgetJob: that trips Prisma's DataError on rows carrying unrecognised fields (see #27730). """ - batcher: _ResetBatcher = self.prisma_client.db.batch_() - for t in updated_teams: - team_id = getattr(t, "team_id", None) - if team_id is None: - continue - batcher.litellm_teamtable.update( - where={"team_id": team_id}, - data={"spend": 0, "budget_reset_at": t.budget_reset_at}, - ) - await batcher.commit() + async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow: + for t in updated_teams: + uow.teams.queue_spend_reset(team_id=t.team_id, budget_reset_at=t.budget_reset_at) async def reset_budget_for_litellm_keys(self): """ diff --git a/litellm/repositories/__init__.py b/litellm/repositories/__init__.py index 1fc3d8dadaf..4f020480f9e 100644 --- a/litellm/repositories/__init__.py +++ b/litellm/repositories/__init__.py @@ -10,6 +10,13 @@ from litellm.repositories.object_permission_repository import ( ObjectPermissionRepository, ) from litellm.repositories.organization_repository import OrganizationRepository +from litellm.repositories.prisma_protocols import ( + BatchTable, + PrismaBatch, + PrismaRecord, + ReadOnlyTable, + SpendLinkedTable, +) from litellm.repositories.project_repository import ProjectRepository from litellm.repositories.table_repositories import ( AccessGroupRepository, @@ -62,6 +69,13 @@ from litellm.repositories.table_repositories import ( WorkflowRunRepository, ) from litellm.repositories.team_repository import TeamRepository +from litellm.repositories.unit_of_work import ( + KeySpendResetWrites, + SpendResetUnitOfWork, + TeamSpendResetWrites, + UserSpendResetWrites, + spend_reset_unit_of_work, +) from litellm.repositories.user_repository import UserRepository from litellm.repositories.verification_token_repository import ( VerificationTokenRepository, @@ -73,6 +87,7 @@ __all__ = [ "AdaptiveRouterStateRepository", "AgentsRepository", "AuditLogRepository", + "BatchTable", "BudgetRepository", "CacheConfigRepository", "ClaudeCodePluginRepository", @@ -91,6 +106,7 @@ __all__ = [ "HealthCheckRepository", "InvitationLinkRepository", "JWTKeyMappingRepository", + "KeySpendResetWrites", "MCPServerRepository", "MCPToolsetRepository", "MCPUserCredentialsRepository", @@ -106,24 +122,32 @@ __all__ = [ "OrganizationRepository", "PolicyAttachmentRepository", "PolicyRepository", + "PrismaBatch", + "PrismaRecord", "PrismaTableRepository", "ProjectRepository", "PromptRepository", + "ReadOnlyTable", "SSOConfigRepository", "SearchToolsRepository", "SkillsRepository", + "SpendLinkedTable", "SpendLogGuardrailIndexRepository", "SpendLogToolIndexRepository", "SpendLogsRepository", + "SpendResetUnitOfWork", "TagRepository", "TeamMembershipRepository", "TeamRepository", + "TeamSpendResetWrites", "ToolRepository", "UISettingsRepository", "UserNotificationsRepository", "UserRepository", + "UserSpendResetWrites", "VerificationTokenRepository", "WorkflowEventRepository", "WorkflowMessageRepository", "WorkflowRunRepository", + "spend_reset_unit_of_work", ] diff --git a/litellm/repositories/prisma_protocols.py b/litellm/repositories/prisma_protocols.py new file mode 100644 index 00000000000..6aff196ff10 --- /dev/null +++ b/litellm/repositories/prisma_protocols.py @@ -0,0 +1,43 @@ +""" +Typed Protocol seams over prisma-client-py surfaces. + +Modules that reach Prisma through an untyped handle (``prisma_client.db`` or a +repository ``.table``) annotate against these Protocols instead of hand-rolling +private ones per file. +""" + +from collections.abc import Mapping, Sequence +from typing import Protocol, TypeVar + +RowT_co = TypeVar("RowT_co", covariant=True) + + +class PrismaRecord(Protocol): + def dict(self) -> Mapping[str, object]: ... + + +class ReadOnlyTable(Protocol): + async def find_many(self, *, where: Mapping[str, object]) -> Sequence[PrismaRecord]: ... + + +class SpendLinkedTable(Protocol[RowT_co]): + async def find_many(self, *, where: Mapping[str, object]) -> Sequence[RowT_co]: ... + + async def update_many(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> int: ... + + +class BatchTable(Protocol): + def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> None: ... + + +class PrismaBatch(Protocol): + @property + def litellm_verificationtoken(self) -> BatchTable: ... + + @property + def litellm_usertable(self) -> BatchTable: ... + + @property + def litellm_teamtable(self) -> BatchTable: ... + + async def commit(self) -> None: ... diff --git a/litellm/repositories/unit_of_work.py b/litellm/repositories/unit_of_work.py new file mode 100644 index 00000000000..682e69d11eb --- /dev/null +++ b/litellm/repositories/unit_of_work.py @@ -0,0 +1,61 @@ +""" +Unit of work over a single Prisma batch. + +``spend_reset_unit_of_work`` opens one ``db.batch_()`` and binds a typed write +repository per table to it, so every update queued through the yielded object +lands in the same transaction. The batch commits when the block exits cleanly +and is abandoned, writing nothing, when the block raises. + +Each write repository queues narrow ``{spend, budget_reset_at}`` updates +instead of full-model writes, which trip ``prisma.errors.DataError`` on rows +carrying fields the update input type rejects (see #27730). +""" + +from collections.abc import AsyncGenerator, Callable +from contextlib import asynccontextmanager +from dataclasses import dataclass +from datetime import datetime + +from litellm.repositories.prisma_protocols import BatchTable, PrismaBatch + + +@dataclass(frozen=True, slots=True) +class KeySpendResetWrites: + table: BatchTable + + def queue_spend_reset(self, token: str, budget_reset_at: datetime | None) -> None: + self.table.update(where={"token": token}, data={"spend": 0, "budget_reset_at": budget_reset_at}) + + +@dataclass(frozen=True, slots=True) +class UserSpendResetWrites: + table: BatchTable + + def queue_spend_reset(self, user_id: str, budget_reset_at: datetime | None) -> None: + self.table.update(where={"user_id": user_id}, data={"spend": 0, "budget_reset_at": budget_reset_at}) + + +@dataclass(frozen=True, slots=True) +class TeamSpendResetWrites: + table: BatchTable + + def queue_spend_reset(self, team_id: str, budget_reset_at: datetime | None) -> None: + self.table.update(where={"team_id": team_id}, data={"spend": 0, "budget_reset_at": budget_reset_at}) + + +@dataclass(frozen=True, slots=True) +class SpendResetUnitOfWork: + keys: KeySpendResetWrites + users: UserSpendResetWrites + teams: TeamSpendResetWrites + + +@asynccontextmanager +async def spend_reset_unit_of_work(new_batch: Callable[[], PrismaBatch]) -> AsyncGenerator[SpendResetUnitOfWork, None]: + batch = new_batch() + yield SpendResetUnitOfWork( + keys=KeySpendResetWrites(table=batch.litellm_verificationtoken), + users=UserSpendResetWrites(table=batch.litellm_usertable), + teams=TeamSpendResetWrites(table=batch.litellm_teamtable), + ) + await batch.commit() diff --git a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py index f04d6f3cf5a..616ad8a0981 100644 --- a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py +++ b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py @@ -14,6 +14,7 @@ import pytest sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import LiteLLM_VerificationToken from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob from litellm.proxy.common_utils.timezone_utils import BudgetResetSettings from litellm.proxy.utils import ProxyLogging @@ -218,6 +219,31 @@ async def run_async_test(coro): # Tests +def test_write_key_reset_updates_skips_none_token_and_still_writes_the_rest(reset_budget_job, mock_prisma_client): + """A key with token=None must be skipped, not queued as where={"token": None}. + + Queueing a None token makes the prisma batch commit raise and aborts the + whole batch, silently dropping every key reset that cycle (the #27730 + blast radius this write path exists to prevent). + """ + reset_at = datetime.now(timezone.utc) + keys = [ + LiteLLM_VerificationToken(token=None, budget_reset_at=reset_at), + LiteLLM_VerificationToken(token="tok-ok", budget_reset_at=reset_at), + ] + + asyncio.run(reset_budget_job._write_key_reset_updates(updated_keys=keys)) + + key_writes = [c for c in mock_prisma_client.db.batch_calls if c["table"] == "key"] + assert key_writes == [ + { + "table": "key", + "where": {"token": "tok-ok"}, + "data": {"spend": 0, "budget_reset_at": reset_at}, + } + ] + + def test_reset_budget_for_key(reset_budget_job, mock_prisma_client): # Setup test data with timezone-aware datetime now = datetime.now(timezone.utc) diff --git a/tests/test_litellm/repositories/test_unit_of_work.py b/tests/test_litellm/repositories/test_unit_of_work.py new file mode 100644 index 00000000000..35f102bbb9d --- /dev/null +++ b/tests/test_litellm/repositories/test_unit_of_work.py @@ -0,0 +1,66 @@ +from datetime import datetime, timezone +from typing import Any, Dict, List, Mapping, Tuple + +import pytest + +from litellm.repositories.unit_of_work import spend_reset_unit_of_work + + +class FakeBatchTable: + def __init__(self, table_name: str, calls: List[Tuple[str, Dict[str, Any], Dict[str, Any]]]): + self._table_name = table_name + self._calls = calls + + def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> None: + self._calls.append((self._table_name, dict(where), dict(data))) + + +class FakeBatch: + def __init__(self): + self.calls: List[Tuple[str, Dict[str, Any], Dict[str, Any]]] = [] + self.commit_count = 0 + self.litellm_verificationtoken = FakeBatchTable("litellm_verificationtoken", self.calls) + self.litellm_usertable = FakeBatchTable("litellm_usertable", self.calls) + self.litellm_teamtable = FakeBatchTable("litellm_teamtable", self.calls) + + async def commit(self) -> None: + self.commit_count += 1 + + +async def test_updates_across_tables_share_one_batch_and_commit_once(): + batch = FakeBatch() + reset_at = datetime(2026, 8, 3, 12, 0, tzinfo=timezone.utc) + + async with spend_reset_unit_of_work(lambda: batch) as uow: + uow.keys.queue_spend_reset(token="tok-1", budget_reset_at=reset_at) + uow.users.queue_spend_reset(user_id="user-1", budget_reset_at=reset_at) + uow.teams.queue_spend_reset(team_id="team-1", budget_reset_at=None) + assert batch.commit_count == 0 + + assert batch.commit_count == 1 + assert batch.calls == [ + ("litellm_verificationtoken", {"token": "tok-1"}, {"spend": 0, "budget_reset_at": reset_at}), + ("litellm_usertable", {"user_id": "user-1"}, {"spend": 0, "budget_reset_at": reset_at}), + ("litellm_teamtable", {"team_id": "team-1"}, {"spend": 0, "budget_reset_at": None}), + ] + + +async def test_raising_inside_block_skips_commit(): + batch = FakeBatch() + + with pytest.raises(RuntimeError, match="boom"): + async with spend_reset_unit_of_work(lambda: batch) as uow: + uow.keys.queue_spend_reset(token="tok-1", budget_reset_at=None) + raise RuntimeError("boom") + + assert batch.commit_count == 0 + + +async def test_empty_block_still_commits_the_batch(): + batch = FakeBatch() + + async with spend_reset_unit_of_work(lambda: batch): + pass + + assert batch.commit_count == 1 + assert batch.calls == [] From a625d1e1ca2ef9dff286ff309a9097fc877bd75a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 23:10:01 -0700 Subject: [PATCH 074/124] feat(otel): stamp service tier attributes on inference spans (#35679) * feat(otel): stamp service tier attributes on inference spans Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): bound requested service tier to known values The requested tier is caller-controlled and reaches the span verbatim, so an arbitrary string lands on every litellm_request span on success and on failure. A 100k character value was stamped uncapped; safe_set_attribute does not truncate and no span limits are configured. Apply KNOWN_REQUEST_SERVICE_TIERS in get_requested_service_tier so both the span attribute and the Prometheus label bound the value the same way. The served tier stays unrestricted since it comes from the provider, so a tier a provider adds later is still reported. Prometheus label behavior is unchanged. * fix: derive known service tiers from the ServiceTier enum The allowlist omitted "fast", which litellm models as a real tier and prices through the priority cost key, so a request naming it resolved to no tier on the span and no Prometheus label. Deriving the set from ServiceTier keeps the two in sync, so a tier added there for cost calculation cannot go missing here. Behavior change: a request with service_tier "fast" now carries the tier on the span and on the Prometheus service_tier label, where it previously resolved to none. Every other value resolves as before. * refactor: build the known service tiers without a mutable intermediate The set comprehension and set literal tripped LIT002, which bounds mutable collections. Concatenating tuples keeps the derivation from ServiceTier while every intermediate stays immutable; the resulting frozenset is unchanged. --------- Co-authored-by: milan Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Yucheng Zhu --- litellm/integrations/opentelemetry.py | 28 ++++ litellm/integrations/prometheus.py | 51 +------- .../litellm_core_utils/service_tier_utils.py | 68 ++++++++++ .../integrations/test_opentelemetry.py | 121 ++++++++++++++++++ .../test_prometheus_service_tier_label.py | 13 +- 5 files changed, 231 insertions(+), 50 deletions(-) create mode 100644 litellm/litellm_core_utils/service_tier_utils.py diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 94902460c81..e6147409ed8 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -24,6 +24,10 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import ( from litellm.integrations.otel.model.semconv import Metric from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.secret_redaction import redact_string +from litellm.litellm_core_utils.service_tier_utils import ( + get_requested_service_tier, + get_served_service_tier, +) from litellm.secret_managers.main import get_secret_bool, str_to_bool from litellm.types.services import ServiceLoggerPayload from litellm.types.utils import ( @@ -74,6 +78,11 @@ PREPROCESSING_DURATION_MS_ATTRIBUTE = "litellm.preprocessing.duration_ms" TEAM_METADATA_ATTRIBUTE = "litellm.team.metadata" MODEL_GROUP_ATTRIBUTE = "litellm.model_group" PROVIDER_MODEL_ATTRIBUTE = "litellm.provider.model" +# semconv names the service tier attributes under the openai namespace, but every +# provider that reports a tier (OpenAI, Anthropic, Bedrock, Groq, Vertex) uses the +# same request param and response field, so both keys carry all of them. +REQUEST_SERVICE_TIER_ATTRIBUTE = "gen_ai.openai.request.service_tier" +RESPONSE_SERVICE_TIER_ATTRIBUTE = "gen_ai.openai.response.service_tier" # Remove the hardcoded LITELLM_RESOURCE dictionary - we'll create it properly later RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request" LITELLM_REQUEST_SPAN_NAME = "litellm_request" @@ -1411,6 +1420,23 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): if provider_model: self.safe_set_attribute(span=span, key=PROVIDER_MODEL_ATTRIBUTE, value=provider_model) + def _set_service_tier_attributes( + self, + span: Span, + standard_logging_payload: StandardLoggingPayload, + ) -> None: + """Stamp the tier the caller asked for and the tier the provider reports it + served, so tier usage is segmentable in traces. Both are optional: a caller + may not name a tier, and streaming responses carry no served tier. + """ + requested_tier = get_requested_service_tier(standard_logging_payload) + if requested_tier is not None: + self.safe_set_attribute(span=span, key=REQUEST_SERVICE_TIER_ATTRIBUTE, value=requested_tier) + + served_tier = get_served_service_tier(standard_logging_payload) + if served_tier is not None: + self.safe_set_attribute(span=span, key=RESPONSE_SERVICE_TIER_ATTRIBUTE, value=served_tier) + @staticmethod def _team_metadata_json(value: Any, allowed_keys: list[str]) -> str | None: """JSON-serialize only the allowlisted sub-keys of a team's metadata. @@ -2310,6 +2336,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): value=response_obj.get("model"), ) + self._set_service_tier_attributes(span=span, standard_logging_payload=standard_logging_payload) + usage = response_obj and response_obj.get("usage") if usage: self.safe_set_attribute( diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 78f4e213f50..b92e4f7e723 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -34,6 +34,9 @@ from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, get_metadata_variable_name_from_kwargs, ) +from litellm.litellm_core_utils.service_tier_utils import ( + get_service_tier_from_standard_logging_payload, +) from litellm.proxy._types import ( LiteLLM_DeletedVerificationToken, LiteLLM_TeamTable, @@ -98,16 +101,6 @@ class _ExcludedLabelMetric: return self._metric.labels(*kept_values) if kept_values else self._metric -# Tiers a caller may name in a request, across the providers that accept the -# parameter: OpenAI ("auto", "default", "flex", "priority", "scale"), Bedrock and -# Groq (subsets of those), Anthropic ("auto", "standard_only") and Vertex, which -# maps "default" to "standard". Used to bound the caller-controlled fallback in -# ``get_service_tier_from_standard_logging_payload``. -KNOWN_REQUEST_SERVICE_TIERS = frozenset( - {"auto", "batch", "default", "flex", "priority", "scale", "standard", "standard_only"} -) - - def _get_budget_metrics_per_request_timeout() -> float: raw = os.getenv("PROMETHEUS_BUDGET_METRICS_PER_REQUEST_TIMEOUT") if raw is None: @@ -4181,44 +4174,6 @@ def get_custom_labels_from_metadata(metadata: dict) -> dict[str, str]: return result -def get_service_tier_from_standard_logging_payload( - standard_logging_payload: StandardLoggingPayload, -) -> str | None: - """ - Resolve the service tier a request ran on, for the ``service_tier`` label. - - The tier the provider actually served wins over the tier the caller asked for, - so latency and spend stay segmentable when the request said ``auto`` and the - provider picked the concrete tier. Providers report the served tier either at - the top level of the response (OpenAI, Bedrock, Groq) or on the usage object - (Anthropic). - - Streaming responses carry no served tier, so the requested tier is the - fallback. That value is caller-controlled and survives param mapping even - where the provider then ignores it (Bedrock and Groq accept the request and - drop an unrecognized tier), so it is only labelled when it names a known - tier; otherwise one caller could mint a Prometheus series per string. Values - the provider itself reports are not caller-controlled and stay unrestricted, - so a tier a provider adds later is still labelled correctly. - """ - response = standard_logging_payload.get("response") - usage_object = standard_logging_payload.get("metadata", {}).get("usage_object") - - served_candidates: tuple[object, ...] = ( - response.get("service_tier") if isinstance(response, dict) else None, - usage_object.get("service_tier") if isinstance(usage_object, dict) else None, - ) - served_tier = next((tier for tier in served_candidates if isinstance(tier, str) and tier), None) - if served_tier is not None: - return served_tier - - model_parameters = standard_logging_payload.get("model_parameters") - requested_tier = model_parameters.get("service_tier") if isinstance(model_parameters, dict) else None - if isinstance(requested_tier, str) and requested_tier in KNOWN_REQUEST_SERVICE_TIERS: - return requested_tier - return None - - def _get_combined_custom_metadata_from_standard_logging_payload( standard_logging_payload: dict | None, ) -> dict[str, Any]: diff --git a/litellm/litellm_core_utils/service_tier_utils.py b/litellm/litellm_core_utils/service_tier_utils.py new file mode 100644 index 00000000000..9e3f5da42c4 --- /dev/null +++ b/litellm/litellm_core_utils/service_tier_utils.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from litellm.types.utils import ServiceTier, StandardLoggingPayload + +# Tiers a caller may name in a request, across the providers that accept the +# parameter: OpenAI ("auto", "default", "flex", "priority", "scale"), Bedrock and +# Groq (subsets of those), Anthropic ("auto", "standard_only") and Vertex, which +# maps "default" to "standard". Bounds the caller-controlled requested tier +# wherever it is recorded. Derived from ``ServiceTier`` so a tier added there for +# cost calculation cannot go missing here. +KNOWN_REQUEST_SERVICE_TIERS = frozenset( + tuple(tier.value for tier in ServiceTier) + ("batch", "default", "scale", "standard", "standard_only") +) + + +def get_served_service_tier(standard_logging_payload: StandardLoggingPayload) -> str | None: + """ + The tier the provider reports it actually served the request on. + + Providers report it either at the top level of the response (OpenAI, Bedrock, + Groq) or on the usage object (Anthropic). Streaming responses carry no served + tier. + """ + response = standard_logging_payload.get("response") + usage_object = standard_logging_payload.get("metadata", {}).get("usage_object") + + served_candidates: tuple[object, ...] = ( + response.get("service_tier") if isinstance(response, dict) else None, + usage_object.get("service_tier") if isinstance(usage_object, dict) else None, + ) + return next((tier for tier in served_candidates if isinstance(tier, str) and tier), None) + + +def get_requested_service_tier(standard_logging_payload: StandardLoggingPayload) -> str | None: + """ + The tier the caller asked for, as sent to the provider. + + The value is caller-controlled and survives param mapping even where the + provider then ignores it (Bedrock and Groq accept the request and drop an + unrecognized tier), so it is only reported when it names a known tier. + """ + model_parameters = standard_logging_payload.get("model_parameters") + requested_tier = model_parameters.get("service_tier") if isinstance(model_parameters, dict) else None + if isinstance(requested_tier, str) and requested_tier in KNOWN_REQUEST_SERVICE_TIERS: + return requested_tier + return None + + +def get_service_tier_from_standard_logging_payload( + standard_logging_payload: StandardLoggingPayload, +) -> str | None: + """ + Resolve the service tier a request ran on, for the Prometheus ``service_tier`` label. + + The tier the provider actually served wins over the tier the caller asked for, + so latency and spend stay segmentable when the request said ``auto`` and the + provider picked the concrete tier. + + Streaming responses carry no served tier, so the requested tier is the + fallback. Values the provider itself reports are not caller-controlled and + stay unrestricted, so a tier a provider adds later is still labelled + correctly. + """ + served_tier = get_served_service_tier(standard_logging_payload) + if served_tier is not None: + return served_tier + + return get_requested_service_tier(standard_logging_payload) diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 05205cb76f2..b300c386326 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -5886,3 +5886,124 @@ class TestOpenTelemetryMetricAttributeFiltering(unittest.TestCase): exporter="console", attributes=attributes ) ) + + +class TestOTELServiceTierAttributes(unittest.TestCase): + """The tier a request asked for and the tier the provider served must land on + the litellm_request span, so tier usage is segmentable in traces.""" + + REQUEST_KEY = "gen_ai.openai.request.service_tier" + RESPONSE_KEY = "gen_ai.openai.response.service_tier" + + def _span_attributes(self, standard_logging_object, response_obj): + otel = OpenTelemetry() + mock_span = MagicMock() + kwargs = { + "model": "gpt-5-mini", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": standard_logging_object.get("model_parameters") or {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": standard_logging_object, + } + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + return {call[0][0]: call[0][1] for call in mock_span.set_attribute.call_args_list} + + def test_served_tier_from_response_and_requested_tier_are_stamped(self): + response_obj = { + "id": "chatcmpl-1", + "model": "gpt-5-mini", + "service_tier": "scale", + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + } + attributes = self._span_attributes( + { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + "model_parameters": {"service_tier": "auto"}, + "response": response_obj, + }, + response_obj, + ) + self.assertEqual(attributes[self.RESPONSE_KEY], "scale") + self.assertEqual(attributes[self.REQUEST_KEY], "auto") + + def test_served_tier_read_from_usage_object(self): + """Anthropic reports the served tier on the usage object, not the top level.""" + response_obj = { + "id": "chatcmpl-2", + "model": "claude-sonnet-4-5", + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + } + attributes = self._span_attributes( + { + "id": "test-id", + "call_type": "completion", + "metadata": {"usage_object": {"service_tier": "priority"}}, + "model_parameters": {}, + "response": response_obj, + }, + response_obj, + ) + self.assertEqual(attributes[self.RESPONSE_KEY], "priority") + self.assertNotIn(self.REQUEST_KEY, attributes) + + def test_no_tier_anywhere_stamps_nothing(self): + response_obj = { + "id": "chatcmpl-3", + "model": "gpt-5-mini", + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + } + attributes = self._span_attributes( + { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + "model_parameters": {}, + "response": response_obj, + }, + response_obj, + ) + self.assertNotIn(self.RESPONSE_KEY, attributes) + self.assertNotIn(self.REQUEST_KEY, attributes) + + def test_unknown_requested_tier_is_not_stamped(self): + """The requested tier is caller-controlled, so an unrecognized value is + dropped rather than written verbatim onto the span.""" + response_obj = { + "id": "chatcmpl-4", + "model": "gpt-5-mini", + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + } + attributes = self._span_attributes( + { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + "model_parameters": {"service_tier": "Z" * 5000}, + "response": response_obj, + }, + response_obj, + ) + self.assertNotIn(self.REQUEST_KEY, attributes) + + def test_served_tier_is_stamped_even_when_unrecognized(self): + """The served tier comes from the provider, not the caller, so a tier a + provider adds later is still stamped.""" + response_obj = { + "id": "chatcmpl-5", + "model": "gpt-5-mini", + "service_tier": "tier-added-by-provider-later", + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + } + attributes = self._span_attributes( + { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + "model_parameters": {}, + "response": response_obj, + }, + response_obj, + ) + self.assertEqual(attributes[self.RESPONSE_KEY], "tier-added-by-provider-later") diff --git a/tests/test_litellm/integrations/test_prometheus_service_tier_label.py b/tests/test_litellm/integrations/test_prometheus_service_tier_label.py index 9d702b19c6c..b2212c4ff41 100644 --- a/tests/test_litellm/integrations/test_prometheus_service_tier_label.py +++ b/tests/test_litellm/integrations/test_prometheus_service_tier_label.py @@ -13,9 +13,9 @@ import datetime import pytest -from litellm.integrations.prometheus import ( +from litellm.integrations.prometheus import PrometheusLogger +from litellm.litellm_core_utils.service_tier_utils import ( KNOWN_REQUEST_SERVICE_TIERS, - PrometheusLogger, get_service_tier_from_standard_logging_payload, ) from litellm.types.integrations.prometheus import ( @@ -230,3 +230,12 @@ async def test_success_event_emits_service_tier_on_latency_and_spend_metrics(): ) finally: _clear_prometheus_registry() + + +def test_allowlist_covers_every_modeled_service_tier(): + """A tier modeled for cost calculation is real traffic, so it must resolve + rather than being dropped as an unknown caller value.""" + from litellm.types.utils import ServiceTier + + missing = {tier.value for tier in ServiceTier} - KNOWN_REQUEST_SERVICE_TIERS + assert not missing, f"ServiceTier values missing from the allowlist: {sorted(missing)}" From 956d5177d1d915adc8084c142d9d2babad1ff7af Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 3 Aug 2026 23:41:21 -0700 Subject: [PATCH 075/124] fix(proxy): log the model cost map reload failure lazily (#35750) The reload-failure warning built its message with an f-string, so the interpolation ran on every failed reload whether or not the warning level was enabled. `test_logging_calls_do_not_build_their_message_eagerly` scans the whole litellm package and asserts zero offenders, so this one call has been reddening `misc / Run tests` on litellm_internal_staging for every branch cut from it Passing the reason as a %-style argument defers the interpolation to `record.getMessage()`, which only runs once the record passes the level check --- litellm/proxy/proxy_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index e8299be8f07..1aeb8814585 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6546,7 +6546,8 @@ class ProxyConfig: reload_result = await refetch_model_cost_map(url=model_cost_map_url) if isinstance(reload_result, ModelCostMapReloadUnavailable): verbose_proxy_logger.warning( - f"Model cost map reload failed ({reload_result.reason}); keeping current pricing data, will retry on the next config poll" + "Model cost map reload failed (%s); keeping current pricing data, will retry on the next config poll", + reload_result.reason, ) return new_model_cost_map = reload_result.model_cost_map From 368dd0be5b9f2c8218e3c1d8626f1a82fc32d4e3 Mon Sep 17 00:00:00 2001 From: Ahmed N <34286755+hMED22@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:08:11 +0100 Subject: [PATCH 076/124] fix(groq): translate web_search_options to the browser_search tool (#34971) --- litellm/constants.py | 1 + litellm/llms/__init__.py | 6 + litellm/llms/groq/chat/transformation.py | 66 ++++- litellm/llms/groq/cost_calculator.py | 27 ++ ...odel_prices_and_context_window_backup.json | 15 ++ litellm/types/utils.py | 1 + model_prices_and_context_window.json | 15 ++ tests/test_litellm/llms/groq/__init__.py | 0 tests/test_litellm/llms/groq/chat/__init__.py | 0 .../chat/test_groq_chat_transformation.py | 247 ++++++++++++++++++ .../llms/groq/test_groq_cost_calculator.py | 56 ++++ 11 files changed, 432 insertions(+), 2 deletions(-) create mode 100644 litellm/llms/groq/cost_calculator.py create mode 100644 tests/test_litellm/llms/groq/__init__.py create mode 100644 tests/test_litellm/llms/groq/chat/__init__.py create mode 100644 tests/test_litellm/llms/groq/chat/test_groq_chat_transformation.py create mode 100644 tests/test_litellm/llms/groq/test_groq_cost_calculator.py diff --git a/litellm/constants.py b/litellm/constants.py index 06421e6ed6a..164f5a77a76 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -313,6 +313,7 @@ MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES = int(os.getenv("MAX_LONG_SIDE_FOR_IMAGE_HIGH_R MAX_TILE_WIDTH = int(os.getenv("MAX_TILE_WIDTH", 512)) MAX_TILE_HEIGHT = int(os.getenv("MAX_TILE_HEIGHT", 512)) OPENAI_FILE_SEARCH_COST_PER_1K_CALLS = float(os.getenv("OPENAI_FILE_SEARCH_COST_PER_1K_CALLS", 2.5 / 1000)) +GROQ_BROWSER_VISIT_WEBSITE_COST_PER_CALL = 1.0 / 1000 # Azure OpenAI Assistants feature costs # Source: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/ AZURE_FILE_SEARCH_COST_PER_GB_PER_DAY = float( diff --git a/litellm/llms/__init__.py b/litellm/llms/__init__.py index a35fe5b2093..72c8bf15b47 100644 --- a/litellm/llms/__init__.py +++ b/litellm/llms/__init__.py @@ -55,6 +55,12 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo from .xai.cost_calculator import cost_per_web_search_request return cost_per_web_search_request(usage=usage, model_info=model_info) + elif custom_llm_provider == "groq": + from .groq.cost_calculator import ( + cost_per_web_search_request as groq_cost_per_web_search_request, + ) + + return groq_cost_per_web_search_request(usage=usage, model_info=model_info) else: return None diff --git a/litellm/llms/groq/chat/transformation.py b/litellm/llms/groq/chat/transformation.py index 35a4a14057f..b319d6067aa 100644 --- a/litellm/llms/groq/chat/transformation.py +++ b/litellm/llms/groq/chat/transformation.py @@ -11,7 +11,7 @@ from typing import ( ) import httpx -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_logger @@ -27,10 +27,20 @@ from litellm.types.llms.openai import ( ChatCompletionToolParam, ChatCompletionToolParamFunctionChunk, ) -from litellm.types.utils import ModelResponse, ModelResponseStream +from litellm.types.utils import ModelResponse, ModelResponseStream, ServerToolUse from ...openai_like.chat.transformation import OpenAILikeChatConfig +GROQ_COMPOUND_MODELS = frozenset({"compound", "compound-mini"}) + + +class GroqExecutedToolIdentity(BaseModel): + name: str | None = None + type: str | None = None + + +_EXECUTED_TOOLS_ADAPTER = TypeAdapter(tuple[GroqExecutedToolIdentity, ...]) + class GroqChatConfig(OpenAILikeChatConfig): frequency_penalty: int | None = None @@ -95,6 +105,12 @@ class GroqChatConfig(OpenAILikeChatConfig): except ValueError: pass + if not ( + self._is_compound_model(model) + or litellm.supports_web_search(model=model, custom_llm_provider=self.custom_llm_provider) + ): + base_params.remove("web_search_options") + try: if litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider): base_params.append("reasoning_effort") @@ -103,6 +119,10 @@ class GroqChatConfig(OpenAILikeChatConfig): return base_params + @staticmethod + def _is_compound_model(model: str) -> bool: + return model.removeprefix("groq/") in GROQ_COMPOUND_MODELS + @overload def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: Literal[True] @@ -238,7 +258,23 @@ class GroqChatConfig(OpenAILikeChatConfig): "response_format", None ) # only remove if it's a json_schema - handled via using groq's tool calling params. # else: model supports native json_schema, let response_format pass through + web_search_options = non_default_params.pop("web_search_options", None) optional_params = super().map_openai_params(non_default_params, optional_params, model, drop_params) + if web_search_options is None: + return optional_params + + if web_search_options: + verbose_logger.info( + "Groq web search enabled; ignoring unsupported web_search_options fields: %s", + sorted(web_search_options), + ) + if self._is_compound_model(model): + return optional_params + if not any(tool.get("type") == "browser_search" for tool in optional_params.get("tools") or ()): + optional_params = self._add_tools_to_optional_params( + optional_params=optional_params, + tools=[{"type": "browser_search"}], # mutable-ok: request tools must be json dicts in a list + ) return optional_params @@ -274,8 +310,34 @@ class GroqChatConfig(OpenAILikeChatConfig): original_service_tier=getattr(model_response, "service_tier") ) setattr(model_response, "service_tier", mapped_service_tier) + self._add_web_search_usage(model_response=model_response) return model_response + def _add_web_search_usage(self, model_response: ModelResponse) -> None: + usage = getattr(model_response, "usage", None) + if usage is None: + return + actions = self._executed_tool_actions(model_response) + searches = actions.count("browser.search") + actions.count("browser_search") + opens = actions.count("browser.open") + if searches == 0 and opens == 0: + return + usage.server_tool_use = ServerToolUse(web_search_requests=searches, browser_open_requests=opens) + + @staticmethod + def _executed_tool_actions(model_response: ModelResponse) -> tuple[str | None, ...]: + try: + return tuple( + identity.name or identity.type + for choice in model_response.choices + for identity in _EXECUTED_TOOLS_ADAPTER.validate_python( + getattr(getattr(choice, "message", None), "executed_tools", None) or () + ) + ) + except ValidationError as e: + verbose_logger.info("Groq executed_tools entries did not match the expected shape; not billed: %s", e) + return () + def _map_groq_service_tier(self, original_service_tier: str | None) -> Literal["auto", "default", "flex"]: """ Ensure groq service tier is OpenAI compatible. diff --git a/litellm/llms/groq/cost_calculator.py b/litellm/llms/groq/cost_calculator.py new file mode 100644 index 00000000000..c31dea0fd4d --- /dev/null +++ b/litellm/llms/groq/cost_calculator.py @@ -0,0 +1,27 @@ +""" +Groq-specific cost helpers. + +Groq bills the built-in browser tool per executed action +(https://groq.com/pricing): `browser.search` at $5 per 1k and +`browser.open` at $1 per 1k. The Groq chat transformation counts both +action kinds off `executed_tools` into `usage.server_tool_use`. +""" + +from typing import TYPE_CHECKING + +from litellm.constants import GROQ_BROWSER_VISIT_WEBSITE_COST_PER_CALL +from litellm.types.utils import Usage + +if TYPE_CHECKING: + from litellm.types.utils import ModelInfo + + +def cost_per_web_search_request(usage: Usage, model_info: "ModelInfo") -> float: + search_costs = model_info.get("search_context_cost_per_query") + cost_per_search = search_costs.get("search_context_size_medium", 0.0) if search_costs else 0.0 + server_tool_use = getattr(usage, "server_tool_use", None) + if server_tool_use is None: + return 0.0 + searches = server_tool_use.web_search_requests or 0 + opens = server_tool_use.browser_open_requests or 0 + return searches * cost_per_search + opens * GROQ_BROWSER_VISIT_WEBSITE_COST_PER_CALL diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 848af54cbcf..749663775ea 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -25681,6 +25681,11 @@ "max_tokens": 32766, "mode": "chat", "output_cost_per_token": 6e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.005, + "search_context_size_low": 0.005, + "search_context_size_medium": 0.005 + }, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, @@ -25697,6 +25702,11 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.005, + "search_context_size_low": 0.005, + "search_context_size_medium": 0.005 + }, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, @@ -25713,6 +25723,11 @@ "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.005, + "search_context_size_low": 0.005, + "search_context_size_medium": 0.005 + }, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index c63f4d53572..902253ca3a4 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1587,6 +1587,7 @@ class PromptTokensDetailsWrapper( class ServerToolUse(BaseModel): web_search_requests: Optional[int] = None tool_search_requests: Optional[int] = None + browser_open_requests: Optional[int] = None def __getitem__(self, key: str) -> Optional[int]: if key not in self.__class__.model_fields: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 56e0391f419..8a8bbdad8f1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25756,6 +25756,11 @@ "max_tokens": 32766, "mode": "chat", "output_cost_per_token": 6e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.005, + "search_context_size_low": 0.005, + "search_context_size_medium": 0.005 + }, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, @@ -25772,6 +25777,11 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.005, + "search_context_size_low": 0.005, + "search_context_size_medium": 0.005 + }, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, @@ -25788,6 +25798,11 @@ "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.005, + "search_context_size_low": 0.005, + "search_context_size_medium": 0.005 + }, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, diff --git a/tests/test_litellm/llms/groq/__init__.py b/tests/test_litellm/llms/groq/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/groq/chat/__init__.py b/tests/test_litellm/llms/groq/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/groq/chat/test_groq_chat_transformation.py b/tests/test_litellm/llms/groq/chat/test_groq_chat_transformation.py new file mode 100644 index 00000000000..b2ba919ac7f --- /dev/null +++ b/tests/test_litellm/llms/groq/chat/test_groq_chat_transformation.py @@ -0,0 +1,247 @@ +import logging +from unittest.mock import patch + +import httpx +import pytest + +import litellm +from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( + StandardBuiltInToolCostTracking, +) +from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.llms.groq.chat.transformation import GroqChatConfig +from litellm.utils import get_optional_params + +WEB_SEARCH_MODELS = ( + "openai/gpt-oss-120b", + "openai/gpt-oss-20b", + "openai/gpt-oss-safeguard-20b", +) + +COMPOUND_MODELS = ("compound", "compound-mini", "groq/compound", "groq/compound-mini") + + +@pytest.fixture +def local_model_cost_map(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +class TestGroqWebSearchOptions: + @pytest.mark.parametrize("model", WEB_SEARCH_MODELS + COMPOUND_MODELS) + def test_supported_on_search_capable_models(self, model: str): + assert "web_search_options" in GroqChatConfig().get_supported_openai_params(model) + + def test_not_supported_on_other_models(self): + assert "web_search_options" not in GroqChatConfig().get_supported_openai_params("llama-3.3-70b-versatile") + + @pytest.mark.parametrize("web_search_options", [{"search_context_size": "high"}, {}]) + def test_translates_to_browser_search_tool(self, web_search_options: dict): + optional_params = get_optional_params( + model="openai/gpt-oss-20b", + custom_llm_provider="groq", + web_search_options=web_search_options, + ) + assert optional_params["tools"] == [{"type": "browser_search"}] + assert "web_search_options" not in optional_params + + def test_no_duplicate_browser_search_tool(self): + optional_params = get_optional_params( + model="openai/gpt-oss-20b", + custom_llm_provider="groq", + web_search_options={"search_context_size": "high"}, + tools=[{"type": "browser_search"}], + ) + assert optional_params["tools"] == [{"type": "browser_search"}] + + def test_caller_function_tools_preserved(self): + function_tool = { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, + }, + } + optional_params = get_optional_params( + model="openai/gpt-oss-20b", + custom_llm_provider="groq", + web_search_options={}, + tools=[function_tool], + ) + assert optional_params["tools"] == [function_tool, {"type": "browser_search"}] + + def test_unsupported_model_drops_param_with_drop_params(self): + optional_params = get_optional_params( + model="llama-3.3-70b-versatile", + custom_llm_provider="groq", + web_search_options={"search_context_size": "high"}, + drop_params=True, + ) + assert "web_search_options" not in optional_params + assert "tools" not in optional_params + + def test_unsupported_model_raises_without_drop_params(self): + with pytest.raises(litellm.UnsupportedParamsError): + get_optional_params( + model="llama-3.3-70b-versatile", + custom_llm_provider="groq", + web_search_options={"search_context_size": "high"}, + drop_params=False, + ) + + @pytest.mark.parametrize("model", COMPOUND_MODELS) + def test_compound_injects_no_tool(self, model: str): + optional_params = get_optional_params( + model=model, + custom_llm_provider="groq", + web_search_options={"search_context_size": "high"}, + ) + assert "web_search_options" not in optional_params + assert "tools" not in optional_params + + def test_ignored_fields_logged_as_info(self, caplog: pytest.LogCaptureFixture): + with caplog.at_level(logging.INFO, logger="LiteLLM"): + get_optional_params( + model="openai/gpt-oss-20b", + custom_llm_provider="groq", + web_search_options={"search_context_size": "high", "user_location": {"type": "approximate"}}, + ) + ignored_fields_records = tuple( + record + for record in caplog.records + if "search_context_size" in record.message and "user_location" in record.message + ) + assert len(ignored_fields_records) == 1 + assert ignored_fields_records[0].levelno == logging.INFO + assert "enabled" in ignored_fields_records[0].message + + def test_empty_options_log_nothing(self, caplog: pytest.LogCaptureFixture): + with caplog.at_level(logging.INFO, logger="LiteLLM"): + get_optional_params( + model="openai/gpt-oss-20b", + custom_llm_provider="groq", + web_search_options={}, + ) + assert not [record for record in caplog.records if "web_search_options" in record.message] + + +def _searched_groq_response(executed_tools: list | None) -> dict: + return { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "openai/gpt-oss-20b", + "service_tier": "auto", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Top headline: example", + **({"executed_tools": executed_tools} if executed_tools is not None else {}), + }, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110}, + } + + +EXECUTED_TOOLS_THREE_SEARCHES_TWO_OPENS = [ + {"name": "browser.search", "type": "browser_search"}, + {"name": "browser.open", "type": "function"}, + {"name": "browser.search", "type": "browser_search"}, + {"type": "browser_search"}, + {"name": "browser.open", "type": "browser_search"}, + {"name": "browser.find", "type": "browser.find"}, +] + +EXECUTED_TOOLS_OPENS_ONLY = [ + {"name": "browser.open", "type": "browser.open"}, + {"name": "browser.open", "type": "function"}, + {"name": "browser.find", "type": "browser.find"}, +] + + +def _groq_completion_with_mocked_response(response_json: dict) -> litellm.ModelResponse: + client = HTTPHandler() + fake_response = httpx.Response( + status_code=200, + json=response_json, + request=httpx.Request("POST", "https://api.groq.com/openai/v1/chat/completions"), + ) + with patch.object(client, "post", return_value=fake_response): + return litellm.completion( + model="groq/openai/gpt-oss-20b", + messages=[{"role": "user", "content": "hi"}], + web_search_options={"search_context_size": "high"}, + api_key="fake-key", + client=client, + ) + + +class TestGroqWebSearchUsageSignal: + @pytest.mark.parametrize( + "executed_tools, expected_searches, expected_opens", + [ + (EXECUTED_TOOLS_THREE_SEARCHES_TWO_OPENS, 3, 2), + (EXECUTED_TOOLS_OPENS_ONLY, 0, 2), + ], + ) + def test_counts_actions_into_usage(self, executed_tools: list, expected_searches: int, expected_opens: int): + response = _groq_completion_with_mocked_response(_searched_groq_response(executed_tools)) + assert response.usage.server_tool_use.web_search_requests == expected_searches + assert response.usage.server_tool_use.browser_open_requests == expected_opens + + def test_no_signal_without_executed_tools(self): + response = _groq_completion_with_mocked_response(_searched_groq_response(None)) + assert getattr(response.usage, "server_tool_use", None) is None + + def test_malformed_executed_tools_skips_billing_without_breaking_response(self): + response = _groq_completion_with_mocked_response( + _searched_groq_response(["not-a-dict", {"name": {"nested": "junk"}}]) + ) + assert response.choices[0].message.content == "Top headline: example" + assert getattr(response.usage, "server_tool_use", None) is None + + def test_response_without_usage_is_left_untouched(self): + model_response = litellm.ModelResponse() + GroqChatConfig()._add_web_search_usage(model_response=model_response) + assert getattr(model_response, "usage", None) is None + + @pytest.mark.usefixtures("local_model_cost_map") + @pytest.mark.parametrize( + "executed_tools, expected_cost", + [ + (EXECUTED_TOOLS_THREE_SEARCHES_TWO_OPENS, 3 * 0.005 + 2 * 0.001), + (EXECUTED_TOOLS_OPENS_ONLY, 2 * 0.001), + ], + ) + def test_response_billed_per_action(self, executed_tools: list, expected_cost: float): + response = _groq_completion_with_mocked_response(_searched_groq_response(executed_tools)) + assert StandardBuiltInToolCostTracking.response_object_includes_web_search_call( + response_object=response, usage=response.usage + ) + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model="groq/openai/gpt-oss-20b", + response_object=response, + usage=response.usage, + custom_llm_provider="groq", + standard_built_in_tools_params={"web_search_options": {"search_context_size": "high"}}, + ) + assert cost == pytest.approx(expected_cost) + + +class TestGroqWebSearchCost: + @pytest.mark.usefixtures("local_model_cost_map") + @pytest.mark.parametrize("model", WEB_SEARCH_MODELS) + @pytest.mark.parametrize("search_context_size", ["low", "medium", "high"]) + def test_browser_search_priced_per_search(self, model: str, search_context_size: str): + cost = StandardBuiltInToolCostTracking.get_cost_for_web_search( + web_search_options={"search_context_size": search_context_size}, + model_info=litellm.get_model_info(model=model, custom_llm_provider="groq"), + ) + assert cost == 0.005 diff --git a/tests/test_litellm/llms/groq/test_groq_cost_calculator.py b/tests/test_litellm/llms/groq/test_groq_cost_calculator.py new file mode 100644 index 00000000000..bc3b8058b7f --- /dev/null +++ b/tests/test_litellm/llms/groq/test_groq_cost_calculator.py @@ -0,0 +1,56 @@ +import pytest + +from litellm.llms.groq.cost_calculator import cost_per_web_search_request +from litellm.types.utils import ModelInfo, ServerToolUse, Usage + +PRICED_MODEL_INFO = ModelInfo( + key="groq/openai/gpt-oss-20b", + litellm_provider="groq", + mode="chat", + search_context_cost_per_query={ + "search_context_size_low": 0.005, + "search_context_size_medium": 0.005, + "search_context_size_high": 0.005, + }, +) + + +def _usage_with_actions(searches: int | None, opens: int | None = None) -> Usage: + return Usage( + prompt_tokens=100, + completion_tokens=10, + total_tokens=110, + server_tool_use=ServerToolUse(web_search_requests=searches, browser_open_requests=opens), + ) + + +def test_bills_per_executed_search(): + cost = cost_per_web_search_request(usage=_usage_with_actions(7), model_info=PRICED_MODEL_INFO) + assert cost == pytest.approx(7 * 0.005) + + +def test_no_searches_costs_nothing(): + cost = cost_per_web_search_request(usage=_usage_with_actions(None), model_info=PRICED_MODEL_INFO) + assert cost == 0.0 + + +def test_missing_usage_signal_costs_nothing(): + usage = Usage(prompt_tokens=100, completion_tokens=10, total_tokens=110) + cost = cost_per_web_search_request(usage=usage, model_info=PRICED_MODEL_INFO) + assert cost == 0.0 + + +def test_missing_pricing_costs_nothing(): + unpriced = ModelInfo(key="groq/openai/gpt-oss-20b", litellm_provider="groq", mode="chat") + cost = cost_per_web_search_request(usage=_usage_with_actions(3), model_info=unpriced) + assert cost == 0.0 + + +def test_bills_visit_website_per_open(): + cost = cost_per_web_search_request(usage=_usage_with_actions(0, opens=15), model_info=PRICED_MODEL_INFO) + assert cost == pytest.approx(15 * 0.001) + + +def test_bills_searches_and_opens_together(): + cost = cost_per_web_search_request(usage=_usage_with_actions(2, opens=3), model_info=PRICED_MODEL_INFO) + assert cost == pytest.approx(2 * 0.005 + 3 * 0.001) From 6b3d4f2380543dd590f876388cf96d83c2e629eb Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 4 Aug 2026 09:24:29 -0700 Subject: [PATCH 077/124] feat(ui): add admin-configurable user banner (#35729) * feat(ui): add admin-configurable user banner Proxy admins can publish a markdown announcement that renders as a dismissible banner on every dashboard page for all authenticated users, editable from Admin Settings > UI Settings without a redeploy. Backed by new /get/user_banner and /update/user_banner endpoints persisting to the existing LiteLLM_UISettings table * fix(ui): re-surface dismissed banner on identical republish Stamp a server-side revision on every banner update and fold it into the client dismissal signature, so unpublishing and republishing the same message reaches users who dismissed the earlier run * fix(ui): stamp banner revision as an opaque uuid instead of a counter Two overlapping admin updates could read the same prior revision and both persist the same incremented value, letting an identical republish collide with a previously dismissed signature. A server-generated uuid per update makes every publication identity unique by construction with no read-modify-write * refactor(ui): drop the server-side banner cache Reads go straight to the single-row table; the dashboard already throttles fetches client-side, so the cache only added staleness windows under concurrent updates and multiple workers * refactor(ui): move banner storage behind a domain repository and drop the store_model_in_db gate UserBannerRepository owns the row shape instead of the endpoint reaching through the generic .table bridge, and publishing no longer depends on the unrelated STORE_MODEL_IN_DB flag; a connected database remains the only requirement --- litellm/proxy/_types.py | 1 + litellm/proxy/proxy_server.py | 4 + .../user_banner_endpoints.py | 130 +++++++++++++ .../repositories/user_banner_repository.py | 20 ++ .../proxy/auth/test_route_checks.py | 53 ++++++ .../test_user_banner_endpoints.py | 175 ++++++++++++++++++ .../admin-panel/_components/AdminPanel.tsx | 8 +- .../hooks/userBanner/useUpdateUserBanner.ts | 19 ++ .../hooks/userBanner/useUserBanner.ts | 21 +++ .../src/app/(dashboard)/layout.test.tsx | 4 + .../src/app/(dashboard)/layout.tsx | 3 + .../UserBannerSettings.test.tsx | 85 +++++++++ .../UserBannerSettings/UserBannerSettings.tsx | 152 +++++++++++++++ .../src/components/UserBanner.test.tsx | 102 ++++++++++ .../src/components/UserBanner.tsx | 71 +++++++ .../src/components/networking.tsx | 23 +++ .../src/components/shared/Alert.tsx | 4 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 151 +++++++++++++++ 18 files changed, 1025 insertions(+), 1 deletion(-) create mode 100644 litellm/proxy/ui_crud_endpoints/user_banner_endpoints.py create mode 100644 litellm/repositories/user_banner_repository.py create mode 100644 tests/test_litellm/proxy/ui_crud_endpoints/test_user_banner_endpoints.py create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/userBanner/useUpdateUserBanner.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/userBanner/useUserBanner.ts create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/UserBannerSettings/UserBannerSettings.test.tsx create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/UserBannerSettings/UserBannerSettings.tsx create mode 100644 ui/litellm-dashboard/src/components/UserBanner.test.tsx create mode 100644 ui/litellm-dashboard/src/components/UserBanner.tsx diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 5063bc790b5..e03d2a562e6 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -558,6 +558,7 @@ class LiteLLMRoutes(enum.Enum): "/models", "/v1/models", "/sso/get/ui_settings", + "/get/user_banner", ] # NOTE: ROUTES ONLY FOR MASTER KEY - only the Master Key should be able to Reset Spend diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1aeb8814585..6b2ef736c5b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -538,6 +538,9 @@ from litellm.proxy.types_utils.utils import get_instance_fn from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( router as ui_crud_endpoints_router, ) +from litellm.proxy.ui_crud_endpoints.user_banner_endpoints import ( + router as user_banner_endpoints_router, +) from litellm.proxy.utils import ( PrismaClient, ProxyLogging, @@ -16538,6 +16541,7 @@ app.include_router(callback_management_endpoints_router) app.include_router(debugging_endpoints_router) app.include_router(rust_control_plane_router) app.include_router(ui_crud_endpoints_router) +app.include_router(user_banner_endpoints_router) app.include_router(team_callback_router) app.include_router(budget_management_router) app.include_router(model_management_router) diff --git a/litellm/proxy/ui_crud_endpoints/user_banner_endpoints.py b/litellm/proxy/ui_crud_endpoints/user_banner_endpoints.py new file mode 100644 index 00000000000..cccd72ec7fd --- /dev/null +++ b/litellm/proxy/ui_crud_endpoints/user_banner_endpoints.py @@ -0,0 +1,130 @@ +import asyncio +import json +from typing import Annotated, Literal + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field, ValidationError, model_validator + +from litellm._uuid import uuid4 +from litellm.proxy._types import LitellmTableNames, LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.repositories.user_banner_repository import USER_BANNER_ROW_ID, UserBannerRepository + +router = APIRouter() + +USER_BANNER_MAX_MESSAGE_LENGTH = 4000 + +UserBannerSeverity = Literal["info", "warning", "error"] + + +class UserBannerUpdate(BaseModel): + enabled: bool = Field( + default=False, + description="If true, the banner is shown to all authenticated dashboard users.", + ) + message: str = Field( + default="", + max_length=USER_BANNER_MAX_MESSAGE_LENGTH, + description="Banner text shown to dashboard users. Markdown is supported.", + ) + severity: UserBannerSeverity = Field( + default="info", + description="Visual style of the banner.", + ) + + @model_validator(mode="after") + def _require_message_when_enabled(self) -> "UserBannerUpdate": + if self.enabled and not self.message.strip(): + raise ValueError("message must be non-empty when the banner is enabled") + return self + + +class UserBanner(UserBannerUpdate): + revision: str = Field( + default="", + description=( + "Server-stamped opaque publish identity; a fresh value is generated on every " + "update so clients re-surface dismissed banners on republish." + ), + ) + + +class UpdateUserBannerResponse(BaseModel): + message: str + banner: UserBanner + + +def parse_user_banner(raw_settings: object) -> UserBanner: + if raw_settings is None: + return UserBanner() + try: + parsed = json.loads(raw_settings) if isinstance(raw_settings, str) else raw_settings + return UserBanner.model_validate(parsed) + except (json.JSONDecodeError, ValidationError): + return UserBanner() + + +@router.get( + "/get/user_banner", + tags=["UI Settings"], # mutable-ok: FastAPI's route decorator only accepts a list + dependencies=[Depends(user_api_key_auth)], # mutable-ok: FastAPI's route decorator only accepts a list + response_model=UserBanner, +) +async def get_user_banner() -> UserBanner: + """ + Get the admin-published dashboard banner. + Readable by any authenticated user; rendered on every dashboard page. + """ + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + return UserBanner() + + raw_settings = await UserBannerRepository(prisma_client).get_raw_settings() + return parse_user_banner(raw_settings) + + +@router.patch( + "/update/user_banner", + tags=["UI Settings"], # mutable-ok: FastAPI's route decorator only accepts a list + response_model=UpdateUserBannerResponse, +) +async def update_user_banner( + banner_update: UserBannerUpdate, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> UpdateUserBannerResponse: + """ + Publish, edit, or unpublish the dashboard banner. + Only proxy admins are allowed to modify it. + """ + from litellm.proxy.proxy_server import create_config_audit_log, prisma_client + + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException(status_code=403, detail="Only proxy admins can update the user banner.") + + if prisma_client is None: + raise HTTPException(status_code=500, detail="Database not connected. Please connect a database.") + + repository = UserBannerRepository(prisma_client) + before = parse_user_banner(await repository.get_raw_settings()) + banner = UserBanner( + enabled=banner_update.enabled, + message=banner_update.message, + severity=banner_update.severity, + revision=uuid4().hex, + ) + + await repository.upsert_settings(json.dumps(banner.model_dump())) + + asyncio.create_task( + create_config_audit_log( + param_name=USER_BANNER_ROW_ID, + action="updated", + before_value=before.model_dump(), + after_value=banner.model_dump(), + user_api_key_dict=user_api_key_dict, + table_name=LitellmTableNames.UI_SETTINGS_TABLE_NAME, + ) + ) + + return UpdateUserBannerResponse(message="User banner updated successfully", banner=banner) diff --git a/litellm/repositories/user_banner_repository.py b/litellm/repositories/user_banner_repository.py new file mode 100644 index 00000000000..3ca7e7ceb1c --- /dev/null +++ b/litellm/repositories/user_banner_repository.py @@ -0,0 +1,20 @@ +from litellm.repositories.table_repositories import PrismaTableRepository + +USER_BANNER_ROW_ID = "user_banner" + + +class UserBannerRepository(PrismaTableRepository): + table_name = "litellm_uisettings" + + async def get_raw_settings(self) -> object: + db_record = await self.table.find_unique( + where={"id": USER_BANNER_ROW_ID} # mutable-ok: prisma filters are plain dicts + ) + return db_record.ui_settings if db_record is not None else None + + async def upsert_settings(self, payload: str) -> None: + row = {"id": USER_BANNER_ROW_ID, "ui_settings": payload} # mutable-ok: prisma rows are plain dicts + await self.table.upsert( + where={"id": USER_BANNER_ROW_ID}, # mutable-ok: prisma filters are plain dicts + data={"create": row, "update": {"ui_settings": payload}}, # mutable-ok: prisma payloads are plain dicts + ) diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 1426876783b..87f5187b5a1 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -86,6 +86,59 @@ def test_compliance_routes_open_to_non_admin_roles(role, route): ) +@pytest.mark.parametrize( + "role", + [ + LitellmUserRoles.INTERNAL_USER.value, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value, + ], +) +def test_user_banner_read_open_to_non_admin_roles(role): + """The dashboard banner renders for every authenticated user, so the read + route must be reachable by non-admin roles.""" + user_obj = LiteLLM_UserTable( + user_id="test_user", + user_email="test@example.com", + user_role=role, + ) + valid_token = UserAPIKeyAuth(user_id="test_user", user_role=role) + request = MagicMock(spec=Request) + request.query_params = {} + + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=role, + route="/get/user_banner", + request=request, + valid_token=valid_token, + request_data={}, + ) + + +def test_user_banner_update_rejected_for_non_admin(): + """Publishing the banner stays admin-only at the route layer.""" + user_obj = LiteLLM_UserTable( + user_id="test_user", + user_email="test@example.com", + user_role=LitellmUserRoles.INTERNAL_USER.value, + ) + valid_token = UserAPIKeyAuth(user_id="test_user", user_role=LitellmUserRoles.INTERNAL_USER.value) + request = MagicMock(spec=Request) + request.query_params = {} + + with pytest.raises(Exception) as exc_info: + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=LitellmUserRoles.INTERNAL_USER.value, + route="/update/user_banner", + request=request, + valid_token=valid_token, + request_data={}, + ) + + assert "Route=/update/user_banner" in str(exc_info.value) + + def test_proxy_admin_viewer_config_update_route_rejected(): """Test that proxy admin viewer users are rejected when trying to call /config/update""" diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_user_banner_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_user_banner_endpoints.py new file mode 100644 index 00000000000..5d4875073fd --- /dev/null +++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_user_banner_endpoints.py @@ -0,0 +1,175 @@ +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi.testclient import TestClient + +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.proxy_server import app + +client = TestClient(app) + +PUBLISHED_BANNER = { + "enabled": True, + "message": "**Scheduled maintenance** tonight at 10 PM UTC. See [status](https://status.example.com).", + "severity": "warning", + "revision": "1f2e3d4c5b6a79881f2e3d4c5b6a7988", +} +PUBLISH_BODY = {k: v for k, v in PUBLISHED_BANNER.items() if k != "revision"} +DISABLED_BANNER = {"enabled": False, "message": "", "severity": "info", "revision": ""} + + +def _auth_override(role: LitellmUserRoles): + async def override() -> UserAPIKeyAuth: + return UserAPIKeyAuth(api_key="sk-test", user_id="test-user", user_role=role) + + return override + + +@pytest.fixture +def admin_auth(): + app.dependency_overrides[user_api_key_auth] = _auth_override(LitellmUserRoles.PROXY_ADMIN) + yield + app.dependency_overrides.pop(user_api_key_auth, None) + + +@pytest.fixture +def internal_user_auth(): + app.dependency_overrides[user_api_key_auth] = _auth_override(LitellmUserRoles.INTERNAL_USER) + yield + app.dependency_overrides.pop(user_api_key_auth, None) + + +@pytest.fixture +def mock_audit_log(monkeypatch): + audit_mock = AsyncMock(return_value=None) + monkeypatch.setattr("litellm.proxy.proxy_server.create_config_audit_log", audit_mock) + return audit_mock + + +def _mock_prisma(monkeypatch, record=None): + mock_prisma = MagicMock() + mock_prisma.db.litellm_uisettings.find_unique = AsyncMock(return_value=record) + mock_prisma.db.litellm_uisettings.upsert = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + return mock_prisma + + +class TestGetUserBanner: + def test_requires_auth(self, monkeypatch): + _mock_prisma(monkeypatch) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-1234") + response = client.get("/get/user_banner") + assert response.status_code in (401, 403) + + def test_defaults_when_no_record(self, admin_auth, monkeypatch): + _mock_prisma(monkeypatch, record=None) + response = client.get("/get/user_banner") + assert response.status_code == 200 + assert response.json() == DISABLED_BANNER + + def test_returns_persisted_record(self, internal_user_auth, monkeypatch): + record = SimpleNamespace(ui_settings=json.dumps(PUBLISHED_BANNER)) + _mock_prisma(monkeypatch, record=record) + response = client.get("/get/user_banner") + assert response.status_code == 200 + assert response.json() == PUBLISHED_BANNER + + @pytest.mark.parametrize( + "raw", + [ + "not valid json", + json.dumps({"enabled": True, "message": "hi", "severity": "bogus"}), + json.dumps({"enabled": True, "message": ""}), + ], + ) + def test_corrupt_record_falls_back_to_disabled(self, admin_auth, monkeypatch, raw): + record = SimpleNamespace(ui_settings=raw) + _mock_prisma(monkeypatch, record=record) + response = client.get("/get/user_banner") + assert response.status_code == 200 + assert response.json() == DISABLED_BANNER + + def test_no_database_returns_disabled_banner(self, admin_auth, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + response = client.get("/get/user_banner") + assert response.status_code == 200 + assert response.json() == DISABLED_BANNER + + +class TestUpdateUserBanner: + def test_rejects_non_admin(self, internal_user_auth, monkeypatch): + mock_prisma = _mock_prisma(monkeypatch) + response = client.patch("/update/user_banner", json=PUBLISH_BODY) + assert response.status_code == 403 + mock_prisma.db.litellm_uisettings.upsert.assert_not_awaited() + + def test_persists_and_round_trips(self, admin_auth, monkeypatch, mock_audit_log): + mock_prisma = _mock_prisma(monkeypatch, record=None) + + response = client.patch("/update/user_banner", json=PUBLISH_BODY) + assert response.status_code == 200 + saved = response.json()["banner"] + assert {k: saved[k] for k in PUBLISH_BODY} == PUBLISH_BODY + assert saved["revision"] != "" + + upsert_kwargs = mock_prisma.db.litellm_uisettings.upsert.await_args.kwargs + assert upsert_kwargs["where"] == {"id": "user_banner"} + persisted_payload = upsert_kwargs["data"]["create"]["ui_settings"] + assert json.loads(persisted_payload) == saved + assert json.loads(upsert_kwargs["data"]["update"]["ui_settings"]) == saved + + mock_prisma.db.litellm_uisettings.find_unique = AsyncMock( + return_value=SimpleNamespace(ui_settings=persisted_payload) + ) + read_back = client.get("/get/user_banner") + assert read_back.status_code == 200 + assert read_back.json() == saved + + def test_republish_same_content_gets_fresh_revision(self, admin_auth, monkeypatch, mock_audit_log): + _mock_prisma(monkeypatch, record=None) + + first = client.patch("/update/user_banner", json=PUBLISH_BODY).json()["banner"]["revision"] + second = client.patch("/update/user_banner", json=PUBLISH_BODY).json()["banner"]["revision"] + assert first != "" + assert second != "" + assert first != second + + def test_client_supplied_revision_is_ignored(self, admin_auth, monkeypatch, mock_audit_log): + _mock_prisma(monkeypatch, record=None) + response = client.patch("/update/user_banner", json={**PUBLISH_BODY, "revision": "spoofed"}) + assert response.status_code == 200 + saved_revision = response.json()["banner"]["revision"] + assert saved_revision != "spoofed" + assert saved_revision != "" + + def test_unpublish_with_empty_message_is_allowed(self, admin_auth, monkeypatch, mock_audit_log): + _mock_prisma(monkeypatch, record=SimpleNamespace(ui_settings=json.dumps(PUBLISHED_BANNER))) + response = client.patch( + "/update/user_banner", + json={"enabled": False, "message": "", "severity": "info"}, + ) + assert response.status_code == 200 + saved = response.json()["banner"] + assert {k: saved[k] for k in ("enabled", "message", "severity")} == { + "enabled": False, + "message": "", + "severity": "info", + } + assert saved["revision"] not in ("", PUBLISHED_BANNER["revision"]) + + @pytest.mark.parametrize( + "payload", + [ + {"enabled": True, "message": "hi", "severity": "critical"}, + {"enabled": True, "message": " ", "severity": "info"}, + {"enabled": True, "message": "x" * 4001, "severity": "info"}, + ], + ) + def test_rejects_invalid_payloads(self, admin_auth, monkeypatch, payload): + mock_prisma = _mock_prisma(monkeypatch) + response = client.patch("/update/user_banner", json=payload) + assert response.status_code == 422 + mock_prisma.db.litellm_uisettings.upsert.assert_not_awaited() diff --git a/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx index 611efd6a588..6af9d65b994 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx @@ -24,6 +24,7 @@ import SCIMConfig from "@/components/SCIM"; import LoggingSettings from "@/components/Settings/AdminSettings/LoggingSettings/LoggingSettings"; import SSOSettings from "@/components/Settings/AdminSettings/SSOSettings/SSOSettings"; import UISettings from "@/components/Settings/AdminSettings/UISettings/UISettings"; +import UserBannerSettings from "@/components/Settings/AdminSettings/UserBannerSettings/UserBannerSettings"; import HashicorpVault from "@/components/Settings/AdminSettings/HashicorpVault/HashicorpVault"; import PluginSettings from "@/components/Settings/AdminSettings/PluginSettings/PluginSettings"; import SSOModals from "@/components/SSOModals"; @@ -362,7 +363,12 @@ const AdminPanel: React.FC = ({ proxySettings }) => { ), - children: , + children: ( +
+ + +
+ ), }, { key: "logging-settings", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/userBanner/useUpdateUserBanner.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/userBanner/useUpdateUserBanner.ts new file mode 100644 index 00000000000..b4441f0d09d --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/userBanner/useUpdateUserBanner.ts @@ -0,0 +1,19 @@ +import { updateUserBanner, UserBannerUpdate } from "@/components/networking"; +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { userBannerKeys } from "./useUserBanner"; + +export const useUpdateUserBanner = (accessToken: string | null) => { + const queryClient = useQueryClient(); + + return useMutation({ + mutationFn: async (banner: UserBannerUpdate) => { + if (!accessToken) { + throw new Error("Access token is required"); + } + return await updateUserBanner(accessToken, banner); + }, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: userBannerKeys.all }); + }, + }); +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/userBanner/useUserBanner.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/userBanner/useUserBanner.ts new file mode 100644 index 00000000000..1e407790fe3 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/userBanner/useUserBanner.ts @@ -0,0 +1,21 @@ +import { getUserBanner, UserBanner } from "@/components/networking"; +import { useQuery, UseQueryOptions } from "@tanstack/react-query"; +import { createQueryKeys } from "../common/queryKeysFactory"; + +export const userBannerKeys = createQueryKeys("userBanner"); + +export const useUserBanner = (accessToken: string | null) => { + const queryOptions: UseQueryOptions = { + queryKey: userBannerKeys.list({}), + queryFn: async () => { + if (!accessToken) { + throw new Error("Access token is required"); + } + return await getUserBanner(accessToken); + }, + enabled: Boolean(accessToken), + staleTime: 60 * 1000, + gcTime: 5 * 60 * 1000, + }; + return useQuery(queryOptions); +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx index f08258900eb..7973855ebd4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx @@ -29,6 +29,10 @@ vi.mock("@/components/LicenseExpiryBanner", () => ({ LicenseExpiryBanner: () => null, })); +vi.mock("@/components/UserBanner", () => ({ + UserBanner: () => null, +})); + vi.mock("@/contexts/ThemeContext", () => ({ ThemeProvider: ({ children }: { children: React.ReactNode }) => <>{children}, })); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx index d92aae30c67..fb3a4db58f7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx @@ -10,6 +10,7 @@ import SidebarProvider from "@/app/(dashboard)/components/SidebarProvider"; import { useRouter, useSearchParams, usePathname } from "next/navigation"; import { DebugWarningBanner } from "@/components/DebugWarningBanner"; import { LicenseExpiryBanner } from "@/components/LicenseExpiryBanner"; +import { UserBanner } from "@/components/UserBanner"; import { MIGRATED_PAGES, migratedHref, legacyPageHref, legacyKeyForPathname } from "@/utils/migratedPages"; import { PluginModeProvider, usePluginMode } from "@/contexts/PluginModeContext"; import { createApiClient } from "@/lib/http/client"; @@ -120,6 +121,7 @@ function DashboardShell({ children }: { children: React.ReactNode }) { +
@@ -142,6 +144,7 @@ function DashboardShell({ children }: { children: React.ReactNode }) { +
{children}
diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/UserBannerSettings/UserBannerSettings.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UserBannerSettings/UserBannerSettings.test.tsx new file mode 100644 index 00000000000..9a0e2e72311 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UserBannerSettings/UserBannerSettings.test.tsx @@ -0,0 +1,85 @@ +import { renderWithProviders, screen } from "../../../../../tests/test-utils"; +import { fireEvent } from "@testing-library/react"; +import { vi } from "vitest"; +import UserBannerSettings from "./UserBannerSettings"; +import { UserBanner } from "@/components/networking"; + +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: vi.fn(() => ({ accessToken: "token" })), +})); + +vi.mock("@/app/(dashboard)/hooks/userBanner/useUserBanner", () => ({ + useUserBanner: vi.fn(), +})); + +vi.mock("@/app/(dashboard)/hooks/userBanner/useUpdateUserBanner", () => ({ + useUpdateUserBanner: vi.fn(), +})); + +vi.mock("@/components/molecules/notifications_manager", () => ({ + default: { success: vi.fn(), fromBackend: vi.fn() }, +})); + +import { useUserBanner } from "@/app/(dashboard)/hooks/userBanner/useUserBanner"; +import { useUpdateUserBanner } from "@/app/(dashboard)/hooks/userBanner/useUpdateUserBanner"; + +const publishedBanner: UserBanner = { + enabled: true, + message: "**Maintenance** tonight at 10 PM UTC.", + severity: "warning", + revision: "rev-a", +}; + +const mockHooks = (banner: UserBanner | undefined, mutate = vi.fn()) => { + vi.mocked(useUserBanner).mockReturnValue({ data: banner, isLoading: false } as any); + vi.mocked(useUpdateUserBanner).mockReturnValue({ mutate, isPending: false } as any); + return mutate; +}; + +describe("UserBannerSettings", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("seeds the form from the persisted banner", () => { + mockHooks(publishedBanner); + renderWithProviders(); + expect(screen.getByLabelText("Message")).toHaveValue(publishedBanner.message); + expect(screen.getByRole("switch", { name: "Publish user banner" })).toHaveAttribute("data-checked"); + }); + + it("shows a live markdown preview with the selected severity icon", () => { + mockHooks(publishedBanner); + const { container } = renderWithProviders(); + expect(screen.getByText("Maintenance")).toBeInTheDocument(); + expect(container.querySelector(".lucide-triangle-alert")).toBeInTheDocument(); + }); + + it("saves the edited draft", () => { + const mutate = mockHooks(publishedBanner); + renderWithProviders(); + fireEvent.change(screen.getByLabelText("Message"), { target: { value: "New announcement" } }); + fireEvent.click(screen.getByRole("button", { name: "Save banner" })); + expect(mutate).toHaveBeenCalledWith( + { enabled: true, message: "New announcement", severity: "warning" }, + expect.anything(), + ); + }); + + it("blocks saving a published banner with an empty message", () => { + const mutate = mockHooks(publishedBanner); + renderWithProviders(); + fireEvent.change(screen.getByLabelText("Message"), { target: { value: " " } }); + expect(screen.getByText("Add a message before publishing.")).toBeInTheDocument(); + const saveButton = screen.getByRole("button", { name: "Save banner" }); + fireEvent.click(saveButton); + expect(mutate).not.toHaveBeenCalled(); + }); + + it("allows unpublishing without a message", () => { + const mutate = mockHooks({ enabled: false, message: "", severity: "info", revision: "" }); + renderWithProviders(); + fireEvent.click(screen.getByRole("button", { name: "Save banner" })); + expect(mutate).toHaveBeenCalledWith({ enabled: false, message: "", severity: "info" }, expect.anything()); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/UserBannerSettings/UserBannerSettings.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UserBannerSettings/UserBannerSettings.tsx new file mode 100644 index 00000000000..83207f89160 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UserBannerSettings/UserBannerSettings.tsx @@ -0,0 +1,152 @@ +"use client"; + +import React, { useState } from "react"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { useUpdateUserBanner } from "@/app/(dashboard)/hooks/userBanner/useUpdateUserBanner"; +import { useUserBanner } from "@/app/(dashboard)/hooks/userBanner/useUserBanner"; +import NotificationManager from "@/components/molecules/notifications_manager"; +import { UserBanner, UserBannerSeverity, UserBannerUpdate } from "@/components/networking"; +import { Alert, AlertDescription } from "@/components/shared/Alert"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Label } from "@/components/ui/label"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Switch } from "@/components/ui/switch"; +import { Textarea } from "@/components/ui/textarea"; +import { SEVERITY_ICONS, UserBannerMarkdown } from "@/components/UserBanner"; +import { Skeleton } from "@/components/ui/skeleton"; + +const SEVERITY_LABELS: Record = { + info: "Info", + warning: "Warning", + error: "Error", +}; + +const EMPTY_BANNER: UserBanner = { enabled: false, message: "", severity: "info", revision: "" }; + +export default function UserBannerSettings() { + const { accessToken } = useAuthorized(); + const { data: banner, isLoading } = useUserBanner(accessToken); + const { mutate: saveBanner, isPending } = useUpdateUserBanner(accessToken); + const persisted = banner ?? EMPTY_BANNER; + + return ( + + ); +} + +interface UserBannerSettingsFormProps { + persisted: UserBanner; + isLoading: boolean; + isPending: boolean; + saveBanner: ReturnType["mutate"]; +} + +function UserBannerSettingsForm({ persisted, isLoading, isPending, saveBanner }: UserBannerSettingsFormProps) { + const [draft, setDraft] = useState({ + enabled: persisted.enabled, + message: persisted.message, + severity: persisted.severity, + }); + + const messageMissing = draft.enabled && draft.message.trim() === ""; + + const handleSave = () => { + saveBanner(draft, { + onSuccess: () => { + NotificationManager.success("User banner updated successfully"); + }, + onError: (error) => { + NotificationManager.fromBackend(error); + }, + }); + }; + + return ( + + + User Banner + + Publish an announcement to all dashboard users. Markdown is supported; the banner appears below the header on + every page until you unpublish it. Users can dismiss it, and it reappears whenever the content changes. + + + + {isLoading ? ( + + ) : ( +
+
+ setDraft({ ...draft, enabled: checked })} + aria-label="Publish user banner" + /> + +
+ +
+ +