From fb9d30224b1d885fac17708144417b092fa6b51f Mon Sep 17 00:00:00 2001 From: milan Date: Fri, 24 Jul 2026 19:04:22 +0000 Subject: [PATCH 001/123] fix(responses): forward proxy client headers to the provider Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/responses/main.py | 7 +++ litellm/responses/utils.py | 15 +++++++ .../test_responses_api_request_body.py | 45 +++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 944cf58df1c..aa5da40a02a 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -927,6 +927,13 @@ def responses( _is_async = kwargs.pop("aresponses", False) is True use_chat_completions_api = _pop_use_chat_completions_api_kw(kwargs) + client_headers = kwargs.get("headers") + extra_headers = ResponsesAPIRequestUtils.merge_client_forwarded_headers( + extra_headers=extra_headers, + client_headers=client_headers if isinstance(client_headers, dict) else None, + ) + local_vars["extra_headers"] = extra_headers + # Convert text_format to text parameter if provided text = ResponsesAPIRequestUtils.convert_text_format_to_text_param(text_format=text_format, text=text) if text is not None: diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 12c890ec91d..096103b3fae 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -89,6 +89,21 @@ class ResponsesAPIRequestUtils: ) return [*merged_input] + @staticmethod + def merge_client_forwarded_headers( + extra_headers: dict[str, Any] | None, + client_headers: dict[str, str] | None, + ) -> dict[str, Any] | None: + """ + Merge headers forwarded by the proxy (`headers` kwarg, set when + `forward_client_headers_to_llm_api` is enabled) into `extra_headers`. + + `extra_headers` wins on conflicts, since it is set explicitly by the caller. + """ + if not client_headers: + return extra_headers + return {**client_headers, **(extra_headers or {})} + @staticmethod def _check_valid_arg( supported_params: Optional[List[str]], diff --git a/tests/test_litellm/responses/test_responses_api_request_body.py b/tests/test_litellm/responses/test_responses_api_request_body.py index 44dfa240d42..0d067a99b97 100644 --- a/tests/test_litellm/responses/test_responses_api_request_body.py +++ b/tests/test_litellm/responses/test_responses_api_request_body.py @@ -259,3 +259,48 @@ async def test_aresponses_bedrock_mantle_service_tier_raises_without_drop_params mock_post.assert_not_called() assert "drop_params" in str(excinfo.value) assert "priority" in str(excinfo.value) + + +async def _aresponses_and_get_request_headers(**request_kwargs) -> dict: + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_headers_test", "gpt-4o"), 200) + + await litellm.aresponses( + model="openai/gpt-4o", + api_key="fake-api-key", + input="hi", + **request_kwargs, + ) + + mock_post.assert_called_once() + return dict(mock_post.call_args.kwargs["headers"]) + + +@pytest.mark.asyncio +async def test_aresponses_forwards_client_headers_kwarg_to_provider(): + """ + The proxy passes client headers it forwards (`forward_client_headers_to_llm_api`) + as a `headers` kwarg; those must reach the provider request. + """ + request_headers = await _aresponses_and_get_request_headers(headers={"x-my-new-header": "hello-from-client"}) + + assert request_headers["x-my-new-header"] == "hello-from-client" + + +@pytest.mark.asyncio +async def test_aresponses_merges_client_headers_with_extra_headers(): + """ + A `headers` kwarg and an explicit `extra_headers` are merged, with + `extra_headers` winning on conflicts. + """ + request_headers = await _aresponses_and_get_request_headers( + headers={"x-my-new-header": "hello-from-client", "x-shared": "from-client"}, + extra_headers={"x-explicit": "from-caller", "x-shared": "from-caller"}, + ) + + assert request_headers["x-my-new-header"] == "hello-from-client" + assert request_headers["x-explicit"] == "from-caller" + assert request_headers["x-shared"] == "from-caller" From 7716e47519426bda2ba55791e9518a0a5459a209 Mon Sep 17 00:00:00 2001 From: milan Date: Fri, 24 Jul 2026 20:16:25 +0000 Subject: [PATCH 002/123] fix(responses): compare forwarded header names case-insensitively Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/responses/utils.py | 9 ++++++++- .../responses/test_responses_api_request_body.py | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 096103b3fae..25a355edbc1 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -99,10 +99,17 @@ class ResponsesAPIRequestUtils: `forward_client_headers_to_llm_api` is enabled) into `extra_headers`. `extra_headers` wins on conflicts, since it is set explicitly by the caller. + Header names are compared case-insensitively, as HTTP defines them. """ if not client_headers: return extra_headers - return {**client_headers, **(extra_headers or {})} + if not extra_headers: + return dict(client_headers) + explicit_names = frozenset(name.lower() for name in extra_headers) + return { + **{name: value for name, value in client_headers.items() if name.lower() not in explicit_names}, + **extra_headers, + } @staticmethod def _check_valid_arg( diff --git a/tests/test_litellm/responses/test_responses_api_request_body.py b/tests/test_litellm/responses/test_responses_api_request_body.py index 0d067a99b97..1c217d1a67f 100644 --- a/tests/test_litellm/responses/test_responses_api_request_body.py +++ b/tests/test_litellm/responses/test_responses_api_request_body.py @@ -304,3 +304,18 @@ async def test_aresponses_merges_client_headers_with_extra_headers(): assert request_headers["x-my-new-header"] == "hello-from-client" assert request_headers["x-explicit"] == "from-caller" assert request_headers["x-shared"] == "from-caller" + + +@pytest.mark.asyncio +async def test_aresponses_client_header_conflict_is_case_insensitive(): + """ + HTTP header names are case-insensitive, so a differently cased client header + must not survive alongside the explicit `extra_headers` value. + """ + request_headers = await _aresponses_and_get_request_headers( + headers={"X-Shared": "from-client"}, + extra_headers={"x-shared": "from-caller"}, + ) + + assert [name for name in request_headers if name.lower() == "x-shared"] == ["x-shared"] + assert request_headers["x-shared"] == "from-caller" From 35592323e24a3740ed2531a26d71145903ac9f2d Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 30 Jul 2026 16:12:08 -0700 Subject: [PATCH 003/123] fix(tag-management): drop unsupported prisma select kwarg from key lookup /tag/list returned HTTP 500 for every internal user with "LiteLLM_VerificationTokenActions.find_many() got an unexpected keyword argument 'select'". The non-admin branch scopes the tag list to keys owned by the caller, and that lookup passed select={"token": True}; prisma-client-py 0.11.0 has no select kwarg on find_many, so the call raised TypeError and the handler's except block turned it into a 500. Since the Admin UI calls /tag/list on load, Tags was broken for every non-admin user. /tag/daily/activity shares the same helper and was failing the same way The kwarg is dropped rather than replaced; the generated client has no projection API, and a user's key set is small enough that selecting all columns is not worth working around The reason this shipped green is that the existing test asserted the call was made with select={"token": True} against an AsyncMock, which accepts any keyword. The verification-token table double now binds each call against the real find_many signature, so an unsupported kwarg raises the same TypeError production does --- .../tag_management_endpoints.py | 2 - .../test_tag_management_endpoints.py | 109 +++++++++++++++--- 2 files changed, 91 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/management_endpoints/tag_management_endpoints.py b/litellm/proxy/management_endpoints/tag_management_endpoints.py index 47a8670e26f..ac53f254981 100644 --- a/litellm/proxy/management_endpoints/tag_management_endpoints.py +++ b/litellm/proxy/management_endpoints/tag_management_endpoints.py @@ -96,7 +96,6 @@ class _VerificationTokenTableClient(Protocol): async def find_many( self, where: Mapping[str, object] | None = None, - select: Mapping[str, object] | None = None, ) -> "Sequence[PrismaVerificationToken]": ... @@ -157,7 +156,6 @@ async def _get_internal_user_api_keys( key_records = await _table(VerificationTokenRepository(prisma_client)).find_many( where={"user_id": user_id}, - select={"token": True}, ) user_api_keys.update(key_record.token for key_record in key_records if getattr(key_record, "token", None)) diff --git a/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py index 76ba0e3dc67..9927c56b847 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py @@ -1,11 +1,13 @@ +import inspect import json import os import sys -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional import pytest from fastapi import HTTPException from fastapi.testclient import TestClient +from prisma.actions import LiteLLM_VerificationTokenActions sys.path.insert( 0, os.path.abspath("../../../..") @@ -21,6 +23,28 @@ from litellm.types.tag_management import TagDeleteRequest, TagInfoRequest, TagNe client = TestClient(app) +class FakeVerificationTokenTable: + """Stand-in for ``prisma_client.db.litellm_verificationtoken``. + + ``AsyncMock`` swallows any keyword argument, so a plain mock cannot catch a + call that the generated prisma client would reject at runtime. This double + binds every call against the real ``find_many`` signature, so passing an + unsupported kwarg (e.g. ``select``) raises the same ``TypeError`` the proxy + surfaces as an HTTP 500. + """ + + def __init__(self, records: List[Any]): + self._records = records + self.calls: List[Dict[str, Any]] = [] + + async def find_many(self, **kwargs: Any) -> List[Any]: + inspect.signature(LiteLLM_VerificationTokenActions.find_many).bind( + self, **kwargs + ) + self.calls.append(kwargs) + return self._records + + @pytest.mark.asyncio async def test_create_and_get_tag(): """ @@ -380,6 +404,7 @@ async def test_list_tags_no_dynamic_tags(): app.dependency_overrides.clear() +@pytest.mark.asyncio async def test_internal_user_list_tags_only_returns_tags_used_by_their_keys(): """ Internal users can view tag usage, but the tag list must be scoped to tags @@ -404,9 +429,8 @@ async def test_internal_user_list_tags_only_returns_tags_used_by_their_keys(): owned_key_record = Mock() owned_key_record.token = "owned-key" - mock_db.litellm_verificationtoken.find_many = AsyncMock( - return_value=[owned_key_record] - ) + fake_token_table = FakeVerificationTokenTable([owned_key_record]) + mock_db.litellm_verificationtoken = fake_token_table mock_db.litellm_dailytagspend.group_by = AsyncMock( return_value=[ @@ -446,10 +470,9 @@ async def test_internal_user_list_tags_only_returns_tags_used_by_their_keys(): "stored-owned-tag", "dynamic-owned-tag", ] - mock_db.litellm_verificationtoken.find_many.assert_awaited_once_with( - where={"user_id": "internal-user-123"}, - select={"token": True}, - ) + assert fake_token_table.calls == [ + {"where": {"user_id": "internal-user-123"}} + ] mock_db.litellm_dailytagspend.group_by.assert_awaited_once_with( by=["tag"], where={ @@ -468,6 +491,54 @@ async def test_internal_user_list_tags_only_returns_tags_used_by_their_keys(): app.dependency_overrides.clear() +@pytest.mark.asyncio +async def test_internal_user_list_tags_does_not_500_on_unsupported_prisma_kwarg(): + """ + Regression: /tag/list returned 500 for every internal user because the + non-admin branch looked up the caller's keys with + ``find_many(select={"token": True})``, and the generated prisma client has no + ``select`` kwarg. This reproduces the reported case exactly: a freshly created + internal user with no tag spend yet, which must get an empty 200 rather than + "LiteLLM_VerificationTokenActions.find_many() got an unexpected keyword + argument 'select'". + """ + from unittest.mock import AsyncMock, Mock + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + mock_user_auth = UserAPIKeyAuth( + api_key="new-user-key", + user_id="brand-new-internal-user", + user_role=LitellmUserRoles.INTERNAL_USER, + ) + app.dependency_overrides[user_api_key_auth] = lambda: mock_user_auth + + try: + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + mock_db = Mock() + mock_prisma.db = mock_db + + key_record = Mock() + key_record.token = "new-user-key" + fake_token_table = FakeVerificationTokenTable([key_record]) + mock_db.litellm_verificationtoken = fake_token_table + + mock_db.litellm_dailytagspend.group_by = AsyncMock(return_value=[]) + mock_db.litellm_tagtable.find_many = AsyncMock(return_value=[]) + + response = client.get( + "/tag/list", headers={"Authorization": "Bearer new-user-key"} + ) + + assert response.status_code == 200, response.text + assert response.json() == [] + assert fake_token_table.calls == [ + {"where": {"user_id": "brand-new-internal-user"}} + ] + finally: + app.dependency_overrides.clear() + + @pytest.mark.asyncio async def test_list_tags_with_date_range_filters_dynamic_tags(): """ @@ -537,9 +608,8 @@ async def test_internal_user_tag_daily_activity_is_scoped_to_their_keys(): owned_key_record = Mock() owned_key_record.token = "owned-key" - mock_db.litellm_verificationtoken.find_many = AsyncMock( - return_value=[owned_key_record] - ) + fake_token_table = FakeVerificationTokenTable([owned_key_record]) + mock_db.litellm_verificationtoken = fake_token_table mock_get_daily_activity.return_value = "daily-activity-response" result = await get_tag_daily_activity( @@ -549,6 +619,7 @@ async def test_internal_user_tag_daily_activity_is_scoped_to_their_keys(): ) assert result == "daily-activity-response" + assert fake_token_table.calls == [{"where": {"user_id": "internal-user-123"}}] mock_get_daily_activity.assert_awaited_once() assert mock_get_daily_activity.await_args.kwargs["api_key"] == ["owned-key"] @@ -583,9 +654,8 @@ async def test_internal_user_tag_daily_activity_rejects_unowned_api_key_filter() owned_key_record = Mock() owned_key_record.token = "owned-key" - mock_db.litellm_verificationtoken.find_many = AsyncMock( - return_value=[owned_key_record] - ) + fake_token_table = FakeVerificationTokenTable([owned_key_record]) + mock_db.litellm_verificationtoken = fake_token_table result = await get_tag_daily_activity( start_date="2025-01-01", end_date="2025-01-31", @@ -593,6 +663,7 @@ async def test_internal_user_tag_daily_activity_rejects_unowned_api_key_filter() user_api_key_dict=mock_user_auth, ) + assert fake_token_table.calls == [{"where": {"user_id": "internal-user-123"}}] assert result.results == [] assert result.metadata.total_spend == 0 assert result.metadata.total_api_requests == 0 @@ -626,7 +697,8 @@ async def test_internal_user_tag_daily_activity_scopes_to_current_key_without_us ): mock_db = Mock() mock_prisma.db = mock_db - mock_db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + fake_token_table = FakeVerificationTokenTable([]) + mock_db.litellm_verificationtoken = fake_token_table mock_get_daily_activity.return_value = "daily-activity-response" result = await get_tag_daily_activity( @@ -636,7 +708,7 @@ async def test_internal_user_tag_daily_activity_scopes_to_current_key_without_us ) assert result == "daily-activity-response" - mock_db.litellm_verificationtoken.find_many.assert_not_awaited() + assert fake_token_table.calls == [] mock_get_daily_activity.assert_awaited_once() assert mock_get_daily_activity.await_args.kwargs["api_key"] == [ "current-owned-key" @@ -669,7 +741,8 @@ async def test_internal_user_tag_daily_activity_without_any_scoped_keys_returns_ ): mock_db = Mock() mock_prisma.db = mock_db - mock_db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + fake_token_table = FakeVerificationTokenTable([]) + mock_db.litellm_verificationtoken = fake_token_table result = await get_tag_daily_activity( start_date="2025-01-01", @@ -680,7 +753,7 @@ async def test_internal_user_tag_daily_activity_without_any_scoped_keys_returns_ assert result.results == [] assert result.metadata.total_spend == 0 assert result.metadata.total_api_requests == 0 - mock_db.litellm_verificationtoken.find_many.assert_not_awaited() + assert fake_token_table.calls == [] mock_get_daily_activity.assert_not_awaited() From d9f53258e9f3a3281706b4692a42c660f01e4e21 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 31 Jul 2026 09:52:46 -0700 Subject: [PATCH 004/123] refactor(test): tighten typing on the tag list verification token double Replaces the double's Any annotations and List/Dict aliases with concrete types, matching the equivalent double in the tool policy tests: kwargs are object, records are Sequence[Mock] held as a tuple, and the call log is list[dict[str, object]]. Behaviour is unchanged; the double still binds every call against the real generated prisma action signature, verified by reintroducing the select kwarg and watching the regression tests fail --- .../test_tag_management_endpoints.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py index 9927c56b847..4fe1b54694f 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py @@ -2,7 +2,8 @@ import inspect import json import os import sys -from typing import Any, Dict, List, Optional +from collections.abc import Sequence +from typing import Optional import pytest from fastapi import HTTPException @@ -13,7 +14,7 @@ sys.path.insert( 0, os.path.abspath("../../../..") ) # Adds the parent directory to the system path -from unittest.mock import patch +from unittest.mock import Mock, patch import litellm from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth @@ -33,11 +34,11 @@ class FakeVerificationTokenTable: surfaces as an HTTP 500. """ - def __init__(self, records: List[Any]): - self._records = records - self.calls: List[Dict[str, Any]] = [] + def __init__(self, records: Sequence[Mock]): + self._records = tuple(records) + self.calls: list[dict[str, object]] = [] - async def find_many(self, **kwargs: Any) -> List[Any]: + async def find_many(self, **kwargs: object) -> tuple[Mock, ...]: inspect.signature(LiteLLM_VerificationTokenActions.find_many).bind( self, **kwargs ) From c0cab45350dc2f7ace66fdc375c92cb4e672d65f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 31 Jul 2026 19:14:42 +0000 Subject: [PATCH 005/123] chore(typing): replace Any kwargs unpacking with validated model parsing Swap `Model(**payload)` for `Model.model_validate(payload)` at the seams where the payload comes back untyped, so basedpyright stops widening every target field to Any. None of the models involved override `__init__`, so validation goes through the same core validator either way. Also route UserRepository through its own typed helpers (find_many, update, find_by_id) instead of the raw Prisma table, drop the redundant `_to_model` override signature, and call generate_key_helper_fn with explicit arguments in the SSO callback rather than splatting an untyped dict. Whole-tree basedpyright: reportAny 21481 -> 20834, reportExplicitAny 7258 -> 7252, with every other rule unchanged or lower. --- basedpyright-code-budget.json | 8 +-- .../proxy/hooks/managed_files.py | 8 +-- .../litellm_core_utils/streaming_handler.py | 4 +- .../llms/azure/responses/transformation.py | 4 +- litellm/llms/custom_httpx/llm_http_handler.py | 10 ++-- .../llms/manus/responses/transformation.py | 4 +- .../hooks/user_management_event_hooks.py | 13 ++--- litellm/proxy/management_endpoints/ui_sso.py | 26 +++++---- .../openai_files_endpoints/common_utils.py | 2 +- litellm/repositories/user_repository.py | 57 ++++++++----------- litellm/router.py | 12 ++-- .../complexity_router/complexity_router.py | 2 +- ruff-strict-budget.json | 4 +- type-discipline-budget.json | 6 +- 14 files changed, 76 insertions(+), 84 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 65142091712..43df27ea2e2 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 31903 + "limit": 31256 }, "reportArgumentType": { "limit": 2645 @@ -24,7 +24,7 @@ "limit": 42 }, "reportExplicitAny": { - "limit": 10214 + "limit": 10208 }, "reportFunctionMemberAccess": { "limit": 11 @@ -33,7 +33,7 @@ "limit": 227 }, "reportIncompatibleMethodOverride": { - "limit": 78 + "limit": 77 }, "reportIncompatibleVariableOverride": { "limit": 12 @@ -99,7 +99,7 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 45366 + "limit": 45357 }, "reportUnknownLambdaType": { "limit": 113 diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 8821736d0ff..d57c1a78f3d 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -215,7 +215,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) if result: - return LiteLLM_ManagedFileTable(**result) + return LiteLLM_ManagedFileTable.model_validate(result) ## CHECK DB db_object = await self.prisma_client.db.litellm_managedfiletable.find_first( @@ -223,7 +223,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) if db_object: - return LiteLLM_ManagedFileTable(**db_object.model_dump()) + return LiteLLM_ManagedFileTable.model_validate(db_object.model_dump()) return None async def delete_unified_file_id( @@ -349,7 +349,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if isinstance(batch.file_object, str) else batch.file_object ) - batch_obj = LiteLLMBatch(**batch_data) + batch_obj = LiteLLMBatch.model_validate(batch_data) batch_obj.id = batch.unified_object_id batch_objects.append(batch_obj) @@ -382,7 +382,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): "flat_model_file_ids": {"hasSome": model_object_ids}, } ) - return [OpenAIFileObject(**file_object.file_object) for file_object in file_ids] + return [OpenAIFileObject.model_validate(file_object.file_object) for file_object in file_ids] async def check_managed_file_id_access( self, data: Dict, user_api_key_dict: UserAPIKeyAuth diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 60dbf7c644a..0ac22b5bb1e 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -131,8 +131,8 @@ class CustomStreamWrapper: self.sent_last_chunk = False self._stream_created_time: float = time.time() - litellm_params: GenericLiteLLMParams = GenericLiteLLMParams( - **self.logging_obj.model_call_details.get("litellm_params", {}) + litellm_params: GenericLiteLLMParams = GenericLiteLLMParams.model_validate( + dict(**self.logging_obj.model_call_details.get("litellm_params", {})) ) self.merge_reasoning_content_in_choices: bool = litellm_params.merge_reasoning_content_in_choices or False self.sent_first_thinking_block = False diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index d0b0dbb070d..1a860cca5a9 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -66,7 +66,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): ) # Create ResponseReasoningItem object from the item data - reasoning_item = ResponseReasoningItem(**item_data) + reasoning_item = ResponseReasoningItem.model_validate(item_data) # Convert back to dict with exclude_none=True to exclude None fields dict_reasoning_item = reasoning_item.model_dump(exclude_none=True) @@ -346,4 +346,4 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIError raise AzureOpenAIError(message=raw_response.text, status_code=raw_response.status_code) - return ResponsesAPIResponse(**raw_response_json) + return ResponsesAPIResponse.model_validate(raw_response_json) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ec1301e5923..d6acbaae434 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6184,10 +6184,12 @@ class BaseLLMHTTPHandler: import websockets from websockets.asyncio.client import ClientConnection - litellm_params = GenericLiteLLMParams( - api_base=api_base, - api_key=api_key, - **kwargs, + litellm_params = GenericLiteLLMParams.model_validate( + { + "api_base": api_base, + "api_key": api_key, + **kwargs, + } ) headers = responses_api_provider_config.validate_environment( headers={}, diff --git a/litellm/llms/manus/responses/transformation.py b/litellm/llms/manus/responses/transformation.py index 0db53f90330..e6fbbac0563 100644 --- a/litellm/llms/manus/responses/transformation.py +++ b/litellm/llms/manus/responses/transformation.py @@ -217,7 +217,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): raw_response_json["id"] = f"unknown-{uuid.uuid4().hex[:8]}" try: - response = ResponsesAPIResponse(**raw_response_json) + response = ResponsesAPIResponse.model_validate(raw_response_json) except Exception: verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct") response = ResponsesAPIResponse.model_construct(**raw_response_json) @@ -305,7 +305,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): raw_response_json["id"] = f"unknown-{uuid.uuid4().hex[:8]}" try: - response = ResponsesAPIResponse(**raw_response_json) + response = ResponsesAPIResponse.model_validate(raw_response_json) except Exception: verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct") response = ResponsesAPIResponse.model_construct(**raw_response_json) diff --git a/litellm/proxy/hooks/user_management_event_hooks.py b/litellm/proxy/hooks/user_management_event_hooks.py index 6122f0594e8..d8cfae5dab0 100644 --- a/litellm/proxy/hooks/user_management_event_hooks.py +++ b/litellm/proxy/hooks/user_management_event_hooks.py @@ -6,8 +6,6 @@ import asyncio from datetime import datetime, timezone from typing import Optional -from pydantic import BaseModel - import litellm from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid @@ -16,7 +14,6 @@ from litellm.proxy._types import ( CommonProxyErrors, LiteLLM_AuditLogs, Litellm_EntityType, - LiteLLM_UserTable, LitellmTableNames, NewUserRequest, NewUserResponse, @@ -58,11 +55,11 @@ class UserManagementEventHooks: try: if prisma_client is None: raise Exception(CommonProxyErrors.db_not_connected_error.value) - user_row: BaseModel = await UserRepository(prisma_client).table.find_first( - where={"user_id": response.user_id} - ) - - user_row_litellm_typed = LiteLLM_UserTable(**user_row.model_dump(exclude_none=True)) + if response.user_id is None: + raise Exception("no user_id returned for the newly created user") + user_row_litellm_typed = await UserRepository(prisma_client).find_by_id(response.user_id) + if user_row_litellm_typed is None: + raise Exception(f"no user row found for user_id={response.user_id}") asyncio.create_task( UserManagementEventHooks.create_internal_user_audit_log( user_id=user_row_litellm_typed.user_id, diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 8682b61f910..110e5883485 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -3137,7 +3137,7 @@ class SSOAuthenticationHandler: _default_team_params = deepcopy(litellm.default_team_params) _new_team_request = team_request.model_dump() _new_team_request.update(_default_team_params) - team_request = NewTeamRequest(**_new_team_request) + team_request = NewTeamRequest.model_validate(_new_team_request) return team_request @staticmethod @@ -3271,14 +3271,6 @@ class SSOAuthenticationHandler: # User might not be already created on first generation of key # But if it is, we want their models preferences - default_ui_key_values: Dict[str, Any] = { - "duration": LITELLM_UI_SESSION_DURATION, - "key_max_budget": litellm.max_ui_session_budget, - "aliases": {}, - "config": {}, - "spend": 0, - "team_id": "litellm-dashboard", - } user_defined_values: Optional[SSOUserDefinedValues] = None if user_custom_sso is not None: @@ -3338,10 +3330,20 @@ class SSOAuthenticationHandler: verbose_proxy_logger.info(f"user_defined_values for creating ui key: {user_defined_values}") - default_ui_key_values.update(user_defined_values) - default_ui_key_values["request_type"] = "key" response = await generate_key_helper_fn( - **default_ui_key_values, # type: ignore + request_type="key", + duration=LITELLM_UI_SESSION_DURATION, + key_max_budget=litellm.max_ui_session_budget, + aliases={}, + config={}, + spend=0, + team_id="litellm-dashboard", + models=user_defined_values["models"], + user_id=user_defined_values["user_id"], + user_email=user_defined_values["user_email"], + user_role=user_defined_values["user_role"], + max_budget=user_defined_values["max_budget"], + budget_duration=user_defined_values["budget_duration"], table_name="key", ) diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index b2e36188681..2960b031cd3 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -1035,7 +1035,7 @@ async def get_batch_from_database( if isinstance(db_batch_object.file_object, str) else db_batch_object.file_object ) - response = LiteLLMBatch(**batch_data) + response = LiteLLMBatch.model_validate(batch_data) response.id = batch_id # The stored batch object has the raw provider input_file_id. Resolve to unified ID. diff --git a/litellm/repositories/user_repository.py b/litellm/repositories/user_repository.py index f0b5dfd8bc5..e93991463f8 100644 --- a/litellm/repositories/user_repository.py +++ b/litellm/repositories/user_repository.py @@ -3,61 +3,56 @@ User repository for database operations on LiteLLM_UserTable. """ import json -from typing import Any, Dict, List, Optional, Type +from typing import Any, Dict, List, Mapping, Optional, Type from litellm.models.user import LiteLLM_UserTable -from litellm.repositories.base_repository import BaseRepository +from litellm.repositories.base_repository import BaseRepository, DbRecord, record_to_dict + +_JSON_ENCODED_COLUMNS = frozenset({"metadata", "model_spend", "model_max_budget"}) class UserRepository(BaseRepository[LiteLLM_UserTable]): """Repository for user database operations.""" @property - def table(self) -> Any: + def table(self) -> Any: # any-ok: Prisma table actions are reached through the untyped client wrapper return self.prisma_client.db.litellm_usertable @property def model_class(self) -> Type[LiteLLM_UserTable]: return LiteLLM_UserTable - def _to_model(self, record: Any) -> Optional[LiteLLM_UserTable]: + def _to_model(self, record: Optional[DbRecord]) -> Optional[LiteLLM_UserTable]: """Convert a database record to a User model.""" if record is None: return None - data = record.dict() if hasattr(record, "dict") else dict(record) + return LiteLLM_UserTable.model_validate( + { + column: json.loads(value) if column in _JSON_ENCODED_COLUMNS and isinstance(value, str) else value + for column, value in record_to_dict(record).items() + } + ) - json_fields = ["metadata", "model_spend", "model_max_budget"] - for field in json_fields: - if isinstance(data.get(field), str): - data[field] = json.loads(data[field]) - - return LiteLLM_UserTable(**data) - - async def find_by_id(self, user_id: str, id_field: str = "user_id") -> Optional[LiteLLM_UserTable]: - return await super().find_by_id(user_id, id_field) + async def find_by_id(self, id_value: str, id_field: str = "user_id") -> Optional[LiteLLM_UserTable]: + return await super().find_by_id(id_value, id_field) async def find_by_email(self, user_email: str) -> Optional[LiteLLM_UserTable]: """Find a user by email.""" - records = await self.table.find_many(where={"user_email": user_email}) - if records: - return self._to_model(records[0]) - return None + records = await self.find_many(where={"user_email": user_email}) + return records[0] if records else None async def find_by_sso_id(self, sso_user_id: str) -> Optional[LiteLLM_UserTable]: """Find a user by SSO ID.""" - record = await self.table.find_unique(where={"sso_user_id": sso_user_id}) - return self._to_model(record) + return await self.find_by_id(sso_user_id, id_field="sso_user_id") async def find_by_organization_id(self, organization_id: str) -> List[LiteLLM_UserTable]: """Find all users in an organization.""" - records = await self.table.find_many(where={"organization_id": organization_id}) - return self._to_model_list(records) + return await self.find_many(where={"organization_id": organization_id}) async def find_by_team_id(self, team_id: str) -> List[LiteLLM_UserTable]: """Find all users in a team.""" - records = await self.table.find_many(where={"teams": {"has": team_id}}) - return self._to_model_list(records) + return await self.find_many(where={"teams": {"has": team_id}}) async def count_billable_users(self) -> int: """Number of users that count toward the license seat limit. @@ -86,7 +81,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): max_budget: Optional[float] = None, user_email: Optional[str] = None, models: Optional[List[str]] = None, - metadata: Optional[Dict[str, Any]] = None, + metadata: Optional[Mapping[str, object]] = None, max_parallel_requests: Optional[int] = None, tpm_limit: Optional[int] = None, rpm_limit: Optional[int] = None, @@ -96,7 +91,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): object_permission_id: Optional[str] = None, ) -> LiteLLM_UserTable: """Create a new user.""" - data: Dict[str, Any] = {"user_id": user_id} + data: Dict[str, object] = {"user_id": user_id} if user_alias is not None: data["user_alias"] = user_alias if team_id is not None: @@ -149,7 +144,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): max_budget: Optional[float] = None, user_email: Optional[str] = None, models: Optional[List[str]] = None, - metadata: Optional[Dict[str, Any]] = None, + metadata: Optional[Mapping[str, object]] = None, max_parallel_requests: Optional[int] = None, tpm_limit: Optional[int] = None, rpm_limit: Optional[int] = None, @@ -159,7 +154,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): object_permission_id: Optional[str] = None, ) -> Optional[LiteLLM_UserTable]: """Update a user.""" - data: Dict[str, Any] = {} + data: Dict[str, object] = {} if user_alias is not None: data["user_alias"] = user_alias if team_id is not None: @@ -212,11 +207,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): if not await self.exists(user_id, id_field="user_id"): return None - record = await self.table.update( - where={"user_id": user_id}, - data={"teams": {"push": team_id}}, - ) - return self._to_model(record) + return await self.update(user_id, {"teams": {"push": team_id}}, id_field="user_id") async def remove_from_team(self, user_id: str, team_id: str) -> Optional[LiteLLM_UserTable]: """Remove a user from a team. diff --git a/litellm/router.py b/litellm/router.py index ac00cdbc2b0..399bfcdf3e1 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8613,9 +8613,9 @@ class Router: deployment = self.get_deployment(model_id=model_id) if deployment is None or self._is_deployment_blocked(deployment): return None - return CredentialLiteLLMParams(**deployment.litellm_params.model_dump(exclude_none=True)).model_dump( - exclude_none=True - ) + return CredentialLiteLLMParams.model_validate( + deployment.litellm_params.model_dump(exclude_none=True) + ).model_dump(exclude_none=True) def get_deployment_by_model_group_name(self, model_group_name: str) -> Optional[Deployment]: """ @@ -8755,9 +8755,9 @@ class Router: return None # Get basic credentials - credentials = CredentialLiteLLMParams(**deployment.litellm_params.model_dump(exclude_none=True)).model_dump( - exclude_none=True - ) + credentials = CredentialLiteLLMParams.model_validate( + deployment.litellm_params.model_dump(exclude_none=True) + ).model_dump(exclude_none=True) # Resolve litellm_credential_name to actual credentials if deployment.litellm_params.litellm_credential_name is not None: diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index b43fe0da4ca..1c6a83c6cd9 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -329,7 +329,7 @@ class ComplexityRouter(CustomLogger): # Parse config - always create a new instance to avoid singleton mutation if complexity_router_config: - self.config = ComplexityRouterConfig(**complexity_router_config) + self.config = ComplexityRouterConfig.model_validate(complexity_router_config) else: self.config = ComplexityRouterConfig() diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index dfdc4efe800..6fb3ed748b6 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -24,7 +24,7 @@ "limit": 130 }, "ANN401": { - "limit": 2010 + "limit": 2009 }, "ASYNC230": { "limit": 14 @@ -324,7 +324,7 @@ "limit": 879 }, "UP006": { - "limit": 12138 + "limit": 12135 }, "UP007": { "limit": 2526 diff --git a/type-discipline-budget.json b/type-discipline-budget.json index c9a1b59cc06..8d5161ec7ea 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 23253 + "limit": 23250 }, "LIT002": { - "limit": 27427 + "limit": 27425 }, "LIT003": { "limit": 292 @@ -24,6 +24,6 @@ "limit": 1004 }, "LIT009": { - "limit": 2474 + "limit": 2473 } } From 2258dc08aaa682f9a49484af685a91293ff12012 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 31 Jul 2026 19:26:04 +0000 Subject: [PATCH 006/123] test(repositories): lock in UserRepository JSON column decoding Covers the columns UserRepository._to_model decodes so a narrower or wider column set fails instead of silently changing what callers read back. --- .../repositories/test_repositories.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_litellm/repositories/test_repositories.py b/tests/test_litellm/repositories/test_repositories.py index c923b722991..8253308f393 100644 --- a/tests/test_litellm/repositories/test_repositories.py +++ b/tests/test_litellm/repositories/test_repositories.py @@ -1905,6 +1905,28 @@ class TestUserRepositoryExtended: ) assert updated.user_email == "new@example.com" + @pytest.mark.asyncio + async def test_find_by_id_decodes_only_the_json_encoded_columns(self, repo): + repo._prisma_client.db.litellm_usertable._records["user-json"] = { + "user_id": "user-json", + "user_email": "json@example.com", + "user_role": '{"not": "json"}', + "teams": [], + "models": [], + "metadata": '{"department": "engineering"}', + "model_spend": '{"gpt-4": 10.5}', + "model_max_budget": '{"gpt-4": 100.0}', + } + + user = await repo.find_by_id("user-json") + + assert user is not None + assert user.metadata == {"department": "engineering"} + assert user.model_spend == {"gpt-4": 10.5} + assert user.model_max_budget == {"gpt-4": 100.0} + assert user.user_email == "json@example.com" + assert user.user_role == '{"not": "json"}' + class TestProjectRepositoryExtended: @pytest.fixture From 26ace642be31d25c8826c27947fef76060ece375 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 31 Jul 2026 19:48:30 +0000 Subject: [PATCH 007/123] test(proxy): cover the user-created audit hook's database read-back The hook resolves the newly created user through UserRepository and builds the audit entry from that row. Pin both halves: the entry carries the persisted row's fields rather than the /user/new response, and a user id that resolves to nothing produces no entry at all. --- .../hooks/test_user_management_event_hooks.py | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 tests/test_litellm/proxy/hooks/test_user_management_event_hooks.py diff --git a/tests/test_litellm/proxy/hooks/test_user_management_event_hooks.py b/tests/test_litellm/proxy/hooks/test_user_management_event_hooks.py new file mode 100644 index 00000000000..102430ab985 --- /dev/null +++ b/tests/test_litellm/proxy/hooks/test_user_management_event_hooks.py @@ -0,0 +1,99 @@ +import asyncio +import json +import sys +from types import SimpleNamespace +from typing import Any, Dict, List, Optional +from unittest.mock import AsyncMock, patch + +import pytest + +import litellm +from litellm.proxy._types import NewUserRequest, NewUserResponse, UserAPIKeyAuth +from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks + + +class FakeUserTable: + def __init__(self, rows: List[Dict[str, Any]]): + self._rows = rows + + async def find_unique(self, where: Dict[str, Any]) -> Optional[Dict[str, Any]]: + return next( + (row for row in self._rows if all(row.get(key) == value for key, value in where.items())), + None, + ) + + +class FakePrismaClient: + def __init__(self, rows: List[Dict[str, Any]]): + self.db = SimpleNamespace(litellm_usertable=FakeUserTable(rows)) + + +async def _run_created_hook(prisma_client: FakePrismaClient, audit_log: AsyncMock) -> None: + proxy_server = SimpleNamespace( + prisma_client=prisma_client, + litellm_proxy_admin_name="admin-user", + ) + with ( + patch.dict(sys.modules, {"litellm.proxy.proxy_server": proxy_server}), + patch.object(litellm, "store_audit_logs", True), + patch( + "litellm.proxy.hooks.user_management_event_hooks.create_audit_log_for_update", + audit_log, + ), + patch.object( + UserManagementEventHooks, + "async_send_user_invitation_email", + AsyncMock(), + ), + ): + await UserManagementEventHooks.async_user_created_hook( + data=NewUserRequest(user_email="new@example.com", send_invite_email=False), + response=NewUserResponse( + user_id="user-1", + user_email="new@example.com", + key="sk-test", + ), + user_api_key_dict=UserAPIKeyAuth(user_id="admin-user", api_key="sk-admin"), + ) + await asyncio.sleep(0) + + +@pytest.mark.asyncio +async def test_created_hook_audit_logs_the_row_read_back_from_the_database(): + """The audit entry must describe the persisted user row, not the /user/new response.""" + prisma_client = FakePrismaClient( + [ + { + "user_id": "user-1", + "user_email": "new@example.com", + "user_role": "proxy_admin", + "models": ["gpt-4"], + "teams": ["team-a"], + "metadata": '{"source": "api"}', + } + ] + ) + audit_log = AsyncMock() + + await _run_created_hook(prisma_client, audit_log) + + audit_log.assert_awaited_once() + request_data = audit_log.await_args.kwargs["request_data"] + assert request_data.object_id == "user-1" + assert request_data.action == "created" + + updated_values = json.loads(request_data.updated_values) + assert updated_values["user_role"] == "proxy_admin" + assert updated_values["models"] == ["gpt-4"] + assert updated_values["teams"] == ["team-a"] + assert updated_values["metadata"] == {"source": "api"} + + +@pytest.mark.asyncio +async def test_created_hook_skips_the_audit_log_when_no_user_row_exists(): + """A user id that resolves to nothing must not produce an audit entry.""" + audit_log = AsyncMock() + + await _run_created_hook(FakePrismaClient([]), audit_log) + + audit_log.assert_not_awaited() From 6062ed7ff19f3c3747391298cf99f464790dc993 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 31 Jul 2026 19:57:12 +0000 Subject: [PATCH 008/123] fix(batches): encode public model group on background-created output file ids CheckBatchCost built unified output file ids with the provider model name, so key model-access checks resolved the file to e.g. gpt-5.5 and every GET /v1/files/{output_file_id}/content failed. Resolve the model group from the batch's managed input file, falling back to the deployment's model_name. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/common_utils/check_batch_cost.py | 33 +++- .../proxy_unit_tests/test_check_batch_cost.py | 183 +++++++++++++++++- 2 files changed, 213 insertions(+), 3 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index f209ab54f64..22f9f40ecd8 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -17,6 +17,7 @@ if TYPE_CHECKING: from litellm.proxy._types import LiteLLM_ManagedObjectTable from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.router import Router + from litellm.types.router import Deployment from litellm.types.utils import LiteLLMBatch @@ -281,6 +282,32 @@ class CheckBatchCost: return deployment_id return None + @classmethod + def _get_managed_file_model_name( + cls, + job: "LiteLLM_ManagedObjectTable", + deployment_info: "Deployment", + ) -> Optional[str]: + """ + Public model group name to encode as ``target_model_names`` on unified output file ids. + + Key model-access checks resolve a managed file id back to a model via its + ``target_model_names``, so this must be the model group the caller requested, never the + underlying provider model (e.g. ``gpt-5.5``), which no key is allowed to call. + """ + from litellm.proxy.openai_files_endpoints.common_utils import ( + convert_b64_uid_to_unified_uid, + get_models_from_unified_file_id, + ) + + input_file_id = cls._get_input_file_id(job) + target_model_names = ( + get_models_from_unified_file_id(convert_b64_uid_to_unified_uid(input_file_id)) if input_file_id else [] + ) + if target_model_names: + return ",".join(target_model_names) + return deployment_info.model_name or None + @staticmethod def _get_input_file_id(job: "LiteLLM_ManagedObjectTable") -> Optional[str]: import json @@ -406,6 +433,10 @@ class CheckBatchCost: managed_files_hook = self.proxy_logging_obj.get_proxy_hook("managed_files") if managed_files_hook is not None: from litellm.proxy._types import UserAPIKeyAuth + + managed_file_model_name = self._get_managed_file_model_name( + job=job, deployment_info=deployment_info + ) _minimal_auth = UserAPIKeyAuth( user_id=job.created_by or "default-user-id", team_id=getattr(job, "team_id", None), @@ -417,7 +448,7 @@ class CheckBatchCost: _unified_file_id = managed_files_hook.get_unified_output_file_id( output_file_id=_raw_file_id, model_id=model_id, - model_name=str(model_name) if model_name else deployment_info.model_name or None, + model_name=managed_file_model_name, ) await managed_files_hook.store_unified_file_id( file_id=_unified_file_id, diff --git a/tests/proxy_unit_tests/test_check_batch_cost.py b/tests/proxy_unit_tests/test_check_batch_cost.py index b822799fb40..a15abd023d8 100644 --- a/tests/proxy_unit_tests/test_check_batch_cost.py +++ b/tests/proxy_unit_tests/test_check_batch_cost.py @@ -674,12 +674,12 @@ class TestCheckBatchCost: mock_hook.get_unified_output_file_id.assert_any_call( output_file_id=raw_output_file_id, model_id="model-123", - model_name="gpt-5-mini", + model_name="gpt-5-batch", ) mock_hook.get_unified_output_file_id.assert_any_call( output_file_id=raw_error_file_id, model_id="model-123", - model_name="gpt-5-mini", + model_name="gpt-5-batch", ) assert mock_hook.store_unified_file_id.await_count == 2 # {raw_file_id: managed_file_id} for each store call @@ -1221,3 +1221,182 @@ class TestUnmanagedBatchCostFlagIsGeneralized: assert vertex_result == ("deploy-vertex", "8823717160934178816") assert bedrock_result == ("deploy-bedrock", TestUnmanagedBedrockRouting._ARN) + + +class TestManagedOutputFileIdEncodesPublicModelGroup: + """LIT-4964 regression: the unified output file id created by the background poller must + encode the public model group as ``target_model_names``, not the provider model. + + Key model-access checks resolve a managed file id back to a model via ``target_model_names``, + so encoding the provider model (e.g. ``gpt-5.5``) makes + ``GET /v1/files/{output_file_id}/content`` fail for every key. + """ + + _PUBLIC_MODEL_GROUP = "gpt-5-batch" + _RAW_OUTPUT_FILE_ID = "file-batch-output-abc123" + + @staticmethod + def _managed_input_file_id(model_group: str) -> str: + import base64 + + unified_id = ( + "litellm_proxy:application/octet-stream;unified_id,c4843482-b176-4901-8292-7523fd0f2c6e;" + f"target_model_names,{model_group}" + ) + return base64.urlsafe_b64encode(unified_id.encode()).decode().rstrip("=") + + def _job(self, input_file_id: str) -> MagicMock: + from litellm.types.utils import LiteLLMBatch + + job = MagicMock() + job.id = "job-lit-4964" + job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA==" + job.created_by = "user-1" + job.team_id = None + job.file_object = LiteLLMBatch( + id="batch-456", + completion_window="24h", + created_at=1, + endpoint="/v1/chat/completions", + input_file_id=input_file_id, + object="batch", + status="completed", + ).model_dump_json() + return job + + async def _run(self, job: MagicMock) -> str: + from litellm_enterprise.proxy.common_utils.check_batch_cost import ( + CheckBatchCost, + ) + from litellm.types.utils import LiteLLMBatch + from enterprise.litellm_enterprise.proxy.hooks.managed_files import ( + _PROXY_LiteLLMManagedFiles, + ) + + router = MagicMock() + router.get_deployment_credentials_with_provider = MagicMock( + return_value={"api_key": "sk-test"} + ) + deployment = MagicMock() + deployment.litellm_params.custom_llm_provider = "azure" + deployment.litellm_params.model = "azure/gpt-5.5" + deployment.model_name = self._PUBLIC_MODEL_GROUP + deployment.model_info.model_dump.return_value = {} + router.get_deployment = MagicMock(return_value=deployment) + + hook = MagicMock() + hook.get_unified_output_file_id = ( + lambda output_file_id, model_id, model_name: _PROXY_LiteLLMManagedFiles.get_unified_output_file_id( + None, output_file_id=output_file_id, model_id=model_id, model_name=model_name + ) + ) + hook.store_unified_file_id = AsyncMock() + proxy_logging_obj = MagicMock() + proxy_logging_obj.get_proxy_hook.return_value = hook + + prisma_client = MagicMock() + prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None) + + instance = CheckBatchCost( + proxy_logging_obj=proxy_logging_obj, + prisma_client=prisma_client, + llm_router=router, + ) + + response = LiteLLMBatch( + id="batch-456", + completion_window="24h", + created_at=1, + endpoint="/v1/chat/completions", + input_file_id=job.file_object, + object="batch", + status="completed", + ) + response.output_file_id = self._RAW_OUTPUT_FILE_ID + + file_content = MagicMock() + file_content.content = b'{"id":"req-1"}' + + with ( + patch( + "litellm.files.main.afile_content", + new_callable=AsyncMock, + return_value=file_content, + ), + patch( + "litellm.batches.batch_utils._get_file_content_as_dictionary", + return_value=[{"id": "req-1"}], + ), + patch( + "litellm.batches.batch_utils.calculate_batch_cost_and_usage", + new_callable=AsyncMock, + return_value=(0.01, {"prompt_tokens": 10}, ["gpt-5.5"]), + ), + patch("litellm.litellm_core_utils.litellm_logging.Logging") as logging_cls, + ): + logging_obj = MagicMock() + logging_obj.async_success_handler = AsyncMock() + logging_cls.return_value = logging_obj + + await instance._track_completed_batch_cost( + job=job, + response=response, + model_id="model-123", + batch_id="batch-456", + prom_logger=None, + ) + + return response.output_file_id + + @pytest.mark.asyncio + async def test_target_model_names_comes_from_input_file_not_provider_model(self): + from litellm.proxy.openai_files_endpoints.common_utils import ( + _is_base64_encoded_unified_file_id, + get_models_from_unified_file_id, + ) + + output_file_id = await self._run( + self._job(self._managed_input_file_id(self._PUBLIC_MODEL_GROUP)) + ) + + decoded = _is_base64_encoded_unified_file_id(output_file_id) + assert get_models_from_unified_file_id(decoded) == [self._PUBLIC_MODEL_GROUP] + assert "gpt-5.5" not in decoded + + @pytest.mark.asyncio + async def test_key_scoped_to_model_group_can_read_the_output_file(self): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.auth_checks import can_key_call_model + from litellm.proxy.auth.auth_utils import ( + _extract_models_from_managed_resource_id, + ) + + output_file_id = await self._run( + self._job(self._managed_input_file_id(self._PUBLIC_MODEL_GROUP)) + ) + + models = _extract_models_from_managed_resource_id(output_file_id, "file_id", None) + assert models == [self._PUBLIC_MODEL_GROUP] + assert ( + await can_key_call_model( + model=models[0], + llm_model_list=None, + valid_token=UserAPIKeyAuth( + api_key="sk-test", models=[self._PUBLIC_MODEL_GROUP] + ), + llm_router=None, + ) + is True + ) + + @pytest.mark.asyncio + async def test_falls_back_to_deployment_model_group_without_managed_input_file(self): + from litellm.proxy.openai_files_endpoints.common_utils import ( + _is_base64_encoded_unified_file_id, + get_models_from_unified_file_id, + ) + + output_file_id = await self._run(self._job("file-raw-provider-input")) + + decoded = _is_base64_encoded_unified_file_id(output_file_id) + assert get_models_from_unified_file_id(decoded) == [self._PUBLIC_MODEL_GROUP] From 594d0d7a0a5513d31e280811dd2eac9603d2f036 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 31 Jul 2026 17:03:57 -0700 Subject: [PATCH 009/123] feat(proxy)!: gate all mock testing request params behind a single config flag Handling of the client-supplied mock testing params was split across three places with different behavior for each. Three were dropped from every proxy request, two reached the router untouched, and a request that asked for a synthetic failure came back as an ordinary success with nothing to indicate that no failure had been injected Put all six behind one opt-in, general_settings. dangerously_allow_mock_testing_request_params, and reject rather than drop when it is unset, so a fallback drill cannot report a pass for a test that never ran. The rejection names the params it saw and the config key to set, which is also the answer for anyone following the older docs The flag is config-file only. It is deliberately absent from ConfigGeneralSettings, and that absence is what makes /config/update drop it on parse and /config/field/update reject it; the tests pin both so the field cannot be added back for tidiness without the reason surfacing. Enabling it logs a startup warning naming every param it unlocks BREAKING CHANGE: mock_timeout and mock_testing_rate_limit_error now require general_settings.dangerously_allow_mock_testing_request_params to be set in config.yaml. Previously they were accepted unconditionally --- litellm/proxy/proxy_server.py | 33 ++++ litellm/proxy/route_llm_request.py | 61 ++++++-- tests/test_litellm/proxy/test_proxy_server.py | 80 ++++++++++ .../proxy/test_route_llm_request.py | 148 +++++++++++++++--- 4 files changed, 291 insertions(+), 31 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a60ea2da019..06e9cc1fbee 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1041,6 +1041,8 @@ async def proxy_startup_event(app: FastAPI): redis_usage_cache=transaction_buffer_redis_cache, ) + ProxyStartupEvent._warn_if_mock_testing_params_enabled(general_settings=general_settings) + ## SEMANTIC TOOL FILTER ## # Read litellm_settings from config for semantic filter initialization try: @@ -7712,6 +7714,37 @@ class ProxyStartupEvent: proxy_logging_obj.startup_event(llm_router=llm_router, redis_usage_cache=redis_usage_cache) + @staticmethod + def _warn_if_mock_testing_params_enabled(general_settings: dict) -> None: + """Announce, loudly, that any caller may inject synthetic failures.""" + from litellm.proxy.route_llm_request import ( + GATED_MOCK_PARAM_NAMES, + MOCK_TESTING_CONFIG_KEY, + ) + + if general_settings.get(MOCK_TESTING_CONFIG_KEY, False) is not True: + return + + verbose_proxy_logger.warning( + "\n%s\n" + " DANGEROUS SETTING ENABLED\n" + " general_settings.%s = true\n" + "\n" + " Any caller with a valid key on this proxy can now inject synthetic\n" + " failures and latency into their own requests using these body params:\n" + "%s\n" + "\n" + " A request using them consumes a connection and a concurrency slot\n" + " without reaching a provider, and returns an error the caller chose.\n" + "\n" + " Intended for testing fallback chains. Do not leave enabled.\n" + "%s", + "=" * 72, + MOCK_TESTING_CONFIG_KEY, + "\n".join(f" {name}" for name in GATED_MOCK_PARAM_NAMES), + "=" * 72, + ) + @staticmethod def _validate_redis_transaction_buffer_config( general_settings: dict, diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 1f5aacc2115..aebdbca86ad 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -8,18 +8,24 @@ import litellm from litellm.proxy._types import UserAPIKeyAuth from litellm.router_utils.common_utils import _is_proxy_admin_request -# Router-internal mock_testing_* flag names — kept in sync with -# ``litellm.types.router.MockRouterTestingParams`` by the test -# ``test_mock_testing_kwarg_names_matches_dataclass``. Hardcoding (rather -# than deriving via ``dataclasses.fields(MockRouterTestingParams)`` at +# Client-supplied params that make the router or the call path fabricate a +# failure or a delay instead of calling the provider. The ``mock_testing_*`` +# names are kept in sync with ``litellm.types.router.MockRouterTestingParams`` +# by ``test_gated_mock_params_cover_mock_router_testing_params``. Hardcoding +# (rather than deriving via ``dataclasses.fields(MockRouterTestingParams)`` at # import time) avoids a cyclic import: ``litellm.types.router`` imports # back into proxy modules before this module finishes loading. -_MOCK_TESTING_KWARG_NAMES: tuple = ( +GATED_MOCK_PARAM_NAMES: tuple[str, ...] = ( "mock_testing_fallbacks", "mock_testing_context_fallbacks", "mock_testing_content_policy_fallbacks", + "mock_testing_rate_limit_error", + "mock_timeout", + "mock_delay", ) +MOCK_TESTING_CONFIG_KEY = "dangerously_allow_mock_testing_request_params" + if TYPE_CHECKING: from litellm.router import Router as _Router @@ -169,6 +175,41 @@ def raise_if_required_body_param_missing(route_type: str, data: Mapping[str, obj ) +class MockTestingParamsDisabledError(HTTPException): + def __init__(self, params: tuple[str, ...]): + super().__init__( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ # mutable-ok: HTTPException.detail has no immutable form; same shape as the sibling errors here + "error": ( + f"Mock testing request params are disabled on this proxy: {', '.join(params)}. " + f"An admin can enable them by setting `general_settings.{MOCK_TESTING_CONFIG_KEY}: true` " + "in config.yaml. This setting cannot be changed from the Admin UI or the API." + ) + }, + ) + + +def raise_if_mock_testing_params_disallowed(data: Mapping[str, object], *, allowed: bool) -> None: + """Reject client-supplied mock testing params unless an admin opted in. + + Rejecting (rather than silently dropping) keeps a request that asked for a + synthetic failure from returning a normal success, which reads as a passing + fallback test that never ran. + """ + if allowed: + return + present = tuple(name for name in GATED_MOCK_PARAM_NAMES if name in data) + if present: + raise MockTestingParamsDisabledError(params=present) + + +def mock_testing_params_allowed() -> bool: + """Read the opt-in from the running proxy's ``general_settings``.""" + import litellm.proxy.proxy_server as proxy_server + + return proxy_server.general_settings.get(MOCK_TESTING_CONFIG_KEY, False) is True + + def get_team_id_from_data(data: dict) -> Optional[str]: """ Get the team id from the data's metadata or litellm_metadata params. @@ -381,12 +422,10 @@ async def route_request( await add_shared_session_to_data(data) - # Strip router-internal mock_testing_* flags. Combined with an - # unauthorized fallback in ``router_settings_override`` they let a - # caller deterministically execute requests against restricted - # models. VERIA-44. - for _key in _MOCK_TESTING_KWARG_NAMES: - data.pop(_key, None) + # Gated here rather than alongside the sibling untrusted-param checks in + # ``litellm_pre_call_utils``: the Responses WebSocket route calls + # ``route_request`` directly and never runs ``add_litellm_data_to_request``. + raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed()) data.pop("enable_tag_filtering", None) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index b9a33bd2cef..192c674771f 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -10678,3 +10678,83 @@ async def test_async_data_generator_forwards_usage_chunk_without_strip_marker(): assert len(data_frames) == 4 assert any('"usage"' in frame and '"completion_tokens":188' in frame.replace(" ", "") for frame in data_frames) assert frames[-1] == "data: [DONE]\n\n" + + +@pytest.mark.asyncio +async def test_config_field_update_rejects_mock_testing_flag(): + """The mock-testing opt-in is deliberately absent from + ``ConfigGeneralSettings`` so that ``/config/field/update`` refuses it. If + someone later adds the field for tidiness, this test fails and tells them + they have just opened an API write path into a config-file-only setting.""" + from fastapi import HTTPException + + from litellm.proxy._types import ConfigFieldUpdate + from litellm.proxy.proxy_server import update_config_general_settings + from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY + + admin = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-test", + ) + + with patch.object(proxy_server_module, "prisma_client", MagicMock()): + with pytest.raises(HTTPException) as exc_info: + await update_config_general_settings( + data=ConfigFieldUpdate( + field_name=MOCK_TESTING_CONFIG_KEY, + field_value=True, + config_type="general_settings", + ), + user_api_key_dict=admin, + ) + + assert exc_info.value.status_code == 400 + + +def test_config_update_body_drops_mock_testing_flag(): + """``/config/update`` parses its body as ``ConfigYAML``, whose + ``general_settings`` is a ``ConfigGeneralSettings``. Undeclared keys are + dropped on parse, so the flag never reaches the DB by that route either.""" + from litellm.proxy._types import ConfigYAML + from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY + + parsed = ConfigYAML.model_validate({"general_settings": {MOCK_TESTING_CONFIG_KEY: True}}) + + assert parsed.general_settings is not None + assert MOCK_TESTING_CONFIG_KEY not in parsed.general_settings.model_dump(exclude_none=True) + + +def test_startup_warns_when_mock_testing_params_enabled(caplog): + """Enabling the opt-in must announce itself, naming every param it + unlocks — the config key says ``mock_testing`` but the gate also covers + ``mock_timeout`` and ``mock_delay``, so coverage cannot be inferred from + the name alone.""" + import logging + + from litellm.proxy.proxy_server import ProxyStartupEvent + from litellm.proxy.route_llm_request import ( + GATED_MOCK_PARAM_NAMES, + MOCK_TESTING_CONFIG_KEY, + ) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + ProxyStartupEvent._warn_if_mock_testing_params_enabled( + general_settings={MOCK_TESTING_CONFIG_KEY: True} + ) + + assert MOCK_TESTING_CONFIG_KEY in caplog.text + for param_name in GATED_MOCK_PARAM_NAMES: + assert param_name in caplog.text + + +def test_startup_is_silent_when_mock_testing_params_disabled(caplog): + """A proxy that never set the opt-in must not emit the warning.""" + import logging + + from litellm.proxy.proxy_server import ProxyStartupEvent + from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + ProxyStartupEvent._warn_if_mock_testing_params_enabled(general_settings={}) + + assert MOCK_TESTING_CONFIG_KEY not in caplog.text diff --git a/tests/test_litellm/proxy/test_route_llm_request.py b/tests/test_litellm/proxy/test_route_llm_request.py index 93b3ef1cce8..e74693a4f6c 100644 --- a/tests/test_litellm/proxy/test_route_llm_request.py +++ b/tests/test_litellm/proxy/test_route_llm_request.py @@ -8,6 +8,8 @@ sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to from unittest.mock import MagicMock +from fastapi import HTTPException + from litellm.proxy.route_llm_request import ProxyModelNotFoundError, route_request @@ -471,38 +473,147 @@ async def test_route_request_with_router_settings_override_preserves_existing(): assert call_kwargs["timeout"] == 30 -def test_mock_testing_kwarg_names_matches_dataclass(): - """``_MOCK_TESTING_KWARG_NAMES`` is hardcoded to avoid a cyclic import - against ``litellm.types.router``. This test guards against drift — - if a new ``mock_testing_*`` field is added to ``MockRouterTestingParams`` - the strip list must be updated to keep covering it.""" +def test_gated_mock_params_cover_mock_router_testing_params(): + """``GATED_MOCK_PARAM_NAMES`` is hardcoded to avoid a cyclic import + against ``litellm.types.router``. This test guards against drift — if a + new ``mock_testing_*`` field is added to ``MockRouterTestingParams`` the + gate must be updated to keep covering it. The gate is a superset: it also + covers params consumed outside that dataclass.""" from dataclasses import fields - from litellm.proxy.route_llm_request import _MOCK_TESTING_KWARG_NAMES + from litellm.proxy.route_llm_request import GATED_MOCK_PARAM_NAMES from litellm.types.router import MockRouterTestingParams - assert set(_MOCK_TESTING_KWARG_NAMES) == {f.name for f in fields(MockRouterTestingParams)} + assert {f.name for f in fields(MockRouterTestingParams)} <= set(GATED_MOCK_PARAM_NAMES) + assert {"mock_testing_rate_limit_error", "mock_timeout", "mock_delay"} <= set(GATED_MOCK_PARAM_NAMES) -@pytest.mark.asyncio @pytest.mark.parametrize( - "mock_flag", + "mock_param", [ "mock_testing_fallbacks", "mock_testing_context_fallbacks", "mock_testing_content_policy_fallbacks", + "mock_testing_rate_limit_error", + "mock_timeout", + "mock_delay", ], ) -async def test_route_request_strips_mock_testing_flags(mock_flag): - """VERIA-44: router-internal testing flags must not survive a - user-supplied request body. Without this strip, an attacker can - combine ``mock_testing_fallbacks=true`` with an unauthorized fallback - in ``router_settings_override`` to deterministically execute requests - against restricted models.""" +def test_mock_params_rejected_when_not_allowed(mock_param): + """Every gated param must be rejected by name when the proxy has not + opted in, and the error must point the caller at the config key.""" + from litellm.proxy.route_llm_request import ( + MOCK_TESTING_CONFIG_KEY, + raise_if_mock_testing_params_disallowed, + ) + + data = {"model": "gpt-3.5-turbo", mock_param: True} + + with pytest.raises(HTTPException) as exc_info: + raise_if_mock_testing_params_disallowed(data, allowed=False) + + assert exc_info.value.status_code == 400 + error_message = exc_info.value.detail["error"] + assert mock_param in error_message + assert MOCK_TESTING_CONFIG_KEY in error_message + + +@pytest.mark.parametrize( + "mock_param", + [ + "mock_testing_fallbacks", + "mock_testing_context_fallbacks", + "mock_testing_content_policy_fallbacks", + "mock_testing_rate_limit_error", + "mock_timeout", + "mock_delay", + ], +) +def test_mock_params_pass_through_when_allowed(mock_param): + """With the opt-in set, gated params must survive untouched — a gate that + rejects correctly but strips anyway would leave the feature unusable.""" + from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed + + data = {"model": "gpt-3.5-turbo", mock_param: True} + + raise_if_mock_testing_params_disallowed(data, allowed=True) + + assert data[mock_param] is True + + +def test_mock_param_gate_reports_every_param_present(): + """A request carrying several gated params must name all of them, so a + caller fixing one is not surprised by the next.""" + from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed + + data = { + "model": "gpt-3.5-turbo", + "mock_testing_fallbacks": True, + "mock_delay": 30, + } + + with pytest.raises(HTTPException) as exc_info: + raise_if_mock_testing_params_disallowed(data, allowed=False) + + error_message = exc_info.value.detail["error"] + assert "mock_testing_fallbacks" in error_message + assert "mock_delay" in error_message + + +def test_ordinary_request_is_not_rejected_by_the_mock_param_gate(): + """The gate must not fire on a request that carries no gated param.""" + from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed + data = { "model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello"}], - mock_flag: True, + "mock_response": "hi", + } + + raise_if_mock_testing_params_disallowed(data, allowed=False) + + +@pytest.mark.asyncio +async def test_route_request_rejects_mock_params_by_default(monkeypatch): + """End-to-end through ``route_request``: with no opt-in configured the + request is rejected before it ever reaches the router.""" + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(proxy_server, "general_settings", {}, raising=False) + + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}], + "mock_testing_fallbacks": True, + } + llm_router = MagicMock() + + with pytest.raises(HTTPException) as exc_info: + await route_request(data, llm_router, None, "acompletion") + + assert exc_info.value.status_code == 400 + llm_router.acompletion.assert_not_called() + + +@pytest.mark.asyncio +async def test_route_request_forwards_mock_params_when_opted_in(monkeypatch): + """End-to-end through ``route_request``: with the opt-in set the param + reaches the router, which is what makes a fallback drill possible.""" + import litellm.proxy.proxy_server as proxy_server + + from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY + + monkeypatch.setattr( + proxy_server, + "general_settings", + {MOCK_TESTING_CONFIG_KEY: True}, + raising=False, + ) + + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}], + "mock_testing_fallbacks": True, } llm_router = MagicMock() llm_router.acompletion.return_value = "ok" @@ -510,10 +621,7 @@ async def test_route_request_strips_mock_testing_flags(mock_flag): await route_request(data, llm_router, None, "acompletion") call_kwargs = llm_router.acompletion.call_args[1] - assert mock_flag not in call_kwargs - # The flag is also gone from the original data dict so any subsequent - # processing (e.g. logging) doesn't see it either. - assert mock_flag not in data + assert call_kwargs["mock_testing_fallbacks"] is True @pytest.mark.parametrize("route_type", ["agenerate_content", "agenerate_content_stream"]) From cee78d61eadadfb2b0a3c25b723db357d1e40687 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 1 Aug 2026 00:38:58 +0000 Subject: [PATCH 010/123] chore(typing): re-ratchet lint budgets after merging staging --- type-discipline-budget.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type-discipline-budget.json b/type-discipline-budget.json index b79aae63b0b..05499e83c42 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -3,7 +3,7 @@ "limit": 23250 }, "LIT002": { - "limit": 27280 + "limit": 27277 }, "LIT003": { "limit": 292 From 764b2337701f972ecafd172c9d0705ed976189b6 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 31 Jul 2026 17:52:01 -0700 Subject: [PATCH 011/123] refactor(proxy): drop an inaccurate comment on the mock testing gate The comment said the Responses WebSocket route never runs add_litellm_data_to_request. It does, via common_processing_pre_call_logic, so the note recorded a request-flow constraint that does not hold The gate stays in route_request, which is the dispatch chokepoint and where the previous handling lived --- litellm/proxy/route_llm_request.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index aebdbca86ad..12e9857219e 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -422,9 +422,6 @@ async def route_request( await add_shared_session_to_data(data) - # Gated here rather than alongside the sibling untrusted-param checks in - # ``litellm_pre_call_utils``: the Responses WebSocket route calls - # ``route_request`` directly and never runs ``add_litellm_data_to_request``. raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed()) data.pop("enable_tag_filtering", None) From d640ace6d88a08d4e1bef6dc68e9fbce54d6c035 Mon Sep 17 00:00:00 2001 From: Shivam Rawat Date: Fri, 31 Jul 2026 18:05:50 -0700 Subject: [PATCH 012/123] fix(rate-limit): enforce token limits when the pre-call increment is zero The atomic check-and-increment path skipped any counter whose increment was <= 0. The dynamic rate limiter always passes a zero token increment pre-call because usage lands on the counters post-response, so on a model configured with only tpm the limiter evaluated no counters at all: no model-wide TPM cap and no priority reservation, in either generous or strict mode. Regressed in dd57ae6691 when the pre-call flow moved off the read-only should_rate_limit check, which did evaluate token limits. Keep zero-increment counters in the payload so they act as a pure check (current + 0 > limit), matching the pre-regression semantics in both the Lua and in-memory paths. Adds unit regressions at the primitive and hook level plus a live e2e covering the priority_generous/priority_strict registry rows. --- .../hooks/parallel_request_limiter_v3.py | 2 +- tests/e2e/models.py | 3 + .../test_dynamic_rate_limit_priority_e2e.py | 242 ++++++++++++++++++ .../hooks/test_dynamic_rate_limiter_v3.py | 92 +++++++ .../hooks/test_parallel_request_limiter_v3.py | 57 +++++ 5 files changed, 395 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/quota_management/ratelimit/test_dynamic_rate_limit_priority_e2e.py diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index b04ef5f7087..486e3cf88a4 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -1341,7 +1341,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): else: limit_value = rate_limit.get("tokens_per_unit") inc_amount = int(increment_amounts.get("tokens", 0) or 0) - if limit_value is None or inc_amount <= 0: + if limit_value is None: continue counter_key = self.create_rate_limit_keys(descriptor_key, descriptor_value, rlt) # Counter-key TTL and window_size are conceptually distinct diff --git a/tests/e2e/models.py b/tests/e2e/models.py index af695acaa5e..f1c0ede0e85 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -43,6 +43,7 @@ class KeyLoggingCallback(BaseModel): class KeyMetadata(BaseModel): logging: list[KeyLoggingCallback] | None = None + priority: str | None = None class ObjectPermission(BaseModel): @@ -97,6 +98,7 @@ class LiteLLMBudgetTable(BaseModel): class KeyInfo(BaseModel): key_alias: str | None = None + metadata: KeyMetadata | None = None models: list[str] = [] tpm_limit: int | None = None rpm_limit: int | None = None @@ -694,6 +696,7 @@ class LiteLLMParamsBody(BaseModel): complexity_router_config: dict[str, object] | None = None mock_response: str | None = None timeout: float | None = None + tpm: int | None = None ModelMode = Literal["batch", "realtime", "image_generation"] diff --git a/tests/e2e/quota_management/ratelimit/test_dynamic_rate_limit_priority_e2e.py b/tests/e2e/quota_management/ratelimit/test_dynamic_rate_limit_priority_e2e.py new file mode 100644 index 00000000000..06b4aa1a0b5 --- /dev/null +++ b/tests/e2e/quota_management/ratelimit/test_dynamic_rate_limit_priority_e2e.py @@ -0,0 +1,242 @@ +"""Live e2e: the v3 dynamic rate limiter's saturation-aware priority reservation. + +Covers quota_management.ratelimit.priority_generous / priority_strict: with +`dynamic_rate_limiter_v3` enabled, a model's TPM capacity is split into priority +reservations, but a reservation is only enforced once the model is saturated. + +- Generous mode (recorded usage below the saturation threshold): a key whose + priority reserves 25% of capacity keeps serving past its reservation, + borrowing the idle capacity (priority_generous.picks_under_tpm) +- Strict mode (recorded usage at/over the threshold): the over-reservation key + is blocked with the priority-flavored 429 while a key of a different priority, + still inside its own reservation, is served (priority_strict.picks_under_tpm) + +The proxy under test must run with this config (and LITELLM_LICENSE set, since +priority reservation is a premium feature): + + litellm_settings: + callbacks: ["dynamic_rate_limiter_v3"] + priority_reservation: + prod: 0.5 + dev: 0.25 + priority_reservation_settings: + saturation_threshold: 0.5 + saturation_check_cache_ttl: 1 + +The constants below mirror those values; if the proxy runs different ones the +tests fail with a message naming the required config rather than skipping. + +The limiter counts a request against the model-wide window pre-call, but tokens +only land on the counters after each response completes (there is no pre-call +token reservation at the model level), so recorded saturation always trails the +traffic that produced it. The tests therefore drive spend by summing each +body's usage.total_tokens (the counter can never be ahead of that sum) and poll +for the strict-mode block instead of expecting it on an exact call. Each test +creates its own /model/new deployment so its 60s rate-limit window and counters +are isolated from concurrent runs. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass + +import pytest +from pydantic import BaseModel, ConfigDict, ValidationError + +from e2e_config import unique_marker +from e2e_http import StreamingResponse, require_successful_call +from lifecycle import ResourceManager +from models import KeyGenerateBody, KeyMetadata, LiteLLMParamsBody +from quota_client import QuotaClient + +pytestmark = pytest.mark.e2e + +BACKEND = "anthropic/claude-haiku-4-5-20251001" +MODEL_TPM = 400 +DEV_PRIORITY = "dev" +PROD_PRIORITY = "prod" +DEV_RESERVED_TOKENS = int(MODEL_TPM * 0.25) +SATURATION_TOKENS = int(MODEL_TPM * 0.5) +CHAT_MAX_TOKENS = 16 +WINDOW_SECONDS = 60 +WINDOW_MARGIN_SECONDS = 10 +STRICT_POLL_SPEND_CEILING = int(MODEL_TPM * 0.7) + +REQUIRED_CONFIG_HINT = ( + "the proxy must run litellm_settings.callbacks=['dynamic_rate_limiter_v3'] with " + "priority_reservation {prod: 0.5, dev: 0.25} and priority_reservation_settings " + "{saturation_threshold: 0.5, saturation_check_cache_ttl: 1}; see this module's docstring" +) + + +class _ChatUsage(BaseModel): + model_config = ConfigDict(extra="ignore") + + total_tokens: int + + +class _ChatBodyWithUsage(BaseModel): + model_config = ConfigDict(extra="ignore") + + usage: _ChatUsage + + +def _total_tokens(outcome: StreamingResponse) -> int: + try: + return _ChatBodyWithUsage.model_validate_json(outcome.body).usage.total_tokens + except ValidationError: + pytest.fail(f"successful chat body must report usage.total_tokens, got: {outcome.body[:300]}") + + +@dataclass(frozen=True, slots=True) +class _Fixture: + model: str + dev_key: str + prod_key: str + + +def _dynamic_limited_model(client: QuotaClient, resources: ResourceManager, label: str) -> _Fixture: + model = f"e2e-dynpri-{label}-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody(model=BACKEND, api_key="os.environ/ANTHROPIC_API_KEY", tpm=MODEL_TPM), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + + def _priority_key(priority: str) -> str: + key = client.proxy.generate_key( + KeyGenerateBody( + models=[model], + metadata=KeyMetadata(priority=priority), + key_alias=f"e2e-dynpri-{label}-{priority}-{unique_marker()}", + ) + ) + resources.defer(lambda: client.proxy.delete_key(key)) + return key + + return _Fixture(model=model, dev_key=_priority_key(DEV_PRIORITY), prod_key=_priority_key(PROD_PRIORITY)) + + +def _chat(client: QuotaClient, key: str, model: str) -> StreamingResponse: + return client.chat(key, model, f"reply with one word {unique_marker()}", max_tokens=CHAT_MAX_TOKENS) + + +@dataclass(frozen=True, slots=True) +class _FirstOk: + sent_at: float + response: StreamingResponse + + +def _first_ok(client: QuotaClient, key: str, model: str) -> _FirstOk: + """First successful call on a fresh key opens the model's rate-limit window; + `sent_at` (captured before the winning send) is a lower bound on the window + start. A fresh key may briefly 401 until the auth cache picks it up, so + retry 401s to a deadline; a 401 never reaches the limiter, so only the + successful call consumes budget.""" + deadline = time.monotonic() + client.proxy.poll_timeout + while True: + sent_at = time.monotonic() + outcome = _chat(client, key, model) + if outcome.ok: + return _FirstOk(sent_at=sent_at, response=outcome) + if outcome.status_code != 401 or time.monotonic() >= deadline: + require_successful_call(outcome) + time.sleep(client.proxy.poll_interval) + + +def _window_guard(first: _FirstOk, spent: int) -> None: + assert time.monotonic() < first.sent_at + WINDOW_SECONDS - WINDOW_MARGIN_SECONDS, ( + f"only {spent} tokens of spend landed before the {WINDOW_SECONDS}s rate-limit window could " + "roll; this test needs every call inside one window" + ) + + +class TestDynamicRateLimitPriority: + @pytest.mark.covers( + "quota_management.ratelimit.priority_generous.picks_under_tpm", + exercised_on=["chat_completions"], + ) + def test_generous_mode_lets_priority_borrow_past_reservation( + self, client: QuotaClient, resources: ResourceManager + ) -> None: + fixture = _dynamic_limited_model(client, resources, "generous") + + info = client.proxy.key_info(fixture.dev_key) + assert info.metadata is not None and info.metadata.priority == DEV_PRIORITY, ( + f"/key/info must echo the key's priority metadata, got {info.metadata}" + ) + + first = _first_ok(client, fixture.dev_key, fixture.model) + spent = _total_tokens(first.response) + while spent <= DEV_RESERVED_TOKENS: + _window_guard(first, spent) + assert spent < SATURATION_TOKENS, ( + f"spend reached the saturation threshold ({spent} of {SATURATION_TOKENS}) before " + f"crossing the dev reservation ({DEV_RESERVED_TOKENS}); shrink per-call spend to " + "keep the borrowing claim observable" + ) + outcome = _chat(client, fixture.dev_key, fixture.model) + assert outcome.status_code != 429, ( + f"dev key was blocked at {spent} recorded tokens, under the saturation threshold " + f"({SATURATION_TOKENS} of {MODEL_TPM}); generous mode must let it borrow past its " + f"{DEV_RESERVED_TOKENS}-token reservation. If the limiter is missing entirely, " + f"{REQUIRED_CONFIG_HINT}. 429 body: {outcome.body[:300]}" + ) + require_successful_call(outcome) + spent += _total_tokens(outcome) + + assert spent > DEV_RESERVED_TOKENS + + @pytest.mark.covers( + "quota_management.ratelimit.priority_strict.picks_under_tpm", + exercised_on=["chat_completions"], + ) + def test_strict_mode_blocks_saturated_priority_but_serves_the_other( + self, client: QuotaClient, resources: ResourceManager + ) -> None: + fixture = _dynamic_limited_model(client, resources, "strict") + + prod_warmup = _first_ok(client, fixture.prod_key, fixture.model) + first = _first_ok(client, fixture.dev_key, fixture.model) + prod_spent = _total_tokens(prod_warmup.response) + dev_spent = _total_tokens(first.response) + + while prod_spent + dev_spent < SATURATION_TOKENS: + _window_guard(prod_warmup, prod_spent + dev_spent) + outcome = _chat(client, fixture.dev_key, fixture.model) + assert outcome.status_code != 429, ( + f"dev key was blocked at {prod_spent + dev_spent} recorded tokens, before the " + f"saturation threshold ({SATURATION_TOKENS} of {MODEL_TPM}); strict enforcement " + f"must not engage early. 429 body: {outcome.body[:300]}" + ) + require_successful_call(outcome) + dev_spent += _total_tokens(outcome) + + while True: + _window_guard(prod_warmup, prod_spent + dev_spent) + assert prod_spent + dev_spent < STRICT_POLL_SPEND_CEILING, ( + f"dev key was still served at {prod_spent + dev_spent} tokens, past the saturation " + f"threshold ({SATURATION_TOKENS}) and {DEV_RESERVED_TOKENS}-token dev reservation; " + f"strict priority enforcement never engaged. Check that {REQUIRED_CONFIG_HINT}" + ) + outcome = _chat(client, fixture.dev_key, fixture.model) + if outcome.status_code == 429: + assert "Priority-based rate limit exceeded" in outcome.body, ( + f"the saturated dev key must get the priority-flavored 429, got: {outcome.body[:300]}" + ) + assert outcome.headers.get("x-litellm-priority") == DEV_PRIORITY, ( + f"the 429 must attribute the blocked priority, headers: " + f"{ {k: v for k, v in outcome.headers.items() if 'litellm' in k} }" + ) + break + require_successful_call(outcome) + dev_spent += _total_tokens(outcome) + + prod_outcome = _chat(client, fixture.prod_key, fixture.model) + require_successful_call(prod_outcome) + prod_spent += _total_tokens(prod_outcome) + assert prod_spent < int(MODEL_TPM * 0.5), ( + f"the prod fairness claim needs prod spend ({prod_spent}) inside its reservation " + f"({int(MODEL_TPM * 0.5)}); shrink per-call spend" + ) diff --git a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py index c8176ca6337..6c717d6f71c 100644 --- a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -1771,3 +1771,95 @@ async def test_priority_429_includes_model_name_and_configured_limits(): assert "Priority: prod" in error_msg, error_msg assert "Rate limit type: tokens" in error_msg, error_msg assert "Model saturation:" in error_msg, error_msg + + +@pytest.mark.asyncio +async def test_tpm_only_model_enforces_priority_and_model_capacity(): + """Regression: a model configured with ONLY tpm (no rpm) must still be + rate limited. + + The atomic check-and-increment path used to drop any counter whose + pre-call increment was zero. Token increments are always zero pre-call + (usage lands on the counters post-call), so on a TPM-only model the + limiter evaluated no counters at all: no model-wide cap, no priority + reservation, in either mode. This test drives the real pre-call -> + log-success -> pre-call flow with no limiter internals mocked. + """ + from fastapi import HTTPException + + from litellm.types.utils import ModelResponse, Usage + + os.environ["LITELLM_LICENSE"] = "test-license-key" + litellm.priority_reservation = {"dev": 0.25, "prod": 0.5} + + dual_cache = DualCache() + handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache) + + model = "tpm-only-model" + llm_router = Router( + model_list=[ + { + "model_name": model, + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": "test-key", + "api_base": "test-base", + "tpm": 400, + }, + } + ] + ) + handler.update_variables(llm_router=llm_router) + + dev_user = UserAPIKeyAuth() + dev_user.metadata = {"priority": "dev"} + prod_user = UserAPIKeyAuth() + prod_user.metadata = {"priority": "prod"} + + async def record_usage(priority: str, total_tokens: int) -> None: + await handler.async_log_success_event( + kwargs={ + "standard_logging_object": { + "metadata": {"user_api_key_auth_metadata": {"priority": priority}}, + }, + "litellm_params": {"metadata": {"model_group": model}}, + }, + response_obj=ModelResponse( + model=model, + usage=Usage(prompt_tokens=0, completion_tokens=total_tokens, total_tokens=total_tokens), + ), + start_time=None, + end_time=None, + ) + + assert ( + await handler.async_pre_call_hook( + user_api_key_dict=dev_user, cache=dual_cache, data={"model": model}, call_type="completion" + ) + is None + ) + + await record_usage("dev", 250) + + with pytest.raises(HTTPException) as dev_blocked: + await handler.async_pre_call_hook( + user_api_key_dict=dev_user, cache=dual_cache, data={"model": model}, call_type="completion" + ) + assert dev_blocked.value.status_code == 429 + assert "Priority-based rate limit exceeded" in dev_blocked.value.detail["error"] + + assert ( + await handler.async_pre_call_hook( + user_api_key_dict=prod_user, cache=dual_cache, data={"model": model}, call_type="completion" + ) + is None + ) + + await record_usage("prod", 200) + + with pytest.raises(HTTPException) as capacity_blocked: + await handler.async_pre_call_hook( + user_api_key_dict=prod_user, cache=dual_cache, data={"model": model}, call_type="completion" + ) + assert capacity_blocked.value.status_code == 429 + assert "Model capacity reached" in capacity_blocked.value.detail["error"] diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 56bfd1829b5..84e41711176 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -4998,3 +4998,60 @@ async def test_split_usage_still_respects_the_configured_limit_type(monkeypatch) token_operations = [op for op in captured_operations if op["key"].endswith(":tokens")] assert token_operations assert all(op["increment_value"] == 7 for op in token_operations) + + +@pytest.mark.asyncio +async def test_atomic_check_with_zero_increment_still_enforces_token_limit(): + """Regression: a zero token increment must still CHECK the token limit. + + The dynamic rate limiter calls atomic_check_and_increment_by_n with + {"requests": 1, "tokens": 0} because tokens land on the counter post-call. + The payload builder used to skip any counter whose increment was <= 0, so a + TPM-only descriptor produced zero counters to evaluate and the call + returned OK with empty statuses; TPM limits were never enforced at all. + """ + from litellm.proxy.hooks.parallel_request_limiter_v3 import RateLimitDescriptor + + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(DualCache()) + ) + descriptor = RateLimitDescriptor( + key="model_saturation_check", + value="tpm-only-model", + rate_limit={"tokens_per_unit": 100, "window_size": 60}, + ) + zero_token_increment: Dict[str, int] = {"requests": 1, "tokens": 0} + + under_limit = await handler.atomic_check_and_increment_by_n( + descriptors=[descriptor], + increments=[zero_token_increment], + ) + assert under_limit["overall_code"] == "OK" + assert [s["rate_limit_type"] for s in under_limit["statuses"]] == ["tokens"] + + counter_key = handler.create_rate_limit_keys( + "model_saturation_check", "tpm-only-model", "tokens" + ) + await handler.async_increment_tokens_with_ttl_preservation( + pipeline_operations=[ + RedisPipelineIncrementOperation( + key=counter_key, increment_value=150, ttl=60 + ) + ], + ) + + over_limit = await handler.atomic_check_and_increment_by_n( + descriptors=[descriptor], + increments=[zero_token_increment], + ) + assert over_limit["overall_code"] == "OVER_LIMIT" + blocked = over_limit["statuses"][0] + assert blocked["rate_limit_type"] == "tokens" + assert blocked["current_limit"] == 100 + + assert ( + await handler.internal_usage_cache.async_get_cache( + key=counter_key, litellm_parent_otel_span=None, local_only=True + ) + == 150 + ) From 836bd927b68a7fbabfb1afcdcbdc6b31dd13b8fc Mon Sep 17 00:00:00 2001 From: Shivam Rawat Date: Fri, 31 Jul 2026 18:05:50 -0700 Subject: [PATCH 013/123] fix(rate-limit): skip negative increments in the atomic payload builder Review feedback: the relaxed predicate admitted negative increments, which both atomic backends would apply as decrements. Restrict the new behavior to zero-valued pure checks and assert negatives neither check nor mutate counters. --- litellm/proxy/hooks/parallel_request_limiter_v3.py | 2 +- .../hooks/test_parallel_request_limiter_v3.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 486e3cf88a4..89559ff72ae 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -1341,7 +1341,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): else: limit_value = rate_limit.get("tokens_per_unit") inc_amount = int(increment_amounts.get("tokens", 0) or 0) - if limit_value is None: + if limit_value is None or inc_amount < 0: continue counter_key = self.create_rate_limit_keys(descriptor_key, descriptor_value, rlt) # Counter-key TTL and window_size are conceptually distinct diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 84e41711176..7b3f00a55a8 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -5055,3 +5055,17 @@ async def test_atomic_check_with_zero_increment_still_enforces_token_limit(): ) == 150 ) + + negative_increment: Dict[str, int] = {"requests": -1, "tokens": -50} + refund_attempt = await handler.atomic_check_and_increment_by_n( + descriptors=[descriptor], + increments=[negative_increment], + ) + assert refund_attempt["overall_code"] == "OK" + assert refund_attempt["statuses"] == [] + assert ( + await handler.internal_usage_cache.async_get_cache( + key=counter_key, litellm_parent_otel_span=None, local_only=True + ) + == 150 + ) From b4ff05be8edb669c0cb033776c74cfc749e9407c Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 31 Jul 2026 18:10:48 -0700 Subject: [PATCH 014/123] fix(proxy): stop model writes 500ing on another pod's delete (#35400) * fix(proxy): stop model writes 500ing on another pod's delete A model write judges the reload it triggers by diffing this pod's router before and after, and reports anything that stopped serving as damage. On a pod that has not yet polled a delete another pod made, the snapshot still lists that model; the reload then evicts it because the db no longer has it, and the guard reads its own correct reconcile as degradation. The row is written and served, but the caller gets a 500. Since propagation between pods is a 30s db poll, any delete followed by a create inside that window can land on a pod that has not caught up, so a delete-then-create pair returns 500 whenever the two requests hit different pods. _delete_deployment already computes exactly the set that settles it: the ids the db and config still want. Thread it up through _update_llm_router, add_deployment and clear_cache to the verdict, and intersect the drop set with it so an id the db no longer has stops counting as collateral. Where no reconcile ran the set is None and every drop is still reported, so a genuinely broken reload is caught as before. _delete_deployment now returns that set instead of a delete count; the count had no callers in the proxy, and the tests asserting it already assert the eviction calls. * test(proxy): fold reload-verdict test commentary into docstrings and assertions Greptile flagged the inline comments against the repo's no-new-comments rule. The case-by-case context moves into the test docstring, and the two return-contract assertions carry their reasoning as failure messages instead. * test: fix clear_cache mock return type in model block/unblock tests --- .../model_management_endpoints.py | 48 ++++++++++--- litellm/proxy/proxy_server.py | 39 +++++++---- .../test_model_management_endpoints.py | 70 ++++++++++++++++++- .../proxy/proxy_server/test_proxy_config.py | 6 +- tests/test_litellm/proxy/test_proxy_server.py | 19 +++-- .../test_update_llm_router_resilience.py | 15 ++-- .../test_litellm/test_model_block_unblock.py | 2 +- 7 files changed, 158 insertions(+), 41 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index b6422d7f5ae..41904e3883b 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -324,7 +324,7 @@ async def patch_model( # Clear cache and reload models (uses config setting or defaults to preserving config models for DB updates) live_before_reload = live_model_ids_snapshot() - await clear_cache() + still_desired_ids = await clear_cache() ## CREATE AUDIT LOG ## asyncio.create_task( @@ -344,6 +344,7 @@ async def patch_model( before=live_before_reload, written_models=[(model_id, getattr(updated_model, "model_info", None))], action="update", + still_desired=still_desired_ids, ) return updated_model @@ -429,7 +430,7 @@ async def _set_model_blocked_status( ) live_before_reload = live_model_ids_snapshot() - await clear_cache() + still_desired_ids = await clear_cache() asyncio.create_task( create_object_audit_log( @@ -450,6 +451,7 @@ async def _set_model_blocked_status( before=live_before_reload, written_models=[(data.model_id, getattr(updated_model, "model_info", None))], action=action, + still_desired=still_desired_ids, ) return updated_model @@ -1355,6 +1357,7 @@ async def add_new_model( """ live_before_reload = live_model_ids_snapshot() + still_desired_ids: frozenset[str] | None = None try: _original_litellm_model_name = model_params.model_name if model_params.model_info.team_id is None: @@ -1369,7 +1372,9 @@ async def add_new_model( user_api_key_dict=user_api_key_dict, prisma_client=prisma_client, ) - await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) + still_desired_ids = await proxy_config.add_deployment( + prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj + ) # don't let failed slack alert block the /model/new response _alerting = general_settings.get("alerting", []) or [] if "slack" in _alerting: @@ -1414,6 +1419,7 @@ async def add_new_model( before=live_before_reload, written_models=[(model_response.model_id, getattr(model_response, "model_info", None))], action="create", + still_desired=still_desired_ids, ) return model_response @@ -1542,7 +1548,7 @@ async def update_model( # Clear cache and reload models (uses config setting or defaults to preserving config models for DB updates) live_before_reload = live_model_ids_snapshot() - await clear_cache() + still_desired_ids = await clear_cache() ## CREATE AUDIT LOG ## asyncio.create_task( create_object_audit_log( @@ -1569,6 +1575,7 @@ async def update_model( before=live_before_reload, written_models=[(_model_id, getattr(model_response, "model_info", None))], action="update", + still_desired=still_desired_ids, ) return model_response @@ -1814,6 +1821,7 @@ def reload_serving_verdict( before: frozenset[str], written_models: Sequence[tuple[str, object]], written_must_serve: bool, + still_desired: frozenset[str] | None = None, ) -> tuple[tuple[str, ...], tuple[str, ...]]: """Judge a write-triggered reload by diffing the router's serving state instead of trusting any layer of the reload stack to report its own failure. @@ -1828,10 +1836,14 @@ def reload_serving_verdict( this write and blaming it would block unrelated metadata fixes - not written but live before and gone now: collateral degradation of this pod caused by the reload this request triggered (a wholesale re-add failure, or a - newly introduced conflict), always reported + newly introduced conflict), reported only when the db still wants that id + + ``still_desired`` is the db + config id set the reload just reconciled against. An + id absent from it was deleted on purpose, most often by another pod this one had not + yet polled, so the reload dropping it is the reconcile working rather than damage. + Without it (no reconcile ran) every drop is reported, which is the safe direction. Returns (written ids violating their obligation, collateral ids no longer served). - Best effort under concurrent admin writes: the snapshot spans only this request. """ now = live_model_ids_snapshot() written_ids = frozenset(model_id for model_id, _ in written_models) @@ -1843,7 +1855,8 @@ def reload_serving_verdict( ) else: missing = tuple(model_id for model_id, _ in written_models if model_id in before and model_id not in now) - collateral = tuple(sorted(before - now - written_ids)) + dropped = before - now - written_ids + collateral = tuple(sorted(dropped if still_desired is None else dropped & still_desired)) return (missing, collateral) @@ -1851,12 +1864,18 @@ def raise_if_reload_degraded_serving( before: frozenset[str], written_models: Sequence[tuple[str, object]], action: str, + still_desired: frozenset[str] | None = None, ) -> None: """The caller-visible error this pod's model-write endpoints owe their caller when the model they wrote is not being served after the reload they triggered. The DB write is durable either way and every other pod reloads on its own interval; this speaks only for the handling pod.""" - missing, collateral = reload_serving_verdict(before=before, written_models=written_models, written_must_serve=True) + missing, collateral = reload_serving_verdict( + before=before, + written_models=written_models, + written_must_serve=True, + still_desired=still_desired, + ) if not missing and not collateral: return missing_clause = ( @@ -1882,9 +1901,12 @@ def raise_if_reload_degraded_serving( ) -async def clear_cache(): +async def clear_cache() -> frozenset[str] | None: """ Clear router caches and reload models. + + Returns the db + config id set the reload reconciled against, or None when no + reload ran, so callers can pass it to raise_if_reload_degraded_serving. """ from litellm.proxy.proxy_server import ( llm_router, @@ -1896,7 +1918,7 @@ async def clear_cache(): if llm_router is None or prisma_client is None: verbose_proxy_logger.debug("llm_router or prisma_client is None, skipping cache clear") - return + return None try: # Only clear DB models, preserve config models @@ -1943,10 +1965,14 @@ async def clear_cache(): llm_router.quality_routers.pop(model_name, None) # Reload only DB models - await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) + still_desired_ids = await proxy_config.add_deployment( + prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj + ) verbose_proxy_logger.debug( f"Cleared {len(db_model_ids)} DB models, preserved {len(config_models)} config models" ) + return still_desired_ids except Exception as e: verbose_proxy_logger.exception(f"Failed to clear cache and reload models. Due to error - {str(e)}") + return None diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a60ea2da019..15d977b75f2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5300,7 +5300,7 @@ class ProxyConfig: _model_info = RouterModelInfo(id=model.model_id, db_model=db_model) return _model_info - async def _delete_deployment(self, db_models: list) -> int: + async def _delete_deployment(self, db_models: list) -> frozenset[str] | None: """ (Helper function of add deployment) -> combined to reduce prisma db calls @@ -5309,14 +5309,16 @@ class ProxyConfig: - Remove any that are missing Return: - - int - returns number of deleted deployments + - frozenset[str] - the ids the db + config say should be served after this + reconcile, so a caller can tell an id this evicted on purpose from one that + went missing. None when no reconcile ran and that set is therefore unknown. """ global user_config_file_path, llm_router combined_id_list = [] ## BASE CASES ## if llm_router is None: - return 0 + return None # NOTE: db_models may be legitimately empty when all DB models have been deleted. # Do NOT short-circuit on len(db_models) == 0 — we must still evict any # DB-sourced deployments that are no longer in the DB. The caller @@ -5337,7 +5339,7 @@ class ProxyConfig: "Skipping deployment cleanup to avoid removing valid models.", str(e), ) - return 0 + return None model_list = config.get("model_list", None) if model_list: for model in model_list: @@ -5361,13 +5363,10 @@ class ProxyConfig: router_model_ids = llm_router.get_model_ids() # Check for model IDs in llm_router not present in combined_id_list and delete them - deleted_deployments = 0 for model_id in router_model_ids: if model_id not in combined_id_list: - is_deleted = llm_router.delete_deployment(id=model_id) - if is_deleted is not None: - deleted_deployments += 1 - return deleted_deployments + llm_router.delete_deployment(id=model_id) + return frozenset(combined_id_list) def _resolve_db_litellm_param(self, key: str, value: object) -> object: if not isinstance(value, str): @@ -5452,9 +5451,11 @@ class ProxyConfig: self, new_models: Optional[Json], proxy_logging_obj: ProxyLogging, - ): + ) -> frozenset[str] | None: global llm_router, llm_model_list, master_key, general_settings + still_desired_ids: frozenset[str] | None = None + # Load config separately so a timeout here doesn't block model loading config_data: dict = {} search_tools = None @@ -5501,7 +5502,7 @@ class ProxyConfig: if search_tools is not None and llm_router is not None: llm_router.search_tools = search_tools ## DELETE MODEL LOGIC - await self._delete_deployment(db_models=models_list) + still_desired_ids = await self._delete_deployment(db_models=models_list) ## ADD MODEL LOGIC self._add_deployment(db_models=models_list) @@ -5527,6 +5528,8 @@ class ProxyConfig: proxy_logging_obj=proxy_logging_obj, ) + return still_desired_ids + def _add_callback_from_db_to_in_memory_litellm_callbacks( self, callback: str, @@ -6159,14 +6162,20 @@ class ProxyConfig: self, prisma_client: PrismaClient, proxy_logging_obj: ProxyLogging, - ): + ) -> frozenset[str] | None: """ - Check db for new models - Check if model id's in router already - If not, add to router + + Returns the ids the db + config say should be served after the reconcile, or + None when no reconcile ran. Callers that judge their own reload need it to tell + a deliberate eviction from a deployment that went missing. """ global llm_router, llm_model_list, master_key, general_settings + still_desired_ids: frozenset[str] | None = None + try: # warm the config cache so the per-param reads below all hit await prefetch_config_params( @@ -6186,7 +6195,9 @@ class ProxyConfig: new_models = await self._get_models_from_db(prisma_client=prisma_client) # update llm router - await self._update_llm_router(new_models=new_models, proxy_logging_obj=proxy_logging_obj) + still_desired_ids = await self._update_llm_router( + new_models=new_models, proxy_logging_obj=proxy_logging_obj + ) db_general_settings = await get_config_param(prisma_client, "general_settings") @@ -6204,6 +6215,8 @@ class ProxyConfig: "litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - {}".format(str(e)) ) + return still_desired_ids + async def _init_non_llm_objects_in_db(self, prisma_client: PrismaClient): """ Use this to read non-llm objects from the db and initialize them diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 1e8add52f74..8dbf38555b3 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -839,7 +839,7 @@ class TestUpdateModel: ), patch( "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache", - new=AsyncMock(return_value=True), + new=AsyncMock(return_value=None), ) as mock_clear_cache, ): await update_model( @@ -3223,7 +3223,7 @@ class TestPatchModelBlockedAuthGate: ), patch( "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache", - new=AsyncMock(return_value=True), + new=AsyncMock(return_value=None), ), ): result = await patch_model( @@ -3314,6 +3314,72 @@ class TestWriteSurfacesReloadDrop: before=frozenset({"m-live", "m-collateral"}), written_models=[("m-live", None)], action="update" ) + def test_a_model_the_db_no_longer_has_is_not_collateral(self, monkeypatch): + """Another pod deleting a model is not this pod's reload breaking. + + A pod that has not yet polled the delete still lists the id when the write + snapshots `before`; the reload it triggers then evicts the id because the db no + longer has it. That eviction is the reconcile working, so it must not fail the + write. `still_desired` is the db + config set the reload reconciled against, so + an id missing from it drops out of the collateral diff. + + The cases below, in order: an id the db no longer wants is not collateral and the + write succeeds; an id the db still wants that stopped serving is real degradation + and still raises, so a genuinely broken reload is caught; and with no reconcile at + all the desired set is unknown, so every drop is reported. + """ + import litellm + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import ( + raise_if_reload_degraded_serving, + reload_serving_verdict, + ) + + live_router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4o", + "litellm_params": {"model": "gpt-4o"}, + "model_info": {"id": "m-live", "db_model": True}, + } + ] + ) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", live_router) + + _, collateral = reload_serving_verdict( + before=frozenset({"m-live", "m-deleted-elsewhere"}), + written_models=[("m-live", None)], + written_must_serve=True, + still_desired=frozenset({"m-live"}), + ) + assert collateral == () + + assert ( + raise_if_reload_degraded_serving( + before=frozenset({"m-live", "m-deleted-elsewhere"}), + written_models=[("m-live", None)], + action="create", + still_desired=frozenset({"m-live"}), + ) + is None + ) + + with pytest.raises(ProxyException, match="m-should-be-serving"): + raise_if_reload_degraded_serving( + before=frozenset({"m-live", "m-should-be-serving"}), + written_models=[("m-live", None)], + action="create", + still_desired=frozenset({"m-live", "m-should-be-serving"}), + ) + + with pytest.raises(ProxyException, match="m-deleted-elsewhere"): + raise_if_reload_degraded_serving( + before=frozenset({"m-live", "m-deleted-elsewhere"}), + written_models=[("m-live", None)], + action="create", + still_desired=None, + ) + class TestModelInfoAsMapping: """The model_info column reaches consumers as a dict or as its JSON string; this is diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 8d1d8185e4d..99287b92b8f 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -1404,12 +1404,12 @@ def test_ProxyConfig_get_model_info_with_id_missing_model_id_raises(monkeypatch) @pytest.mark.asyncio -async def test_ProxyConfig__delete_deployment_empty_returns_zero(monkeypatch): +async def test_ProxyConfig__delete_deployment_no_router_returns_none(monkeypatch): monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None) pc = ProxyConfig() result = await pc._delete_deployment(db_models=[]) - snapshot = {"deleted": result, "router_was": "none", "empty_db_models": True} - assert snapshot == {"deleted": 0, "router_was": "none", "empty_db_models": True} + snapshot = {"still_desired": result, "router_was": "none", "empty_db_models": True} + assert snapshot == {"still_desired": None, "router_was": "none", "empty_db_models": True} @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index b9a33bd2cef..1ecf7e8b83a 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -2459,13 +2459,11 @@ async def test_delete_deployment_type_mismatch(): patch("litellm.proxy.proxy_server.user_config_file_path", "test_config.yaml"), ): # Call the function under test - deleted_count = await pc._delete_deployment(db_models=[]) + still_desired = await pc._delete_deployment(db_models=[]) # The two SHA-hash models have no corresponding entry in combined_id_list # and must be evicted. - assert ( - deleted_count == 2 - ), f"Expected 2 deletions (SHA-hash models), got {deleted_count}" + assert len(deleted_ids) == 2, f"Expected 2 deletions (SHA-hash models), got {deleted_ids}" assert ( "a96e12e76b36a57cfae57a41288eb41567629cac89b4828c6f7074afc3534695" in deleted_ids @@ -2485,6 +2483,12 @@ async def test_delete_deployment_type_mismatch(): "12345679" not in deleted_ids ), f"Model 12345679 should NOT be deleted. Deleted IDs: {deleted_ids}" + assert still_desired is not None + assert {"12345678", "12345679"} <= still_desired, ( + "the int-keyed config models must come back as strings in the desired set, so a " + f"caller judging its own reload reads them as wanted rather than evicted; got {still_desired}" + ) + @pytest.mark.asyncio async def test_get_config_from_file(tmp_path, monkeypatch): @@ -9119,10 +9123,13 @@ class TestDeleteDeploymentSync: with patch.object( proxy_config, "get_config", AsyncMock(return_value={"model_list": []}) ): - count = await proxy_config._delete_deployment(db_models=[]) + still_desired = await proxy_config._delete_deployment(db_models=[]) mock_router.delete_deployment.assert_called_once_with(id="model-id-to-evict") - assert count == 1 + assert still_desired == frozenset(), ( + "an empty db and an empty config want nothing, which must stay distinct from " + f"the None returned when no reconcile ran at all; got {still_desired}" + ) @pytest.mark.asyncio async def test_update_llm_router_skips_update_on_db_fetch_failure(self): diff --git a/tests/test_litellm/proxy/test_update_llm_router_resilience.py b/tests/test_litellm/proxy/test_update_llm_router_resilience.py index fd0df4805e6..a7dc9c1783e 100644 --- a/tests/test_litellm/proxy/test_update_llm_router_resilience.py +++ b/tests/test_litellm/proxy/test_update_llm_router_resilience.py @@ -115,8 +115,10 @@ class TestDeleteDeploymentResilience: """Test _delete_deployment handles get_config failures gracefully.""" @pytest.mark.asyncio - async def test_returns_zero_when_get_config_times_out(self): - """Should return 0 (no deletions) when get_config fails, not raise.""" + async def test_returns_none_when_get_config_times_out(self): + """Should return None (no reconcile ran, desired set unknown) when get_config + fails, not raise. A caller judging its own reload must not read that as "the db + wants nothing" and blame the reload for every model it serves.""" proxy_config = ProxyConfig() db_models = [_make_db_model("gpt-5.1", "db-id-1")] @@ -136,8 +138,8 @@ class TestDeleteDeploymentResilience: ): result = await proxy_config._delete_deployment(db_models=db_models) - # Should safely return 0 instead of raising - assert result == 0 + # Should safely return None instead of raising + assert result is None # Should NOT have deleted any deployments mock_router.delete_deployment.assert_not_called() @@ -175,5 +177,8 @@ class TestDeleteDeploymentResilience: result = await proxy_config._delete_deployment(db_models=db_models) # "stale-id" should have been deleted (not in db_models or config) - assert result == 1 mock_router.delete_deployment.assert_called_once_with(id="stale-id") + assert result == frozenset({"db-id-1", "config-id-1"}), ( + "the returned set must be what the db + config still want, so a caller can " + f"tell that eviction apart from a deployment that went missing; got {result}" + ) diff --git a/tests/test_litellm/test_model_block_unblock.py b/tests/test_litellm/test_model_block_unblock.py index dc0098e405e..ff66bedf0dc 100644 --- a/tests/test_litellm/test_model_block_unblock.py +++ b/tests/test_litellm/test_model_block_unblock.py @@ -36,7 +36,7 @@ def _setup_model_block_mocks(monkeypatch, *, updated_blocked: bool): mock_router = MagicMock() mock_router.get_model_ids.return_value = [model_id] - mock_clear_cache = AsyncMock(return_value=True) + mock_clear_cache = AsyncMock(return_value=None) mock_audit_log = AsyncMock(return_value=None) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) From e38df02d859457e0d1003587e2d2e711c21cca24 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 31 Jul 2026 18:18:48 -0700 Subject: [PATCH 015/123] fix(ui): show pass through route selections and match team id substrings in team search (#35319) * fix(ui): show pass through route selections in team/key forms and match team id substrings in team search Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * perf(teams): keep team id search index-friendly with a prefix match Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(teams): keep /v2/team/list search id matching exact by default and add an opt-in prefix mode Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: milan Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../management_endpoints/team_endpoints.py | 13 ++++- .../management_endpoints/team_endpoints.py | 4 +- .../test_team_endpoints.py | 57 +++++++++++++++++-- .../app/(dashboard)/hooks/teams/useTeams.ts | 3 + ui/litellm-dashboard/src/components/Teams.tsx | 36 ++++++------ .../components/TeamsPage/TeamsTable.test.tsx | 13 +++++ .../src/components/TeamsPage/TeamsTable.tsx | 1 + .../PassThroughRoutesSelector.tsx | 2 +- .../organisms/create_key_button.tsx | 2 - .../src/components/team/TeamInfo.test.tsx | 51 +++++++++++++++++ .../src/components/team/TeamInfo.tsx | 35 ++++++------ .../components/templates/key_edit_view.tsx | 37 ++++++------ ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 +- 13 files changed, 189 insertions(+), 69 deletions(-) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index c35c17aa359..fa01f43d049 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -131,6 +131,7 @@ from litellm.types.proxy.management_endpoints.team_endpoints import ( BulkUpdateTeamMemberPermissionsRequest, BulkUpdateTeamMemberPermissionsResponse, GetTeamMemberPermissionsResponse, + TeamIdSearchMatch, TeamListItem, TeamListResponse, TeamMemberAddResult, @@ -3973,6 +3974,7 @@ async def _build_team_list_where_conditions( user_id: Optional[str], use_deleted_table: bool, search: Optional[str] = None, + search_team_id_match: TeamIdSearchMatch = "exact", org_admin_org_ids: Optional[List[str]] = None, user_api_key_cache: Optional[Any] = None, proxy_logging_obj: Optional[Any] = None, @@ -3996,7 +3998,7 @@ async def _build_team_list_where_conditions( if search: where_conditions["OR"] = [ - {"team_id": search}, + ({"team_id": {"startsWith": search}} if search_team_id_match == "prefix" else {"team_id": search}), {"team_alias": {"contains": search, "mode": "insensitive"}}, ] @@ -4230,8 +4232,14 @@ async def list_team_v2( ), search: Optional[str] = fastapi.Query( default=None, - description="Combined search: matches teams whose 'team_id' equals the value OR whose 'team_alias' contains it (case-insensitive).", + description="Combined search: matches teams whose 'team_id' matches the value OR whose 'team_alias' contains it (case-insensitive).", ), + search_team_id_match: Annotated[ + TeamIdSearchMatch, + fastapi.Query( + description="How 'search' matches 'team_id': 'exact' (default) or 'prefix' for a case-sensitive prefix match." + ), + ] = "exact", page: int = fastapi.Query(default=1, description="Page number for pagination", ge=1), page_size: int = fastapi.Query(default=10, description="Number of teams per page", ge=1, le=100), sort_by: Optional[str] = fastapi.Query( @@ -4308,6 +4316,7 @@ async def list_team_v2( user_id=user_id, use_deleted_table=use_deleted_table, search=search, + search_team_id_match=search_team_id_match, org_admin_org_ids=org_admin_org_ids, user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, diff --git a/litellm/types/proxy/management_endpoints/team_endpoints.py b/litellm/types/proxy/management_endpoints/team_endpoints.py index 0e555535874..2d9387da956 100644 --- a/litellm/types/proxy/management_endpoints/team_endpoints.py +++ b/litellm/types/proxy/management_endpoints/team_endpoints.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Union from pydantic import BaseModel @@ -11,6 +11,8 @@ from litellm.proxy._types import ( Member, ) +TeamIdSearchMatch = Literal["exact", "prefix"] + class GetTeamMemberPermissionsRequest(BaseModel): """Request to get the team member permissions for a team""" diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index a658a4b7353..25a0ff644f8 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -3627,9 +3627,9 @@ async def test_list_team_v2_with_invalid_status(): @pytest.mark.asyncio async def test_list_team_v2_search_builds_or_clause(): """ - `search` should be passed as a Prisma OR across team_id (exact) and - team_alias (case-insensitive contains), so the UI can hit a single - backend filter with either a UUID or a name fragment. + `search` should be passed as a Prisma OR across an exact team_id match and a + case-insensitive team_alias contains, so the UI needs one backend filter. + Exact id matching is the documented default and must not change. """ from unittest.mock import AsyncMock, Mock, patch @@ -3671,6 +3671,54 @@ async def test_list_team_v2_search_builds_or_clause(): } +@pytest.mark.asyncio +async def test_list_team_v2_search_team_id_match_prefix(): + """ + Opting into `search_team_id_match="prefix"` should widen the team_id side of + the search OR to an index-friendly prefix match, so the first characters of a + team id quoted in a proxy error find the team. + """ + from unittest.mock import AsyncMock, Mock, patch + + from fastapi import Request + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.team_endpoints import list_team_v2 + + mock_request = Mock(spec=Request) + mock_admin = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma_client: + mock_db = Mock() + mock_prisma_client.db = mock_db + mock_db.litellm_teamtable.find_many = AsyncMock(return_value=[]) + mock_db.litellm_teamtable.count = AsyncMock(return_value=0) + + await list_team_v2( + http_request=mock_request, + user_id=None, + organization_id=None, + team_id=None, + team_alias=None, + search="66c432fa", + search_team_id_match="prefix", + user_api_key_dict=mock_admin, + page=1, + page_size=10, + status=None, + ) + + find_many_kwargs = mock_db.litellm_teamtable.find_many.call_args.kwargs + assert find_many_kwargs["where"] == { + "OR": [ + {"team_id": {"startsWith": "66c432fa"}}, + {"team_alias": {"contains": "66c432fa", "mode": "insensitive"}}, + ] + } + + @pytest.mark.asyncio async def test_list_team_v2_search_composes_with_user_id_filter(): """ @@ -3724,6 +3772,7 @@ async def test_list_team_v2_search_composes_with_user_id_filter(): team_id=None, team_alias=None, search="team_a", + search_team_id_match="prefix", user_api_key_dict=mock_user_api_key_dict, page=1, page_size=10, @@ -3733,7 +3782,7 @@ async def test_list_team_v2_search_composes_with_user_id_filter(): find_many_kwargs = mock_db.litellm_teamtable.find_many.call_args.kwargs where = find_many_kwargs["where"] assert where["OR"] == [ - {"team_id": "team_a"}, + {"team_id": {"startsWith": "team_a"}}, {"team_alias": {"contains": "team_a", "mode": "insensitive"}}, ] assert where["team_id"] == {"in": ["team_a", "team_b"]} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts index f532c44ffd7..4061026b94d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts @@ -24,6 +24,7 @@ export interface TeamListCallOptions { teamID?: string | null; team_alias?: string | null; search?: string | null; + searchTeamIdMatch?: "exact" | "prefix" | null; userID?: string | null; sortBy?: string | null; sortOrder?: string | null; @@ -48,6 +49,7 @@ export const teamListCall = async ( organization_id: options.organizationID, team_alias: options.team_alias, search: options.search, + search_team_id_match: options.searchTeamIdMatch, user_id: options.userID, page, page_size: pageSize, @@ -213,6 +215,7 @@ const deletedTeamListCall = async ( organization_id: options.organizationID, team_alias: options.team_alias, search: options.search, + search_team_id_match: options.searchTeamIdMatch, user_id: options.userID, page, page_size: pageSize, diff --git a/ui/litellm-dashboard/src/components/Teams.tsx b/ui/litellm-dashboard/src/components/Teams.tsx index 0a3c7fc736d..582f75578ed 100644 --- a/ui/litellm-dashboard/src/components/Teams.tsx +++ b/ui/litellm-dashboard/src/components/Teams.tsx @@ -957,25 +957,23 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser placeholder="Select vector stores (optional)" /> - - - form.setFieldValue("allowed_passthrough_routes", values)} - value={form.getFieldValue("allowed_passthrough_routes")} - accessToken={accessToken || ""} - placeholder="Select pass through routes (optional)" - disabled={!premiumUser || !isProxyAdminRole(userRole || "")} - /> - + + diff --git a/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.test.tsx b/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.test.tsx index 9469be12128..09bd3e9f245 100644 --- a/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.test.tsx +++ b/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.test.tsx @@ -196,6 +196,19 @@ describe("server-side filtering maps controls to the right query params", () => expect(mockUseTeamsTable).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ search: "platform" })); }); }); + + it("opts into team id prefix matching so a partial id from a proxy error finds the team", async () => { + renderTable(); + fireEvent.change(screen.getByTestId("datatable-search"), { target: { value: "66c432fa" } }); + + await waitFor(() => { + expect(mockUseTeamsTable).toHaveBeenLastCalledWith( + 1, + 50, + expect.objectContaining({ search: "66c432fa", searchTeamIdMatch: "prefix" }), + ); + }); + }); }); describe("non-admin scoping", () => { diff --git a/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.tsx b/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.tsx index 3b75db52b16..5f5a6f26c0a 100644 --- a/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.tsx +++ b/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.tsx @@ -66,6 +66,7 @@ export function TeamsTable({ userRole, userID, onSelectTeam, onEditTeam, onDelet team_alias: getFilterValue("alias"), teamID: getFilterValue("team_id"), search: searchQuery.trim() || undefined, + searchTeamIdMatch: "prefix" as const, userID: isAdminView ? undefined : userID ?? undefined, sortBy: sorting[0]?.id, sortOrder: toSortOrder(sorting), diff --git a/ui/litellm-dashboard/src/components/common_components/PassThroughRoutesSelector.tsx b/ui/litellm-dashboard/src/components/common_components/PassThroughRoutesSelector.tsx index 5cea88b2af2..e02125dea56 100644 --- a/ui/litellm-dashboard/src/components/common_components/PassThroughRoutesSelector.tsx +++ b/ui/litellm-dashboard/src/components/common_components/PassThroughRoutesSelector.tsx @@ -3,7 +3,7 @@ import { Select } from "antd"; import { getPassThroughEndpointsCall } from "../networking"; interface PassThroughRoutesSelectorProps { - onChange: (selectedRoutes: string[]) => void; + onChange?: (selectedRoutes: string[]) => void; value?: string[]; className?: string; accessToken: string; diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 711652eb783..00f36f016b7 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -1385,8 +1385,6 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp } > form.setFieldValue("allowed_passthrough_routes", values)} - value={form.getFieldValue("allowed_passthrough_routes")} accessToken={accessToken} placeholder={ !premiumUser diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx index 25365d26a12..712cff80649 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx @@ -18,6 +18,7 @@ vi.mock("@/components/networking", () => ({ getTeamPermissionsCall: vi.fn(), organizationInfoCall: vi.fn(), getRouterSettingsCall: vi.fn().mockResolvedValue({ fields: [] }), + getPassThroughEndpointsCall: vi.fn(), })); vi.mock("@/components/utils/dataUtils", () => ({ @@ -1142,4 +1143,54 @@ describe("TeamInfoView", () => { expect(within(dropdown).getByTitle("opt-in")).toBeInTheDocument(); }); }); + + describe("allowed pass through routes", () => { + beforeEach(() => { + testQueryClient.clear(); + vi.mocked(networking.teamInfoCall).mockResolvedValue(createMockTeamData({ models: ["gpt-4"] })); + vi.mocked(networking.teamUpdateCall).mockResolvedValue({ data: {}, team_id: "123" } as any); + vi.mocked(networking.getPassThroughEndpointsCall).mockResolvedValue({ + endpoints: [{ path: "/bedrock-passthrough", methods: ["POST"] }], + }); + }); + + it("should show a route picked from the dropdown in the field and save it", async () => { + const user = userEvent.setup({ delay: null }); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.queryAllByText("Test Team").length).toBeGreaterThan(0); + }); + + await user.click(screen.getByRole("tab", { name: "Settings" })); + await user.click(await screen.findByRole("button", { name: /edit settings/i })); + + const routesLabel = await screen.findByText("Allowed Pass Through Routes"); + const routesFormItem = routesLabel.closest(".ant-form-item") as HTMLElement; + + await user.click(within(routesFormItem).getByRole("combobox")); + + const option = await screen.findByTitle("POST /bedrock-passthrough"); + await user.click(option); + + await waitFor(() => { + expect(within(routesFormItem).getByText(/\/bedrock-passthrough/)).toBeInTheDocument(); + }); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => { + expect(networking.teamUpdateCall).toHaveBeenCalledWith( + "test-token", + expect.objectContaining({ + team_id: "123", + metadata: expect.objectContaining({ + allowed_passthrough_routes: ["/bedrock-passthrough"], + }), + }), + ); + }); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index acd5a8966a7..34570043f52 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -1361,25 +1361,22 @@ const TeamInfoView: React.FC = ({ /> - - - form.setFieldValue("allowed_passthrough_routes", values)} - value={form.getFieldValue("allowed_passthrough_routes")} - accessToken={accessToken || ""} - placeholder="Select pass through routes" - disabled={!premiumUser || !is_proxy_admin} - /> - + + diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx index 962c6bc3568..4b6c266eb8f 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx @@ -689,26 +689,23 @@ export function KeyEditView({ - - - form.setFieldValue("allowed_passthrough_routes", values)} - value={form.getFieldValue("allowed_passthrough_routes")} - accessToken={accessToken || ""} - placeholder={ - !premiumUser - ? "Premium feature - Upgrade to set allowed pass through routes by key" - : Array.isArray(keyData.metadata?.allowed_passthrough_routes) && - keyData.metadata.allowed_passthrough_routes.length > 0 - ? `Current: ${keyData.metadata.allowed_passthrough_routes.join(", ")}` - : "Select or enter allowed pass through routes" - } - disabled={!premiumUser} - /> - + + 0 + ? `Current: ${keyData.metadata.allowed_passthrough_routes.join(", ")}` + : "Select or enter allowed pass through routes" + } + disabled={!premiumUser} + /> diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 9a301f1c474..109d638fb9c 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -58372,8 +58372,10 @@ export interface operations { team_id?: string | null; /** @description Only return teams which this 'team_alias' belongs to. Supports partial matching. */ team_alias?: string | null; - /** @description Combined search: matches teams whose 'team_id' equals the value OR whose 'team_alias' contains it (case-insensitive). */ + /** @description Combined search: matches teams whose 'team_id' matches the value OR whose 'team_alias' contains it (case-insensitive). */ search?: string | null; + /** @description How 'search' matches 'team_id': 'exact' (default) or 'prefix' for a case-sensitive prefix match. */ + search_team_id_match?: "exact" | "prefix"; /** @description Page number for pagination */ page?: number; /** @description Number of teams per page */ From 7c388b1fd97a88d11e7afacc8a4b6b056ae8ffeb Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 1 Aug 2026 01:39:35 +0000 Subject: [PATCH 016/123] refactor(proxy): rename the audit hook's user row variable The typed-vs-raw distinction the _litellm_typed suffix marked is gone now that the repository returns LiteLLM_UserTable directly. --- litellm/proxy/hooks/user_management_event_hooks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/hooks/user_management_event_hooks.py b/litellm/proxy/hooks/user_management_event_hooks.py index d8cfae5dab0..e40db7f3f1f 100644 --- a/litellm/proxy/hooks/user_management_event_hooks.py +++ b/litellm/proxy/hooks/user_management_event_hooks.py @@ -57,18 +57,18 @@ class UserManagementEventHooks: raise Exception(CommonProxyErrors.db_not_connected_error.value) if response.user_id is None: raise Exception("no user_id returned for the newly created user") - user_row_litellm_typed = await UserRepository(prisma_client).find_by_id(response.user_id) - if user_row_litellm_typed is None: + user_row = await UserRepository(prisma_client).find_by_id(response.user_id) + if user_row is None: raise Exception(f"no user row found for user_id={response.user_id}") asyncio.create_task( UserManagementEventHooks.create_internal_user_audit_log( - user_id=user_row_litellm_typed.user_id, + user_id=user_row.user_id, action="created", litellm_changed_by=user_api_key_dict.user_id, user_api_key_dict=user_api_key_dict, litellm_proxy_admin_name=litellm_proxy_admin_name, before_value=None, - after_value=user_row_litellm_typed.model_dump_json(exclude_none=True), + after_value=user_row.model_dump_json(exclude_none=True), ) ) except Exception as e: From e8e2e07ef67010685839f79c0b0ad18dd9ec7ced Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 31 Jul 2026 19:57:52 -0700 Subject: [PATCH 017/123] fix(proxy): align team member add with existing user provisioning rules Adding a team member by a user_id with no user row created that row as a side effect for any caller permitted to add members, while creating users directly is restricted to proxy admins. Restrict that path to proxy admins too; adding an existing user, and inviting a new one by user_email (where the user_id is allocated server-side), are unchanged. Also record the membership change, and any user row it creates, in the audit log, matching /team/update, /user/new and /key/*. --- .../management_endpoints/team_endpoints.py | 147 +++++++++++ .../test_team_endpoints.py | 236 ++++++++++++++++++ 2 files changed, 383 insertions(+) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index fa01f43d049..7b5ce33cb8b 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -13,6 +13,7 @@ import asyncio import json import math import traceback +from collections.abc import Sequence from datetime import datetime, timezone from typing import Annotated, Any, Dict, List, Mapping, Optional, Tuple, Union, cast @@ -2426,6 +2427,123 @@ def _emit_team_members_metric(team: LiteLLM_TeamTable) -> None: verbose_proxy_logger.debug("Prometheus: failed to emit team members metric: %s", str(e)) +async def _resolve_existing_member_user_ids( + members: Sequence[Member], + prisma_client: PrismaClient, +) -> frozenset[str]: + """Return the caller-supplied user_ids that already have a user row.""" + user_repository = UserRepository(prisma_client) + found = await asyncio.gather( + *(user_repository.find_by_id(member.user_id) for member in members if member.user_id is not None) + ) + return frozenset(user.user_id for user in found if user is not None and user.user_id is not None) + + +def _pre_existing_user_ids( + members: Sequence[Member], + caller_supplied_user_ids: frozenset[str], + existing_user_ids: frozenset[str], +) -> frozenset[str]: + """Return the user_ids that already had a user row before this request. + + Combines the caller-supplied ids that resolved to a user with the ids + ``_validate_and_populate_member_user_info`` filled in, which it only does + from a matched user row. Deriving it that way keeps this in step with the + email matching that resolution performs, rather than repeating it here. + """ + populated_user_ids = frozenset( + member.user_id + for member in members + if member.user_id is not None and member.user_id not in caller_supplied_user_ids + ) + return existing_user_ids | populated_user_ids + + +def _validate_member_user_id_provisioning( + members: Sequence[Member], + existing_user_ids: frozenset[str], + user_api_key_dict: UserAPIKeyAuth, +) -> None: + """Restrict adding a caller-chosen user_id that has no user row yet to proxy admins. + + Team and org admins keep the ability to add users that already exist and to + invite new ones by user_email, where the user_id is allocated server-side. + """ + if user_api_key_dict.user_role in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN.value, + ): + return + + unknown_user_ids = tuple( + member.user_id for member in members if member.user_id is not None and member.user_id not in existing_user_ids + ) + if not unknown_user_ids: + return + + raise HTTPException( + status_code=403, + detail={ # mutable-ok: HTTPException detail must be a plain mapping to keep this route's {"error": ...} response shape + "error": ( + "Only proxy admins can add a user_id that does not exist yet: {}. " + "Add the member by user_email to invite a new user, or ask a proxy admin " + "to create the user first.".format(", ".join(unknown_user_ids)) + ) + }, + ) + + +def _members_audit_value(members: Sequence[Member]) -> str: + """Serialize a team's member list for an audit-log value. + + The audit-log columns hold a JSON object, so the member list is nested + under a key rather than serialized as a top-level array. + """ + return safe_dumps( + { # mutable-ok: the audit-log JSON column rejects a top-level array, so this value must be an object + "members_with_roles": tuple(member.model_dump() for member in members) + } + ) + + +async def _create_team_member_add_audit_logs( + team_id: str, + updated_users: Sequence[LiteLLM_UserTable], + existing_user_ids: frozenset[str], + before_members: Sequence[Member], + after_members: Sequence[Member], + user_api_key_dict: UserAPIKeyAuth, + litellm_proxy_admin_name: str, +) -> None: + """Record the membership change, and any user row it created, in the audit log.""" + from litellm.proxy.management_helpers.audit_logs import create_object_audit_log + + for user in updated_users: + if user.user_id is None or user.user_id in existing_user_ids: + continue + await create_object_audit_log( + object_id=user.user_id, + action="created", + litellm_changed_by=None, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + table_name=LitellmTableNames.USER_TABLE_NAME, + before_value=None, + after_value=safe_dumps(user.model_dump(exclude_none=True)), + ) + + await create_object_audit_log( + object_id=team_id, + action="updated", + litellm_changed_by=None, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + table_name=LitellmTableNames.TEAM_TABLE_NAME, + before_value=_members_audit_value(before_members), + after_value=_members_audit_value(after_members), + ) + + async def _validate_and_populate_member_user_info( member: Member, prisma_client: PrismaClient, @@ -2606,6 +2724,19 @@ async def team_member_add( data=data, ) + requested_members = tuple(data.member) if isinstance(data.member, list) else (data.member,) + caller_supplied_user_ids = frozenset(member.user_id for member in requested_members if member.user_id is not None) + existing_user_ids = await _resolve_existing_member_user_ids( + members=requested_members, + prisma_client=prisma_client, + ) + _validate_member_user_id_provisioning( + members=requested_members, + existing_user_ids=existing_user_ids, + user_api_key_dict=user_api_key_dict, + ) + members_before_add = tuple(complete_team_data.members_with_roles) + # Validate and populate user_email/user_id for members before processing if isinstance(data.member, Member): await _validate_and_populate_member_user_info( @@ -2619,6 +2750,12 @@ async def team_member_add( prisma_client=prisma_client, ) + pre_existing_user_ids = _pre_existing_user_ids( + members=requested_members, + caller_supplied_user_ids=caller_supplied_user_ids, + existing_user_ids=existing_user_ids, + ) + ( updated_team, updated_users, @@ -2637,6 +2774,16 @@ async def team_member_add( _emit_team_members_metric(complete_team_data) + await _create_team_member_add_audit_logs( + team_id=data.team_id, + updated_users=updated_users, + existing_user_ids=pre_existing_user_ids, + before_members=members_before_add, + after_members=tuple(complete_team_data.members_with_roles), + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ) + return TeamAddMemberResponse.model_validate( { **updated_team.model_dump(), diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 25a0ff644f8..b1438447e4f 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -10338,3 +10338,239 @@ async def test_list_available_teams_filters_joined_and_validates_rows(monkeypatc assert result[0].team_alias == "open team" find_many_kwargs = mock_prisma_client.db.litellm_teamtable.find_many.call_args.kwargs assert find_many_kwargs["where"] == {"team_id": {"in": ["team-open"]}} + + +def _provisioning_caller(role: LitellmUserRoles) -> UserAPIKeyAuth: + return UserAPIKeyAuth(user_id="caller-1", user_role=role) + + +def test_validate_member_user_id_provisioning_allows_proxy_admin(): + """Proxy admins may add a user_id that has no user row yet.""" + from litellm.proxy.management_endpoints.team_endpoints import ( + _validate_member_user_id_provisioning, + ) + + _validate_member_user_id_provisioning( + members=[Member(user_id="brand-new", role="user")], + existing_user_ids=frozenset(), + user_api_key_dict=_provisioning_caller(LitellmUserRoles.PROXY_ADMIN), + ) + + +def test_validate_member_user_id_provisioning_rejects_unknown_user_id_for_non_proxy_admin(): + """A non-proxy-admin cannot add a user_id that has no user row yet.""" + from litellm.proxy.management_endpoints.team_endpoints import ( + _validate_member_user_id_provisioning, + ) + + with pytest.raises(HTTPException) as exc_info: + _validate_member_user_id_provisioning( + members=[Member(user_id="brand-new", role="user")], + existing_user_ids=frozenset(), + user_api_key_dict=_provisioning_caller(LitellmUserRoles.INTERNAL_USER), + ) + + assert exc_info.value.status_code == 403 + assert "brand-new" in str(exc_info.value.detail) + + +def test_validate_member_user_id_provisioning_allows_existing_user_id_for_non_proxy_admin(): + """A non-proxy-admin may still add a user that already exists.""" + from litellm.proxy.management_endpoints.team_endpoints import ( + _validate_member_user_id_provisioning, + ) + + _validate_member_user_id_provisioning( + members=[Member(user_id="already-here", role="user")], + existing_user_ids=frozenset({"already-here"}), + user_api_key_dict=_provisioning_caller(LitellmUserRoles.INTERNAL_USER), + ) + + +def test_validate_member_user_id_provisioning_allows_email_only_member_for_non_proxy_admin(): + """Inviting by user_email stays open to non-proxy-admins; the user_id is server-allocated.""" + from litellm.proxy.management_endpoints.team_endpoints import ( + _validate_member_user_id_provisioning, + ) + + _validate_member_user_id_provisioning( + members=[Member(user_email="invitee@example.com", role="user")], + existing_user_ids=frozenset(), + user_api_key_dict=_provisioning_caller(LitellmUserRoles.INTERNAL_USER), + ) + + +def test_validate_member_user_id_provisioning_rejects_unknown_user_id_paired_with_email(): + """Supplying a user_email alongside an unknown user_id does not lift the restriction.""" + from litellm.proxy.management_endpoints.team_endpoints import ( + _validate_member_user_id_provisioning, + ) + + with pytest.raises(HTTPException) as exc_info: + _validate_member_user_id_provisioning( + members=[Member(user_id="chosen-id", user_email="invitee@example.com", role="user")], + existing_user_ids=frozenset(), + user_api_key_dict=_provisioning_caller(LitellmUserRoles.INTERNAL_USER), + ) + + assert exc_info.value.status_code == 403 + + +def test_validate_member_user_id_provisioning_reports_every_unknown_member(): + """A bulk add names each unknown user_id rather than only the first.""" + from litellm.proxy.management_endpoints.team_endpoints import ( + _validate_member_user_id_provisioning, + ) + + with pytest.raises(HTTPException) as exc_info: + _validate_member_user_id_provisioning( + members=[ + Member(user_id="known", role="user"), + Member(user_id="unknown-a", role="user"), + Member(user_id="unknown-b", role="user"), + ], + existing_user_ids=frozenset({"known"}), + user_api_key_dict=_provisioning_caller(LitellmUserRoles.INTERNAL_USER), + ) + + detail = str(exc_info.value.detail) + assert "unknown-a" in detail + assert "unknown-b" in detail + + +@pytest.mark.asyncio +async def test_resolve_existing_member_user_ids_matches_caller_supplied_user_ids(): + """Only caller-supplied user_ids are looked up; unknown ones resolve to nothing.""" + from litellm.proxy.management_endpoints.team_endpoints import ( + _resolve_existing_member_user_ids, + ) + + prisma_client = MagicMock() + + async def find_by_id(user_id): + if user_id == "by-id": + return LiteLLM_UserTable(user_id="by-id", max_budget=None, spend=0.0, user_email=None, models=[]) + return None + + with patch("litellm.proxy.management_endpoints.team_endpoints.UserRepository") as repo: + repo.return_value.find_by_id = AsyncMock(side_effect=find_by_id) + + resolved = await _resolve_existing_member_user_ids( + members=[ + Member(user_id="by-id", role="user"), + Member(user_id="missing", role="user"), + Member(user_email="someone@example.com", role="user"), + ], + prisma_client=prisma_client, + ) + + assert resolved == frozenset({"by-id"}) + + +def test_pre_existing_user_ids_counts_ids_filled_in_by_member_resolution(): + """An id the member-resolution step filled in came from a matched row, so it pre-existed. + + This is what keeps a case-variant email invite of an existing user from being + recorded as a newly created user. + """ + from litellm.proxy.management_endpoints.team_endpoints import _pre_existing_user_ids + + # member arrived email-only; resolution matched an existing row and filled in the id + resolved_member = Member(user_id="matched-existing", user_email="Someone@Example.com", role="user") + + assert _pre_existing_user_ids( + members=[resolved_member], + caller_supplied_user_ids=frozenset(), + existing_user_ids=frozenset(), + ) == frozenset({"matched-existing"}) + + +def test_pre_existing_user_ids_excludes_caller_supplied_ids_that_do_not_exist(): + """A caller-supplied id that resolved to nothing is genuinely new, so it stays out.""" + from litellm.proxy.management_endpoints.team_endpoints import _pre_existing_user_ids + + assert _pre_existing_user_ids( + members=[Member(user_id="brand-new", role="user"), Member(user_id="already-here", role="user")], + caller_supplied_user_ids=frozenset({"brand-new", "already-here"}), + existing_user_ids=frozenset({"already-here"}), + ) == frozenset({"already-here"}) + + +def test_members_audit_value_serializes_to_a_json_object(): + """The audit-log columns hold a JSON object; a top-level array is rejected by the DB.""" + from litellm.proxy.management_endpoints.team_endpoints import _members_audit_value + + payload = json.loads(_members_audit_value([Member(user_id="u1", role="admin"), Member(user_id="u2", role="user")])) + + assert isinstance(payload, dict) + assert [m["user_id"] for m in payload["members_with_roles"]] == ["u1", "u2"] + + +@pytest.mark.asyncio +async def test_team_member_add_audits_a_user_created_from_a_list_payload(monkeypatch): + """A user created by a list payload must still be reported as newly created. + + For a list payload the member-list reconciliation back-fills the caller's own + Member objects with the ids of users this request just created. The set of + pre-existing ids therefore has to be captured before that runs, otherwise a + freshly created user looks like it was already there and no creation is recorded. + """ + from litellm.proxy._types import TeamMemberAddRequest + from litellm.proxy.management_endpoints.team_endpoints import team_member_add + + team_id = "team-list-audit" + created_user_id = "generated-uuid-for-new-invitee" + member = Member(user_email="invitee@example.com", role="user") + + mock_prisma_client = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + monkeypatch.setattr("litellm.proxy.proxy_server.litellm_proxy_admin_name", "default_user_id") + + team_row = LiteLLM_TeamTable(team_id=team_id, members_with_roles=[]) + created_user = LiteLLM_UserTable( + user_id=created_user_id, user_email="invitee@example.com", max_budget=None, spend=0.0, models=[] + ) + updated_team = MagicMock() + updated_team.model_dump.return_value = {"team_id": team_id, "members_with_roles": []} + + async def fake_add_team_members_to_team(**kwargs): + # mirrors _update_team_members_list: the list branch mutates the caller's Member in place + member.user_id = created_user_id + return updated_team, [created_user], [] + + with ( + patch( + "litellm.proxy.management_endpoints.team_endpoints.get_team_object", + new_callable=AsyncMock, + return_value=team_row, + ), + patch( + "litellm.proxy.management_endpoints.team_endpoints._validate_team_member_add_permissions", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.team_endpoints._validate_and_populate_member_user_info", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.team_endpoints._resolve_existing_member_user_ids", + new_callable=AsyncMock, + return_value=frozenset(), + ), + patch( + "litellm.proxy.management_endpoints.team_endpoints._add_team_members_to_team", + side_effect=fake_add_team_members_to_team, + ), + patch( + "litellm.proxy.management_endpoints.team_endpoints._create_team_member_add_audit_logs", + new_callable=AsyncMock, + ) as mock_audit, + ): + await team_member_add( + data=TeamMemberAddRequest(team_id=team_id, member=[member]), + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin-1"), + ) + + mock_audit.assert_called_once() + assert created_user_id not in mock_audit.call_args.kwargs["existing_user_ids"] From 629d58443ea77bd6156db09f0dfa1280acd6fd20 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 31 Jul 2026 20:08:02 -0700 Subject: [PATCH 018/123] feat(proxy): push config sync to pods via redis pub/sub After any management write to a DB-backed config table, publish an invalidation event on the coordination Redis; every pod runs a subscriber that debounces, jitters, and triggers an immediate add_deployment plus get_credentials resync. The interval polls stay as slow reconciliation fallback and behavior without Redis is unchanged since publish and subscribe both no-op. --- .../proxy/common_utils/config_sync_pubsub.py | 258 ++++++++ .../key_management_endpoints.py | 5 + .../model_management_endpoints.py | 7 + litellm/proxy/proxy_server.py | 27 +- .../proxy_setting_endpoints.py | 2 + litellm/proxy/utils.py | 11 +- .../repositories/credentials_repository.py | 6 +- litellm/repositories/model_repository.py | 6 +- litellm/repositories/table_repositories.py | 7 +- ruff.toml | 2 +- .../common_utils/test_config_sync_pubsub.py | 600 ++++++++++++++++++ .../repositories/test_repositories.py | 58 +- 12 files changed, 978 insertions(+), 11 deletions(-) create mode 100644 litellm/proxy/common_utils/config_sync_pubsub.py create mode 100644 tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py diff --git a/litellm/proxy/common_utils/config_sync_pubsub.py b/litellm/proxy/common_utils/config_sync_pubsub.py new file mode 100644 index 00000000000..4292c67c8c2 --- /dev/null +++ b/litellm/proxy/common_utils/config_sync_pubsub.py @@ -0,0 +1,258 @@ +import asyncio +import json +import random +from collections.abc import Awaitable, Callable +from dataclasses import asdict, dataclass +from typing import TYPE_CHECKING, Protocol, cast # noqa: TID251 # untyped prisma/redis boundary needs cast + +from litellm._logging import verbose_proxy_logger + +if TYPE_CHECKING: + from litellm.caching.redis_cache import RedisCache + + +class _ConfigSyncPubSub(Protocol): + def subscribe(self, *channels: str) -> Awaitable[object]: ... + + def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Awaitable[object]: ... + + def aclose(self) -> Awaitable[object]: ... + + +class _ConfigSyncPubSubClient(Protocol): + def publish(self, channel: str, message: str) -> Awaitable[int]: ... + + def pubsub(self) -> _ConfigSyncPubSub: ... + + +CONFIG_SYNC_CHANNEL = "litellm_proxy.config_change" +CONFIG_SYNC_DEBOUNCE_SECONDS = 1.0 +CONFIG_SYNC_JITTER_MAX_SECONDS = 5.0 +_POLL_TIMEOUT_SECONDS = 1.0 +_BACKOFF_INITIAL_SECONDS = 5.0 +_BACKOFF_MAX_SECONDS = 60.0 + +_WRITE_ACTION_NAMES: frozenset[str] = frozenset( + {"create", "create_many", "update", "update_many", "upsert", "delete", "delete_many"} +) + +_CONFIG_SYNCED_TABLE_NAMES: frozenset[str] = frozenset( + { + "litellm_proxymodeltable", + "litellm_credentialstable", + "litellm_guardrailstable", + "litellm_policytable", + "litellm_policyattachmenttable", + "litellm_managedvectorstorestable", + "litellm_managedvectorstoreindextable", + "litellm_mcpservertable", + "litellm_agentstable", + "litellm_prompttable", + "litellm_searchtoolstable", + "litellm_ssoconfig", + "litellm_cacheconfig", + "litellm_configoverrides", + } +) + + +def coordination_redis_cache() -> "RedisCache | None": + from litellm.proxy.proxy_server import redis_usage_cache + + return redis_usage_cache + + +def config_sync_channel(redis_cache: "RedisCache") -> str: + if redis_cache.namespace is None: + return CONFIG_SYNC_CHANNEL + return f"{redis_cache.namespace}:{CONFIG_SYNC_CHANNEL}" + + +def _raw_async_client(redis_cache: "RedisCache") -> object: + return cast( # cast-ok: redis-py generics leave the client type partially unknown + object, + redis_cache.init_async_client(), # pyright: ignore[reportUnknownMemberType] # redis generics + ) + + +def _pubsub_capable_client(redis_cache: "RedisCache") -> _ConfigSyncPubSubClient | None: + from redis.asyncio import Redis + + client = _raw_async_client(redis_cache) + if isinstance(client, Redis): + return cast(_ConfigSyncPubSubClient, client) # cast-ok: protocol view of the standalone redis client + return None + + +@dataclass(frozen=True, slots=True) +class _ConfigChangeMessage: + object_type: str + + +def _config_change_message_json(object_type: str) -> str: + return json.dumps(asdict(_ConfigChangeMessage(object_type=object_type))) + + +async def publish_config_change(redis_cache: "RedisCache | None", object_type: str) -> None: + if redis_cache is None: + return + try: + client = _pubsub_capable_client(redis_cache) + if client is None: + verbose_proxy_logger.debug( + "config sync publish for %s skipped: cluster redis client has no pub/sub support", + object_type, + ) + return + await client.publish(config_sync_channel(redis_cache), _config_change_message_json(object_type)) + except Exception as e: # noqa: BLE001 # best-effort publish; writes must never fail on redis errors + verbose_proxy_logger.warning("config sync publish for %s failed: %s", object_type, e) + + +async def publish_config_change_for_object_type(object_type: str) -> None: + await publish_config_change(redis_cache=coordination_redis_cache(), object_type=object_type) + + +class _PublishOnWriteActions: + __slots__ = ("_actions", "_object_type", "_publish") + + def __init__(self, actions: object, object_type: str, publish: Callable[[str], Awaitable[None]]) -> None: + self._actions = actions + self._object_type = object_type + self._publish = publish + + def __getattr__(self, name: str) -> object: + attribute = cast(object, getattr(self._actions, name)) # cast-ok: getattr on dynamic prisma actions + if name not in _WRITE_ACTION_NAMES: + return attribute + write_action = cast(Callable[..., Awaitable[object]], attribute) # cast-ok: prisma actions are untyped + object_type = self._object_type + publish = self._publish + + async def _write_then_publish( + *args: object, + **kwargs: object, # kwargs-ok: transparent passthrough to untyped prisma action + ) -> object: + result = await write_action(*args, **kwargs) + await publish(object_type) + return result + + return _write_then_publish + + +def wrap_table_actions_for_config_sync( + actions: object, + table_name: str, + publish: Callable[[str], Awaitable[None]] = publish_config_change_for_object_type, +) -> object: + if table_name not in _CONFIG_SYNCED_TABLE_NAMES: + return actions + return _PublishOnWriteActions(actions=actions, object_type=table_name, publish=publish) + + +class ConfigSyncSubscriber: + __slots__ = ( + "_backoff_initial_seconds", + "_backoff_max_seconds", + "_debounce_seconds", + "_jitter_max_seconds", + "_redis_cache", + "_resync_callbacks", + "_rng", + "_sleep", + "_task", + ) + + def __init__( + self, + redis_cache: "RedisCache", + resync_callbacks: tuple[Callable[[], Awaitable[None]], ...], + debounce_seconds: float = CONFIG_SYNC_DEBOUNCE_SECONDS, + jitter_max_seconds: float = CONFIG_SYNC_JITTER_MAX_SECONDS, + backoff_initial_seconds: float = _BACKOFF_INITIAL_SECONDS, + backoff_max_seconds: float = _BACKOFF_MAX_SECONDS, + rng: random.Random | None = None, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + ) -> None: + self._redis_cache = redis_cache + self._resync_callbacks = resync_callbacks + self._debounce_seconds = debounce_seconds + self._jitter_max_seconds = jitter_max_seconds + self._backoff_initial_seconds = backoff_initial_seconds + self._backoff_max_seconds = backoff_max_seconds + self._rng = rng if rng is not None else random.Random() + self._sleep = sleep + self._task: asyncio.Task[None] | None = None + + def start(self) -> None: + if self._task is not None: + return + self._task = asyncio.create_task(self._run()) + + async def stop(self) -> None: + task = self._task + if task is None: + return + self._task = None + _ = task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + async def _run(self) -> None: + backoff_seconds = self._backoff_initial_seconds + while True: + try: + client = _pubsub_capable_client(self._redis_cache) + if client is None: + verbose_proxy_logger.warning( + "config sync subscriber disabled: cluster redis client has no pub/sub support; " + "interval polling remains the only sync mechanism" + ) + return + pubsub = client.pubsub() + try: + await pubsub.subscribe(config_sync_channel(self._redis_cache)) + backoff_seconds = self._backoff_initial_seconds + await self._consume(pubsub) + finally: + await self._close_pubsub(pubsub) + except asyncio.CancelledError: + raise + except Exception as e: # noqa: BLE001 # any redis failure falls through to backoff and reconnect + verbose_proxy_logger.warning( + "config sync subscriber redis error: %s; reconnecting in %.0fs", + e, + backoff_seconds, + ) + await self._sleep(backoff_seconds) + backoff_seconds = min(backoff_seconds * 2, self._backoff_max_seconds) + + async def _consume(self, pubsub: _ConfigSyncPubSub) -> None: + while True: + message = await pubsub.get_message(ignore_subscribe_messages=True, timeout=_POLL_TIMEOUT_SECONDS) + if message is None: + continue + await self._sleep(self._debounce_seconds + self._rng.uniform(0.0, self._jitter_max_seconds)) + await self._drain_pending(pubsub) + await self._run_resync_callbacks() + + @staticmethod + async def _drain_pending(pubsub: _ConfigSyncPubSub) -> None: + while await pubsub.get_message(ignore_subscribe_messages=True, timeout=0) is not None: + pass + + async def _run_resync_callbacks(self) -> None: + for callback in self._resync_callbacks: + try: + await callback() + except Exception as e: # noqa: BLE001 # one failing resync callback must not kill the subscriber + verbose_proxy_logger.warning("config sync resync callback failed: %s", e) + + @staticmethod + async def _close_pubsub(pubsub: _ConfigSyncPubSub) -> None: + try: + await pubsub.aclose() + except Exception as e: # noqa: BLE001 # best-effort close of a possibly-broken connection + verbose_proxy_logger.debug("config sync pubsub close failed: %s", e) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index a94a75fdfa3..1fbfda8bf8a 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -60,6 +60,10 @@ from litellm.proxy.common_utils.callback_utils import ( decrypt_callback_vars, encrypt_callback_vars, ) +from litellm.proxy.common_utils.config_sync_pubsub import ( + coordination_redis_cache, + publish_config_change, +) from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_keys from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache @@ -4240,6 +4244,7 @@ async def _rotate_master_key( await tx.litellm_proxymodeltable.create_many( data=new_models, ) + await publish_config_change(redis_cache=coordination_redis_cache(), object_type="litellm_proxymodeltable") # 3. process config table try: config = await ConfigRepository(prisma_client).table.find_many() diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index b6422d7f5ae..21395658dac 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -38,6 +38,10 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.config_sync_pubsub import ( + coordination_redis_cache, + publish_config_change, +) from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper from litellm.proxy.management_endpoints.common_utils import _is_user_team_admin from litellm.proxy.management_endpoints.team_endpoints import ( @@ -809,6 +813,9 @@ async def delete_team_models( await tx.litellm_proxymodeltable.delete_many(where={"model_id": {"in": model_ids}}) deleted_model_ids.extend(model_ids) + if deleted_model_ids: + await publish_config_change(redis_cache=coordination_redis_cache(), object_type="litellm_proxymodeltable") + if llm_router is not None: for model_id in deleted_model_ids: llm_router.delete_deployment(id=model_id) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a60ea2da019..6bb0032b38c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -301,6 +301,7 @@ from litellm.proxy.common_request_processing import ( create_response, ) from litellm.proxy.common_utils.callback_utils import initialize_callbacks_on_proxy +from litellm.proxy.common_utils.config_sync_pubsub import ConfigSyncSubscriber from litellm.proxy.common_utils.debug_utils import init_verbose_loggers from litellm.proxy.common_utils.debug_utils import router as debugging_endpoints_router from litellm.proxy.common_utils.encrypt_decrypt_utils import ( @@ -546,6 +547,7 @@ from litellm.proxy.utils import ( _get_redoc_url, _is_projected_spend_over_limit, _is_valid_team_configs, + evict_config_param, get_config_param, get_custom_url, get_error_message_str, @@ -1151,6 +1153,12 @@ async def proxy_startup_event(app: FastAPI): except Exception as e: verbose_proxy_logger.error(f"Error stopping DB health watchdog task: {e}") + if proxy_config.config_sync_subscriber is not None: + try: + await proxy_config.config_sync_subscriber.stop() + except Exception as e: + verbose_proxy_logger.error(f"Error stopping config sync subscriber: {e}") + await proxy_shutdown_event() # type: ignore[reportGeneralTypeIssues] @@ -3837,6 +3845,7 @@ class ProxyConfig: self._last_semantic_filter_config: Optional[Dict[str, Any]] = None self._last_hashicorp_vault_config: Optional[Dict[str, Any]] = None self.worker_registry: List["WorkerRegistryEntry"] = [] + self.config_sync_subscriber: ConfigSyncSubscriber | None = None def is_yaml(self, config_file_path: str) -> bool: if not os.path.isfile(config_file_path): @@ -6495,7 +6504,7 @@ class ProxyConfig: }, }, ) - await invalidate_config_param("model_cost_map_reload_config") + await evict_config_param("model_cost_map_reload_config") verbose_proxy_logger.info( f"Model cost map reloaded successfully. Models count: {len(new_model_cost_map) if new_model_cost_map else 0}" @@ -6590,7 +6599,7 @@ class ProxyConfig: }, }, ) - await invalidate_config_param("anthropic_beta_headers_reload_config") + await evict_config_param("anthropic_beta_headers_reload_config") # Count providers in config provider_count = sum(1 for k in new_config.keys() if k != "provider_aliases" and k != "description") @@ -8165,6 +8174,20 @@ class ProxyStartupEvent: ) await proxy_config.get_credentials(prisma_client=prisma_client) + if redis_usage_cache is not None and proxy_config.config_sync_subscriber is None: + + async def _resync_config_from_db() -> None: + await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) + + async def _resync_credentials_from_db() -> None: + await proxy_config.get_credentials(prisma_client=prisma_client) + + proxy_config.config_sync_subscriber = ConfigSyncSubscriber( + redis_cache=redis_usage_cache, + resync_callbacks=(_resync_config_from_db, _resync_credentials_from_db), + ) + proxy_config.config_sync_subscriber.start() + if store_model_in_db is not True: await proxy_config.init_mcp_servers_from_db() if prisma_client is not None: diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 8f3f8ad1bfc..e4872a4b6b5 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -21,6 +21,7 @@ from litellm.proxy.config_resolvers.sso import ( SSO_SECRET_FIELDS, resolve_sso_config, ) +from litellm.proxy.utils import invalidate_config_param from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.table_repositories import ( SSOConfigRepository, @@ -971,6 +972,7 @@ async def update_sso_settings( "param_value": json.dumps(filtered_env_vars, default=str), }, ) + await invalidate_config_param("environment_variables") except Exception as e: raise HTTPException( status_code=500, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 2ca251a3211..fbcca73e779 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -119,6 +119,10 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.route_checks import RouteChecks +from litellm.proxy.common_utils.config_sync_pubsub import ( + coordination_redis_cache, + publish_config_change, +) from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.db.create_views import ( create_missing_views, @@ -2971,9 +2975,14 @@ async def get_config_param(prisma_client: Any, param_name: str) -> Optional[Any] return row +async def evict_config_param(param_name: str) -> None: + await litellm_config_cache.async_delete_cache(_config_cache_key(param_name)) + + async def invalidate_config_param(param_name: str) -> None: """Evict from both cache layers; call after every LiteLLM_Config write.""" - await litellm_config_cache.async_delete_cache(_config_cache_key(param_name)) + await evict_config_param(param_name) + await publish_config_change(redis_cache=coordination_redis_cache(), object_type=param_name) async def prefetch_config_params(prisma_client: Any, param_names: List[str]) -> None: diff --git a/litellm/repositories/credentials_repository.py b/litellm/repositories/credentials_repository.py index b5a315d233c..8e4b9ac0be7 100644 --- a/litellm/repositories/credentials_repository.py +++ b/litellm/repositories/credentials_repository.py @@ -9,6 +9,7 @@ so reads return the stored values verbatim. from typing import Any, Dict, Optional from litellm.models.credentials import CredentialItem +from litellm.proxy.common_utils.config_sync_pubsub import wrap_table_actions_for_config_sync class CredentialsRepository: @@ -25,7 +26,10 @@ class CredentialsRepository: @property def table(self) -> Any: - return self.prisma_client.db.litellm_credentialstable + return wrap_table_actions_for_config_sync( + actions=self.prisma_client.db.litellm_credentialstable, + table_name="litellm_credentialstable", + ) @staticmethod def _to_model(record: Any) -> Optional[CredentialItem]: diff --git a/litellm/repositories/model_repository.py b/litellm/repositories/model_repository.py index 0da51519964..50a50cc60d6 100644 --- a/litellm/repositories/model_repository.py +++ b/litellm/repositories/model_repository.py @@ -7,6 +7,7 @@ from typing import Any, Dict, List, Optional, Type from litellm.models.model import LiteLLM_ProxyModelTable from litellm.repositories.base_repository import BaseRepository +from litellm.proxy.common_utils.config_sync_pubsub import wrap_table_actions_for_config_sync from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, @@ -22,7 +23,10 @@ class ModelRepository(BaseRepository[LiteLLM_ProxyModelTable]): @property def table(self) -> Any: - return self.prisma_client.db.litellm_proxymodeltable + return wrap_table_actions_for_config_sync( + actions=self.prisma_client.db.litellm_proxymodeltable, + table_name="litellm_proxymodeltable", + ) @property def model_class(self) -> Type[LiteLLM_ProxyModelTable]: diff --git a/litellm/repositories/table_repositories.py b/litellm/repositories/table_repositories.py index 54008c0950c..af8be986831 100644 --- a/litellm/repositories/table_repositories.py +++ b/litellm/repositories/table_repositories.py @@ -9,6 +9,8 @@ methods; richer repositories live in their own modules. from typing import Any +from litellm.proxy.common_utils.config_sync_pubsub import wrap_table_actions_for_config_sync + class PrismaTableRepository: """Base for repositories that expose a single Prisma table.""" @@ -26,7 +28,10 @@ class PrismaTableRepository: @property def table(self) -> Any: - return getattr(self.prisma_client.db, self.table_name) + return wrap_table_actions_for_config_sync( + actions=getattr(self.prisma_client.db, self.table_name), + table_name=self.table_name, + ) class PolicyRepository(PrismaTableRepository): diff --git a/ruff.toml b/ruff.toml index 2ea9d7260fb..b652e206f41 100644 --- a/ruff.toml +++ b/ruff.toml @@ -6,7 +6,7 @@ lint.extend-select = ["T20", "PGH004", "RUF008", "RUF009", "RUF100"] # litellm's own ruff config both rely on suppressions this config can't see. lint.external = [ # Enforced by the strict-rule gate (scripts/ruff_strict_gate.py + ruff-strict.toml) - "C901", + "C901", "TID251", # Enforced by upstream litellm's ruff config, but not run in this repo's CI "PLC0415", "E402", "BLE001", "ARG002", "S102", "S324", "S606", "D401", "F403", "F405", ] diff --git a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py new file mode 100644 index 00000000000..8a8ced8bc41 --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py @@ -0,0 +1,600 @@ +import asyncio +import json +import random +from typing import Callable, Coroutine, Iterable, List, Optional, Tuple +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from redis.asyncio import Redis + +import litellm +from litellm.proxy.common_utils.config_sync_pubsub import ( + CONFIG_SYNC_CHANNEL, + CONFIG_SYNC_JITTER_MAX_SECONDS, + ConfigSyncSubscriber, + _CONFIG_SYNCED_TABLE_NAMES, + _PublishOnWriteActions, + _WRITE_ACTION_NAMES, + publish_config_change, + wrap_table_actions_for_config_sync, +) + +_EXPECTED_WRITE_ACTION_NAMES = ( + "create", + "create_many", + "delete", + "delete_many", + "update", + "update_many", + "upsert", +) + +_EXPECTED_CONFIG_SYNCED_TABLE_NAMES = frozenset( + { + "litellm_agentstable", + "litellm_cacheconfig", + "litellm_configoverrides", + "litellm_credentialstable", + "litellm_guardrailstable", + "litellm_managedvectorstoreindextable", + "litellm_managedvectorstorestable", + "litellm_mcpservertable", + "litellm_policyattachmenttable", + "litellm_policytable", + "litellm_prompttable", + "litellm_proxymodeltable", + "litellm_searchtoolstable", + "litellm_ssoconfig", + } +) + + +class _RecordingRedisClient(Redis): + def __init__(self) -> None: + self.published: List[Tuple[str, str]] = [] + + async def publish(self, channel: str, message: str) -> int: + self.published.append((channel, message)) + return 1 + + +class _FailingPublishRedisClient(Redis): + def __init__(self) -> None: + pass + + async def publish(self, channel: str, message: str) -> int: + raise ConnectionError("redis down") + + +class _NotRedisClient: + def __init__(self) -> None: + self.published: List[Tuple[str, str]] = [] + + async def publish(self, channel: str, message: str) -> int: + self.published.append((channel, message)) + return 1 + + +class _QueuePubSub: + def __init__(self, initial_messages: Iterable[str] = ()) -> None: + self.queue: "asyncio.Queue[str]" = asyncio.Queue() + for message in initial_messages: + self.queue.put_nowait(message) + self.subscribed_channels: List[str] = [] + self.closed = False + + async def subscribe(self, *channels: str) -> None: + self.subscribed_channels.extend(channels) + + async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[str]: + if timeout == 0: + try: + return self.queue.get_nowait() + except asyncio.QueueEmpty: + return None + try: + return await asyncio.wait_for(self.queue.get(), timeout) + except asyncio.TimeoutError: + return None + + async def aclose(self) -> None: + self.closed = True + + +class _BrokenPubSub(_QueuePubSub): + async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[str]: + raise ConnectionError("connection lost") + + +class _ScriptedPubSubRedisClient(Redis): + def __init__(self, pubsubs: Iterable[_QueuePubSub]) -> None: + self._scripted_pubsubs = iter(pubsubs) + + def pubsub(self) -> _QueuePubSub: + return next(self._scripted_pubsubs) + + +class _FakeRedisCache: + def __init__(self, client: object, namespace: Optional[str] = None) -> None: + self._client = client + self.namespace = namespace + + def init_async_client(self) -> object: + return self._client + + +class _ExplodingRedisCache: + namespace: Optional[str] = None + + def init_async_client(self) -> object: + raise ConnectionError("cannot connect") + + +def _recording_callback( + events: List[str], name: str, fired: asyncio.Event +) -> Callable[[], Coroutine[None, None, None]]: + async def callback() -> None: + events.append(name) + fired.set() + + return callback + + +async def test_publish_noops_when_redis_cache_is_none() -> None: + await publish_config_change(redis_cache=None, object_type="litellm_proxymodeltable") + + +async def test_publish_sends_object_type_json_on_channel() -> None: + client = _RecordingRedisClient() + cache = _FakeRedisCache(client) + + await publish_config_change(redis_cache=cache, object_type="litellm_proxymodeltable") + + assert len(client.published) == 1 + channel, message = client.published[0] + assert channel == "litellm_proxy.config_change" + assert json.loads(message) == {"object_type": "litellm_proxymodeltable"} + + +async def test_publish_uses_namespaced_channel() -> None: + client = _RecordingRedisClient() + cache = _FakeRedisCache(client, namespace="prod-eu") + + await publish_config_change(redis_cache=cache, object_type="litellm_credentialstable") + + assert client.published[0][0] == "prod-eu:litellm_proxy.config_change" + + +async def test_publish_swallows_redis_publish_errors() -> None: + cache = _FakeRedisCache(_FailingPublishRedisClient()) + + await publish_config_change(redis_cache=cache, object_type="litellm_proxymodeltable") + + +async def test_publish_swallows_client_init_errors() -> None: + await publish_config_change(redis_cache=_ExplodingRedisCache(), object_type="litellm_proxymodeltable") + + +async def test_publish_skips_clients_without_pubsub_support() -> None: + client = _NotRedisClient() + cache = _FakeRedisCache(client) + + await publish_config_change(redis_cache=cache, object_type="litellm_proxymodeltable") + + assert client.published == [] + + +async def test_subscriber_runs_injected_callbacks_in_order_on_message() -> None: + pubsub = _QueuePubSub() + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) + events: List[str] = [] + fired = asyncio.Event() + subscriber = ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=( + _recording_callback(events, "add_deployment", asyncio.Event()), + _recording_callback(events, "get_credentials", fired), + ), + debounce_seconds=0.01, + jitter_max_seconds=0.0, + ) + + subscriber.start() + pubsub.queue.put_nowait(json.dumps({"object_type": "litellm_proxymodeltable"})) + await asyncio.wait_for(fired.wait(), timeout=5) + await subscriber.stop() + + assert events == ["add_deployment", "get_credentials"] + assert pubsub.subscribed_channels == [CONFIG_SYNC_CHANNEL] + assert pubsub.closed is True + + +async def test_burst_within_debounce_window_coalesces_into_one_resync() -> None: + burst = [json.dumps({"object_type": "litellm_proxymodeltable"}) for _ in range(5)] + pubsub = _QueuePubSub(initial_messages=burst) + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) + resyncs: List[str] = [] + fired = asyncio.Event() + subscriber = ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=(_recording_callback(resyncs, "resync", fired),), + debounce_seconds=0.05, + jitter_max_seconds=0.0, + ) + + subscriber.start() + await asyncio.wait_for(fired.wait(), timeout=5) + await asyncio.sleep(0.3) + await subscriber.stop() + + assert resyncs == ["resync"] + assert pubsub.queue.empty() + + +async def test_subscriber_subscribes_on_namespaced_channel_and_resyncs() -> None: + pubsub = _QueuePubSub() + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub]), namespace="prod-eu") + resyncs: List[str] = [] + fired = asyncio.Event() + subscriber = ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=(_recording_callback(resyncs, "resync", fired),), + debounce_seconds=0.01, + jitter_max_seconds=0.0, + ) + + subscriber.start() + pubsub.queue.put_nowait(json.dumps({"object_type": "litellm_proxymodeltable"})) + await asyncio.wait_for(fired.wait(), timeout=5) + await subscriber.stop() + + assert pubsub.subscribed_channels == ["prod-eu:litellm_proxy.config_change"] + assert resyncs == ["resync"] + + +class _MaxJitterRandom(random.Random): + def uniform(self, a: float, b: float) -> float: + return b + + +async def test_debounce_sleep_adds_jitter_from_injected_rng() -> None: + pubsub = _QueuePubSub(initial_messages=[json.dumps({"object_type": "litellm_proxymodeltable"})]) + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) + sleeps: List[float] = [] + fired = asyncio.Event() + + async def recording_sleep(seconds: float) -> None: + sleeps.append(seconds) + + subscriber = ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=(_recording_callback([], "resync", fired),), + debounce_seconds=1.0, + jitter_max_seconds=4.0, + rng=_MaxJitterRandom(), + sleep=recording_sleep, + ) + + subscriber.start() + await asyncio.wait_for(fired.wait(), timeout=5) + await subscriber.stop() + + assert sleeps == [5.0] + + +def test_default_jitter_window_is_nonzero() -> None: + assert CONFIG_SYNC_JITTER_MAX_SECONDS > 0 + + +async def test_redis_error_leads_to_backoff_and_resubscribe() -> None: + broken = _BrokenPubSub() + healthy = _QueuePubSub(initial_messages=[json.dumps({"object_type": "litellm_credentialstable"})]) + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([broken, healthy])) + resyncs: List[str] = [] + fired = asyncio.Event() + subscriber = ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=(_recording_callback(resyncs, "resync", fired),), + debounce_seconds=0.01, + jitter_max_seconds=0.0, + backoff_initial_seconds=0.02, + backoff_max_seconds=0.05, + ) + + subscriber.start() + await asyncio.wait_for(fired.wait(), timeout=5) + task = subscriber._task + assert task is not None + assert task.done() is False + await subscriber.stop() + + assert broken.subscribed_channels == [CONFIG_SYNC_CHANNEL] + assert broken.closed is True + assert healthy.subscribed_channels == [CONFIG_SYNC_CHANNEL] + assert resyncs == ["resync"] + + +async def test_failing_resync_callback_does_not_kill_subscriber() -> None: + pubsub = _QueuePubSub() + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) + resyncs: List[str] = [] + fired = asyncio.Event() + + async def failing_callback() -> None: + raise RuntimeError("resync exploded") + + subscriber = ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=(failing_callback, _recording_callback(resyncs, "resync", fired)), + debounce_seconds=0.01, + jitter_max_seconds=0.0, + ) + + subscriber.start() + pubsub.queue.put_nowait("change") + await asyncio.wait_for(fired.wait(), timeout=5) + fired.clear() + pubsub.queue.put_nowait("change") + await asyncio.wait_for(fired.wait(), timeout=5) + await subscriber.stop() + + assert resyncs == ["resync", "resync"] + + +async def test_stop_cancels_subscriber_cleanly() -> None: + pubsub = _QueuePubSub() + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) + subscriber = ConfigSyncSubscriber(redis_cache=cache, resync_callbacks=(), debounce_seconds=0.01) + + subscriber.start() + await asyncio.sleep(0.05) + task = subscriber._task + assert task is not None + await subscriber.stop() + + assert task.done() is True + assert subscriber._task is None + assert pubsub.closed is True + await subscriber.stop() + + +async def test_stop_before_start_is_a_noop() -> None: + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([])) + subscriber = ConfigSyncSubscriber(redis_cache=cache, resync_callbacks=()) + + await subscriber.stop() + + +async def test_subscriber_exits_without_callbacks_when_client_lacks_pubsub() -> None: + cache = _FakeRedisCache(_NotRedisClient()) + resyncs: List[str] = [] + subscriber = ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=(_recording_callback(resyncs, "resync", asyncio.Event()),), + ) + + subscriber.start() + task = subscriber._task + assert task is not None + await asyncio.wait_for(task, timeout=5) + + assert resyncs == [] + + +class _FakeTableActions: + def __init__(self, calls: List[Tuple[str, str]]) -> None: + self._calls = calls + + async def create(self, **kwargs: object) -> object: + self._calls.append(("write", "create")) + return {"id": "m-1"} + + async def find_many(self, **kwargs: object) -> object: + self._calls.append(("read", "find_many")) + return [] + + +class _AllWritesTableActions: + def __init__(self, calls: List[str]) -> None: + self._calls = calls + + def __getattr__(self, name: str) -> Callable[..., Coroutine[None, None, str]]: + async def action(*args: object, **kwargs: object) -> str: + self._calls.append(name) + return name + + return action + + +def _recording_publish(calls: List[Tuple[str, str]]) -> Callable[[str], Coroutine[None, None, None]]: + async def publish(object_type: str) -> None: + calls.append(("publish", object_type)) + + return publish + + +def test_wrapper_passes_through_unsynced_tables() -> None: + actions = object() + + wrapped = wrap_table_actions_for_config_sync(actions=actions, table_name="litellm_spendlogs") + + assert wrapped is actions + + +async def test_wrapper_publishes_table_name_after_write() -> None: + calls: List[Tuple[str, str]] = [] + wrapped = wrap_table_actions_for_config_sync( + actions=_FakeTableActions(calls), + table_name="litellm_proxymodeltable", + publish=_recording_publish(calls), + ) + + result = await wrapped.create(data={"model_name": "gpt-5.2"}) + + assert result == {"id": "m-1"} + assert calls == [("write", "create"), ("publish", "litellm_proxymodeltable")] + + +async def test_wrapper_does_not_publish_on_reads() -> None: + calls: List[Tuple[str, str]] = [] + wrapped = wrap_table_actions_for_config_sync( + actions=_FakeTableActions(calls), + table_name="litellm_proxymodeltable", + publish=_recording_publish(calls), + ) + + result = await wrapped.find_many(where={}) + + assert result == [] + assert calls == [("read", "find_many")] + + +def test_write_action_names_are_pinned() -> None: + assert _WRITE_ACTION_NAMES == frozenset(_EXPECTED_WRITE_ACTION_NAMES) + + +def test_config_synced_table_membership_is_pinned() -> None: + assert _CONFIG_SYNCED_TABLE_NAMES == _EXPECTED_CONFIG_SYNCED_TABLE_NAMES + + +def test_tool_telemetry_table_writes_pass_through_unwrapped() -> None: + actions = object() + + wrapped = wrap_table_actions_for_config_sync(actions=actions, table_name="litellm_tooltable") + + assert wrapped is actions + + +@pytest.mark.parametrize("action_name", _EXPECTED_WRITE_ACTION_NAMES) +async def test_wrapper_publishes_for_every_write_action(action_name: str) -> None: + write_calls: List[str] = [] + publish_calls: List[Tuple[str, str]] = [] + wrapped = wrap_table_actions_for_config_sync( + actions=_AllWritesTableActions(write_calls), + table_name="litellm_guardrailstable", + publish=_recording_publish(publish_calls), + ) + + result = await getattr(wrapped, action_name)(data={}) + + assert result == action_name + assert write_calls == [action_name] + assert publish_calls == [("publish", "litellm_guardrailstable")] + + +async def test_model_repository_write_publishes_via_live_coordination_cache() -> None: + from litellm.proxy import proxy_server + from litellm.proxy.proxy_server import _set_redis_usage_cache + from litellm.repositories.model_repository import ModelRepository + + client = _RecordingRedisClient() + prisma_client = MagicMock() + prisma_client.db.litellm_proxymodeltable.update = AsyncMock(return_value={"model_id": "m-1"}) + repository = ModelRepository(prisma_client) + table = repository.table + assert isinstance(table, _PublishOnWriteActions) + + previous_cache = proxy_server.redis_usage_cache + _set_redis_usage_cache(_FakeRedisCache(client)) + try: + await table.update(where={"model_id": "m-1"}, data={"model_name": "gpt-5.2"}) + finally: + _set_redis_usage_cache(previous_cache) + + prisma_client.db.litellm_proxymodeltable.update.assert_awaited_once_with( + where={"model_id": "m-1"}, data={"model_name": "gpt-5.2"} + ) + assert len(client.published) == 1 + channel, message = client.published[0] + assert channel == CONFIG_SYNC_CHANNEL + assert json.loads(message) == {"object_type": "litellm_proxymodeltable"} + + +async def test_invalidate_config_param_publishes_param_name() -> None: + from litellm.proxy import proxy_server + from litellm.proxy.proxy_server import _set_redis_usage_cache + from litellm.proxy.utils import invalidate_config_param + + client = _RecordingRedisClient() + previous_cache = proxy_server.redis_usage_cache + _set_redis_usage_cache(_FakeRedisCache(client)) + try: + await invalidate_config_param("environment_variables") + finally: + _set_redis_usage_cache(previous_cache) + + assert len(client.published) == 1 + channel, message = client.published[0] + assert channel == CONFIG_SYNC_CHANNEL + assert json.loads(message) == {"object_type": "environment_variables"} + + +async def test_evict_config_param_does_not_publish() -> None: + from litellm.proxy import proxy_server + from litellm.proxy.proxy_server import _set_redis_usage_cache + from litellm.proxy.utils import evict_config_param + + client = _RecordingRedisClient() + previous_cache = proxy_server.redis_usage_cache + _set_redis_usage_cache(_FakeRedisCache(client)) + try: + await evict_config_param("model_cost_map_reload_config") + finally: + _set_redis_usage_cache(previous_cache) + + assert client.published == [] + + +def _reload_config_prisma_client() -> MagicMock: + config_record = MagicMock() + config_record.param_value = {"interval_hours": 6, "force_reload": True} + prisma_client = MagicMock() + prisma_client.get_generic_data = AsyncMock(return_value=config_record) + prisma_client.db.litellm_config.upsert = AsyncMock(return_value=None) + return prisma_client + + +async def test_model_cost_map_reload_does_not_publish_config_change() -> None: + from litellm.proxy import proxy_server + from litellm.proxy.proxy_server import ProxyConfig, _set_redis_usage_cache + from litellm.proxy.utils import litellm_config_cache + from litellm.utils import _invalidate_model_cost_lowercase_map + + litellm_config_cache.flush_cache() + prisma_client = _reload_config_prisma_client() + client = _RecordingRedisClient() + previous_cache = proxy_server.redis_usage_cache + original_model_cost = litellm.model_cost.copy() + _set_redis_usage_cache(_FakeRedisCache(client)) + try: + with patch("litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map") as mock_get_map: + mock_get_map.return_value = {"gpt-5.2": {"input_cost_per_token": 0.001}} + await ProxyConfig()._check_and_reload_model_cost_map(prisma_client=prisma_client) + finally: + litellm.model_cost = original_model_cost + _invalidate_model_cost_lowercase_map() + _set_redis_usage_cache(previous_cache) + + prisma_client.db.litellm_config.upsert.assert_awaited_once() + assert client.published == [] + + +async def test_anthropic_beta_headers_reload_does_not_publish_config_change() -> None: + from litellm.proxy import proxy_server + from litellm.proxy.proxy_server import ProxyConfig, _set_redis_usage_cache + from litellm.proxy.utils import litellm_config_cache + + litellm_config_cache.flush_cache() + prisma_client = _reload_config_prisma_client() + client = _RecordingRedisClient() + previous_cache = proxy_server.redis_usage_cache + _set_redis_usage_cache(_FakeRedisCache(client)) + try: + with patch("litellm.anthropic_beta_headers_manager.reload_beta_headers_config") as mock_reload: + mock_reload.return_value = {} + await ProxyConfig()._check_and_reload_anthropic_beta_headers(prisma_client=prisma_client) + finally: + _set_redis_usage_cache(previous_cache) + + prisma_client.db.litellm_config.upsert.assert_awaited_once() + assert client.published == [] diff --git a/tests/test_litellm/repositories/test_repositories.py b/tests/test_litellm/repositories/test_repositories.py index c923b722991..994e73d33f5 100644 --- a/tests/test_litellm/repositories/test_repositories.py +++ b/tests/test_litellm/repositories/test_repositories.py @@ -307,6 +307,15 @@ class TestModelRepository: client = MockPrismaClient() return ModelRepository(client) + def test_table_is_wrapped_for_config_sync(self, repo): + from litellm.proxy.common_utils.config_sync_pubsub import ( + _PublishOnWriteActions, + ) + + table = repo.table + assert isinstance(table, _PublishOnWriteActions) + assert table._actions is repo.prisma_client.db.litellm_proxymodeltable + @pytest.mark.asyncio @patch( "litellm.repositories.model_repository.encrypt_value_helper", @@ -1313,6 +1322,15 @@ class TestCredentialsRepository: client = MockPrismaClient() return CredentialsRepository(client) + def test_table_is_wrapped_for_config_sync(self, repo): + from litellm.proxy.common_utils.config_sync_pubsub import ( + _PublishOnWriteActions, + ) + + table = repo.table + assert isinstance(table, _PublishOnWriteActions) + assert table._actions is repo.prisma_client.db.litellm_credentialstable + @pytest.mark.asyncio async def test_create(self, repo): record = await repo.create( @@ -2188,6 +2206,9 @@ class TestConfigRepositoryDeepCopy: class TestPrismaTableRepository: def test_table_property_returns_named_delegate(self): + from litellm.proxy.common_utils.config_sync_pubsub import ( + _PublishOnWriteActions, + ) from litellm.repositories.table_repositories import ( AgentsRepository, PolicyRepository, @@ -2197,9 +2218,11 @@ class TestPrismaTableRepository: agents = AgentsRepository(prisma_client) policy = PolicyRepository(prisma_client) - assert agents.table is prisma_client.db.litellm_agentstable - assert policy.table is prisma_client.db.litellm_policytable - assert agents.table is not policy.table + assert isinstance(agents.table, _PublishOnWriteActions) + assert isinstance(policy.table, _PublishOnWriteActions) + assert agents.table._actions is prisma_client.db.litellm_agentstable + assert policy.table._actions is prisma_client.db.litellm_policytable + assert agents.table._actions is not policy.table._actions def test_table_access_raises_without_db(self): from litellm.repositories.table_repositories import SpendLogsRepository @@ -2208,8 +2231,28 @@ class TestPrismaTableRepository: with pytest.raises(RuntimeError, match="No DB Connected"): _ = repo.table + CONFIG_SYNCED_TABLE_NAMES = frozenset( + { + "litellm_agentstable", + "litellm_cacheconfig", + "litellm_configoverrides", + "litellm_guardrailstable", + "litellm_managedvectorstoreindextable", + "litellm_managedvectorstorestable", + "litellm_mcpservertable", + "litellm_policyattachmenttable", + "litellm_policytable", + "litellm_prompttable", + "litellm_searchtoolstable", + "litellm_ssoconfig", + } + ) + def test_each_repository_binds_its_own_table_name(self): import litellm.repositories.table_repositories as tr + from litellm.proxy.common_utils.config_sync_pubsub import ( + _PublishOnWriteActions, + ) prisma_client = MagicMock() repos = [ @@ -2226,7 +2269,14 @@ class TestPrismaTableRepository: assert name.startswith("litellm_") assert name not in seen, f"duplicate table_name {name}" seen.add(name) - assert repo_cls(prisma_client).table is getattr(prisma_client.db, name) + table = repo_cls(prisma_client).table + raw_actions = getattr(prisma_client.db, name) + if name in self.CONFIG_SYNCED_TABLE_NAMES: + assert isinstance(table, _PublishOnWriteActions), name + assert table._actions is raw_actions + else: + assert table is raw_actions, name + assert self.CONFIG_SYNCED_TABLE_NAMES <= seen def _json_path_equals( From a8018f7500fe5ed801803eada2fbe1c219f30af9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 31 Jul 2026 21:08:32 -0700 Subject: [PATCH 019/123] fix(proxy): throttle pub/sub resyncs and stop publishing startup-only config params Caps fleet-wide reload rate at one resync per 10s per pod so a burst of authenticated writes cannot amplify into continuous cross-pod reloads, and skips publishing config params (environment_variables, router_settings) that no resync callback applies outside proxy startup --- .../proxy/common_utils/config_sync_pubsub.py | 43 +++ litellm/proxy/proxy_server.py | 56 ++-- litellm/proxy/utils.py | 7 +- .../common_utils/test_config_sync_pubsub.py | 302 +++++++++++++++++- 4 files changed, 380 insertions(+), 28 deletions(-) diff --git a/litellm/proxy/common_utils/config_sync_pubsub.py b/litellm/proxy/common_utils/config_sync_pubsub.py index 4292c67c8c2..e4521c397b7 100644 --- a/litellm/proxy/common_utils/config_sync_pubsub.py +++ b/litellm/proxy/common_utils/config_sync_pubsub.py @@ -1,6 +1,7 @@ import asyncio import json import random +import time from collections.abc import Awaitable, Callable from dataclasses import asdict, dataclass from typing import TYPE_CHECKING, Protocol, cast # noqa: TID251 # untyped prisma/redis boundary needs cast @@ -28,6 +29,7 @@ class _ConfigSyncPubSubClient(Protocol): CONFIG_SYNC_CHANNEL = "litellm_proxy.config_change" CONFIG_SYNC_DEBOUNCE_SECONDS = 1.0 CONFIG_SYNC_JITTER_MAX_SECONDS = 5.0 +CONFIG_SYNC_MIN_RESYNC_INTERVAL_SECONDS = 10.0 _POLL_TIMEOUT_SECONDS = 1.0 _BACKOFF_INITIAL_SECONDS = 5.0 _BACKOFF_MAX_SECONDS = 60.0 @@ -55,6 +57,15 @@ _CONFIG_SYNCED_TABLE_NAMES: frozenset[str] = frozenset( } ) +_RESYNC_APPLIED_CONFIG_PARAM_NAMES: frozenset[str] = frozenset( + { + "general_settings", + "litellm_settings", + "model_cost_map_reload_config", + "anthropic_beta_headers_reload_config", + } +) + def coordination_redis_cache() -> "RedisCache | None": from litellm.proxy.proxy_server import redis_usage_cache @@ -113,6 +124,16 @@ async def publish_config_change_for_object_type(object_type: str) -> None: await publish_config_change(redis_cache=coordination_redis_cache(), object_type=object_type) +async def publish_config_param_change(param_name: str) -> None: + if param_name not in _RESYNC_APPLIED_CONFIG_PARAM_NAMES: + verbose_proxy_logger.debug( + "config sync publish for %s skipped: no resync callback applies this param outside proxy startup", + param_name, + ) + return + await publish_config_change_for_object_type(param_name) + + class _PublishOnWriteActions: __slots__ = ("_actions", "_object_type", "_publish") @@ -156,6 +177,9 @@ class ConfigSyncSubscriber: "_backoff_max_seconds", "_debounce_seconds", "_jitter_max_seconds", + "_last_resync_at", + "_min_resync_interval_seconds", + "_monotonic", "_redis_cache", "_resync_callbacks", "_rng", @@ -169,20 +193,25 @@ class ConfigSyncSubscriber: resync_callbacks: tuple[Callable[[], Awaitable[None]], ...], debounce_seconds: float = CONFIG_SYNC_DEBOUNCE_SECONDS, jitter_max_seconds: float = CONFIG_SYNC_JITTER_MAX_SECONDS, + min_resync_interval_seconds: float = CONFIG_SYNC_MIN_RESYNC_INTERVAL_SECONDS, backoff_initial_seconds: float = _BACKOFF_INITIAL_SECONDS, backoff_max_seconds: float = _BACKOFF_MAX_SECONDS, rng: random.Random | None = None, sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + monotonic: Callable[[], float] = time.monotonic, ) -> None: self._redis_cache = redis_cache self._resync_callbacks = resync_callbacks self._debounce_seconds = debounce_seconds self._jitter_max_seconds = jitter_max_seconds + self._min_resync_interval_seconds = min_resync_interval_seconds self._backoff_initial_seconds = backoff_initial_seconds self._backoff_max_seconds = backoff_max_seconds self._rng = rng if rng is not None else random.Random() self._sleep = sleep + self._monotonic = monotonic self._task: asyncio.Task[None] | None = None + self._last_resync_at: float | None = None def start(self) -> None: if self._task is not None: @@ -235,8 +264,22 @@ class ConfigSyncSubscriber: if message is None: continue await self._sleep(self._debounce_seconds + self._rng.uniform(0.0, self._jitter_max_seconds)) + await self._wait_for_min_resync_interval() await self._drain_pending(pubsub) await self._run_resync_callbacks() + self._last_resync_at = self._monotonic() + + async def _wait_for_min_resync_interval(self) -> None: + if self._last_resync_at is None: + return + seconds_until_next_resync = self._min_resync_interval_seconds - (self._monotonic() - self._last_resync_at) + if seconds_until_next_resync <= 0: + return + verbose_proxy_logger.debug( + "config sync resync throttled for %.1fs to cap fleet-wide reload rate", + seconds_until_next_resync, + ) + await self._sleep(seconds_until_next_resync) @staticmethod async def _drain_pending(pubsub: _ConfigSyncPubSub) -> None: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6bb0032b38c..7d3e58bfe52 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1153,11 +1153,7 @@ async def proxy_startup_event(app: FastAPI): except Exception as e: verbose_proxy_logger.error(f"Error stopping DB health watchdog task: {e}") - if proxy_config.config_sync_subscriber is not None: - try: - await proxy_config.config_sync_subscriber.stop() - except Exception as e: - verbose_proxy_logger.error(f"Error stopping config sync subscriber: {e}") + await proxy_config.stop_config_sync_subscriber() await proxy_shutdown_event() # type: ignore[reportGeneralTypeIssues] @@ -6213,6 +6209,38 @@ class ProxyConfig: "litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - {}".format(str(e)) ) + def start_config_sync_subscriber( + self, + prisma_client: PrismaClient, + proxy_logging_obj: ProxyLogging, + redis_cache: Optional[RedisCache], + ) -> None: + if redis_cache is None or self.config_sync_subscriber is not None: + return + + async def _resync_config_from_db() -> None: + await self.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) + + async def _resync_credentials_from_db() -> None: + await self.get_credentials(prisma_client=prisma_client) + + subscriber = ConfigSyncSubscriber( + redis_cache=redis_cache, + resync_callbacks=(_resync_config_from_db, _resync_credentials_from_db), + ) + self.config_sync_subscriber = subscriber + subscriber.start() + + async def stop_config_sync_subscriber(self) -> None: + subscriber = self.config_sync_subscriber + if subscriber is None: + return + self.config_sync_subscriber = None + try: + await subscriber.stop() + except Exception as e: + verbose_proxy_logger.error(f"Error stopping config sync subscriber: {e}") + async def _init_non_llm_objects_in_db(self, prisma_client: PrismaClient): """ Use this to read non-llm objects from the db and initialize them @@ -8174,19 +8202,11 @@ class ProxyStartupEvent: ) await proxy_config.get_credentials(prisma_client=prisma_client) - if redis_usage_cache is not None and proxy_config.config_sync_subscriber is None: - - async def _resync_config_from_db() -> None: - await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) - - async def _resync_credentials_from_db() -> None: - await proxy_config.get_credentials(prisma_client=prisma_client) - - proxy_config.config_sync_subscriber = ConfigSyncSubscriber( - redis_cache=redis_usage_cache, - resync_callbacks=(_resync_config_from_db, _resync_credentials_from_db), - ) - proxy_config.config_sync_subscriber.start() + proxy_config.start_config_sync_subscriber( + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + redis_cache=redis_usage_cache, + ) if store_model_in_db is not True: await proxy_config.init_mcp_servers_from_db() diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index fbcca73e779..5d2d29efa12 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -119,10 +119,7 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.route_checks import RouteChecks -from litellm.proxy.common_utils.config_sync_pubsub import ( - coordination_redis_cache, - publish_config_change, -) +from litellm.proxy.common_utils.config_sync_pubsub import publish_config_param_change from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.db.create_views import ( create_missing_views, @@ -2982,7 +2979,7 @@ async def evict_config_param(param_name: str) -> None: async def invalidate_config_param(param_name: str) -> None: """Evict from both cache layers; call after every LiteLLM_Config write.""" await evict_config_param(param_name) - await publish_config_change(redis_cache=coordination_redis_cache(), object_type=param_name) + await publish_config_param_change(param_name) async def prefetch_config_params(prisma_client: Any, param_names: List[str]) -> None: diff --git a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py index 8a8ced8bc41..f50eef4f1cc 100644 --- a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py +++ b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py @@ -11,9 +11,11 @@ import litellm from litellm.proxy.common_utils.config_sync_pubsub import ( CONFIG_SYNC_CHANNEL, CONFIG_SYNC_JITTER_MAX_SECONDS, + CONFIG_SYNC_MIN_RESYNC_INTERVAL_SECONDS, ConfigSyncSubscriber, _CONFIG_SYNCED_TABLE_NAMES, _PublishOnWriteActions, + _RESYNC_APPLIED_CONFIG_PARAM_NAMES, _WRITE_ACTION_NAMES, publish_config_change, wrap_table_actions_for_config_sync, @@ -48,6 +50,17 @@ _EXPECTED_CONFIG_SYNCED_TABLE_NAMES = frozenset( } ) +_EXPECTED_RESYNC_APPLIED_CONFIG_PARAM_NAMES = frozenset( + { + "anthropic_beta_headers_reload_config", + "general_settings", + "litellm_settings", + "model_cost_map_reload_config", + } +) + +_STARTUP_ONLY_CONFIG_PARAM_NAMES = ("environment_variables", "router_settings") + class _RecordingRedisClient(Redis): def __init__(self) -> None: @@ -106,6 +119,31 @@ class _BrokenPubSub(_QueuePubSub): raise ConnectionError("connection lost") +class _CloseFailingBrokenPubSub(_BrokenPubSub): + async def aclose(self) -> None: + raise ConnectionError("close failed") + + +class _EmptyPollsThenMessagePubSub(_QueuePubSub): + def __init__(self, empty_polls: int, initial_messages: Iterable[str] = ()) -> None: + super().__init__(initial_messages=initial_messages) + self.remaining_empty_polls = empty_polls + + async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[str]: + if timeout != 0 and self.remaining_empty_polls > 0: + self.remaining_empty_polls -= 1 + return None + return await super().get_message(ignore_subscribe_messages=ignore_subscribe_messages, timeout=timeout) + + +class _FakeClock: + def __init__(self, now: float = 1000.0) -> None: + self.now = now + + def __call__(self) -> float: + return self.now + + class _ScriptedPubSubRedisClient(Redis): def __init__(self, pubsubs: Iterable[_QueuePubSub]) -> None: self._scripted_pubsubs = iter(pubsubs) @@ -286,6 +324,158 @@ def test_default_jitter_window_is_nonzero() -> None: assert CONFIG_SYNC_JITTER_MAX_SECONDS > 0 +def test_default_min_resync_interval_caps_reload_rate() -> None: + assert CONFIG_SYNC_MIN_RESYNC_INTERVAL_SECONDS > CONFIG_SYNC_JITTER_MAX_SECONDS + + +def _throttled_subscriber( + cache: object, + events: List[str], + fired: asyncio.Event, + clock: _FakeClock, + min_resync_interval_seconds: float = 10.0, +) -> ConfigSyncSubscriber: + async def recording_sleep(seconds: float) -> None: + events.append(f"sleep:{seconds}") + await asyncio.sleep(0) + + async def resync() -> None: + events.append("resync") + fired.set() + + return ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=(resync,), + debounce_seconds=0.0, + jitter_max_seconds=0.0, + min_resync_interval_seconds=min_resync_interval_seconds, + sleep=recording_sleep, + monotonic=clock, + ) + + +async def test_resync_arriving_inside_min_interval_waits_out_the_remainder() -> None: + pubsub = _QueuePubSub() + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) + events: List[str] = [] + fired = asyncio.Event() + clock = _FakeClock() + subscriber = _throttled_subscriber(cache=cache, events=events, fired=fired, clock=clock) + + subscriber.start() + pubsub.queue.put_nowait("change") + await asyncio.wait_for(fired.wait(), timeout=5) + fired.clear() + clock.now += 4.0 + pubsub.queue.put_nowait("change") + await asyncio.wait_for(fired.wait(), timeout=5) + await subscriber.stop() + + assert events == ["sleep:0.0", "resync", "sleep:0.0", "sleep:6.0", "resync"] + + +async def test_resync_after_min_interval_elapsed_is_not_throttled() -> None: + pubsub = _QueuePubSub() + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) + events: List[str] = [] + fired = asyncio.Event() + clock = _FakeClock() + subscriber = _throttled_subscriber(cache=cache, events=events, fired=fired, clock=clock) + + subscriber.start() + pubsub.queue.put_nowait("change") + await asyncio.wait_for(fired.wait(), timeout=5) + fired.clear() + clock.now += 30.0 + pubsub.queue.put_nowait("change") + await asyncio.wait_for(fired.wait(), timeout=5) + await subscriber.stop() + + assert events == ["sleep:0.0", "resync", "sleep:0.0", "resync"] + + +async def test_writes_during_the_throttle_wait_collapse_into_the_next_resync() -> None: + pubsub = _QueuePubSub() + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) + events: List[str] = [] + fired = asyncio.Event() + clock = _FakeClock() + subscriber = _throttled_subscriber(cache=cache, events=events, fired=fired, clock=clock) + + subscriber.start() + pubsub.queue.put_nowait("change") + await asyncio.wait_for(fired.wait(), timeout=5) + fired.clear() + for _ in range(5): + pubsub.queue.put_nowait("change") + await asyncio.wait_for(fired.wait(), timeout=5) + await asyncio.sleep(0.1) + await subscriber.stop() + + assert events.count("resync") == 2 + assert pubsub.queue.empty() + + +async def test_polls_without_messages_do_not_trigger_resyncs() -> None: + pubsub = _EmptyPollsThenMessagePubSub(empty_polls=3, initial_messages=["change"]) + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) + resyncs: List[str] = [] + fired = asyncio.Event() + subscriber = ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=(_recording_callback(resyncs, "resync", fired),), + debounce_seconds=0.01, + jitter_max_seconds=0.0, + ) + + subscriber.start() + await asyncio.wait_for(fired.wait(), timeout=5) + await asyncio.sleep(0.1) + await subscriber.stop() + + assert pubsub.remaining_empty_polls == 0 + assert resyncs == ["resync"] + + +async def test_failing_pubsub_close_still_reconnects() -> None: + broken = _CloseFailingBrokenPubSub() + healthy = _QueuePubSub(initial_messages=["change"]) + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([broken, healthy])) + resyncs: List[str] = [] + fired = asyncio.Event() + subscriber = ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=(_recording_callback(resyncs, "resync", fired),), + debounce_seconds=0.01, + jitter_max_seconds=0.0, + backoff_initial_seconds=0.02, + backoff_max_seconds=0.05, + ) + + subscriber.start() + await asyncio.wait_for(fired.wait(), timeout=5) + await subscriber.stop() + + assert healthy.subscribed_channels == [CONFIG_SYNC_CHANNEL] + assert resyncs == ["resync"] + + +async def test_second_start_does_not_open_a_second_subscription() -> None: + pubsub = _QueuePubSub() + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) + subscriber = ConfigSyncSubscriber(redis_cache=cache, resync_callbacks=(), debounce_seconds=0.01) + + subscriber.start() + task = subscriber._task + subscriber.start() + assert task is not None + assert subscriber._task is task + await asyncio.sleep(0.05) + await subscriber.stop() + + assert pubsub.subscribed_channels == [CONFIG_SYNC_CHANNEL] + + async def test_redis_error_leads_to_backoff_and_resubscribe() -> None: broken = _BrokenPubSub() healthy = _QueuePubSub(initial_messages=[json.dumps({"object_type": "litellm_credentialstable"})]) @@ -328,6 +518,7 @@ async def test_failing_resync_callback_does_not_kill_subscriber() -> None: resync_callbacks=(failing_callback, _recording_callback(resyncs, "resync", fired)), debounce_seconds=0.01, jitter_max_seconds=0.0, + min_resync_interval_seconds=0.0, ) subscriber.start() @@ -510,7 +701,7 @@ async def test_model_repository_write_publishes_via_live_coordination_cache() -> assert json.loads(message) == {"object_type": "litellm_proxymodeltable"} -async def test_invalidate_config_param_publishes_param_name() -> None: +async def _publish_calls_for_invalidated_param(param_name: str) -> List[Tuple[str, str]]: from litellm.proxy import proxy_server from litellm.proxy.proxy_server import _set_redis_usage_cache from litellm.proxy.utils import invalidate_config_param @@ -519,14 +710,30 @@ async def test_invalidate_config_param_publishes_param_name() -> None: previous_cache = proxy_server.redis_usage_cache _set_redis_usage_cache(_FakeRedisCache(client)) try: - await invalidate_config_param("environment_variables") + await invalidate_config_param(param_name) finally: _set_redis_usage_cache(previous_cache) + return client.published - assert len(client.published) == 1 - channel, message = client.published[0] + +async def test_invalidate_config_param_publishes_params_a_resync_applies() -> None: + published = await _publish_calls_for_invalidated_param("general_settings") + + assert len(published) == 1 + channel, message = published[0] assert channel == CONFIG_SYNC_CHANNEL - assert json.loads(message) == {"object_type": "environment_variables"} + assert json.loads(message) == {"object_type": "general_settings"} + + +@pytest.mark.parametrize("param_name", _STARTUP_ONLY_CONFIG_PARAM_NAMES) +async def test_invalidate_config_param_does_not_publish_startup_only_params(param_name: str) -> None: + published = await _publish_calls_for_invalidated_param(param_name) + + assert published == [] + + +def test_resync_applied_config_param_membership_is_pinned() -> None: + assert _RESYNC_APPLIED_CONFIG_PARAM_NAMES == _EXPECTED_RESYNC_APPLIED_CONFIG_PARAM_NAMES async def test_evict_config_param_does_not_publish() -> None: @@ -598,3 +805,88 @@ async def test_anthropic_beta_headers_reload_does_not_publish_config_change() -> prisma_client.db.litellm_config.upsert.assert_awaited_once() assert client.published == [] + + +class _StopFailingSubscriber(ConfigSyncSubscriber): + async def stop(self) -> None: + raise RuntimeError("stop failed") + + +async def test_proxy_config_subscriber_resyncs_deployments_and_credentials() -> None: + from litellm.proxy.proxy_server import ProxyConfig + + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([_QueuePubSub()])) + config = ProxyConfig() + prisma_client = MagicMock() + proxy_logging_obj = MagicMock() + calls: List[Tuple[str, object, object]] = [] + + async def fake_add_deployment(prisma_client: object, proxy_logging_obj: object) -> None: + calls.append(("add_deployment", prisma_client, proxy_logging_obj)) + + async def fake_get_credentials(prisma_client: object) -> None: + calls.append(("get_credentials", prisma_client, None)) + + config.add_deployment = fake_add_deployment + config.get_credentials = fake_get_credentials + config.start_config_sync_subscriber( + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + redis_cache=cache, + ) + subscriber = config.config_sync_subscriber + assert subscriber is not None + for callback in subscriber._resync_callbacks: + await callback() + await config.stop_config_sync_subscriber() + + assert calls == [ + ("add_deployment", prisma_client, proxy_logging_obj), + ("get_credentials", prisma_client, None), + ] + assert config.config_sync_subscriber is None + assert subscriber._task is None + + +async def test_proxy_config_does_not_start_subscriber_without_coordination_redis() -> None: + from litellm.proxy.proxy_server import ProxyConfig + + config = ProxyConfig() + + config.start_config_sync_subscriber( + prisma_client=MagicMock(), + proxy_logging_obj=MagicMock(), + redis_cache=None, + ) + + assert config.config_sync_subscriber is None + + +async def test_proxy_config_keeps_the_first_subscriber_on_repeat_start() -> None: + from litellm.proxy.proxy_server import ProxyConfig + + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([_QueuePubSub()])) + config = ProxyConfig() + + config.start_config_sync_subscriber(prisma_client=MagicMock(), proxy_logging_obj=MagicMock(), redis_cache=cache) + first = config.config_sync_subscriber + config.start_config_sync_subscriber(prisma_client=MagicMock(), proxy_logging_obj=MagicMock(), redis_cache=cache) + second = config.config_sync_subscriber + await config.stop_config_sync_subscriber() + + assert first is not None + assert second is first + + +async def test_proxy_config_shutdown_survives_a_failing_subscriber_stop() -> None: + from litellm.proxy.proxy_server import ProxyConfig + + config = ProxyConfig() + config.config_sync_subscriber = _StopFailingSubscriber( + redis_cache=_FakeRedisCache(_ScriptedPubSubRedisClient([])), + resync_callbacks=(), + ) + + await config.stop_config_sync_subscriber() + + assert config.config_sync_subscriber is None From 2a13bbe1cb502ba472ad41fecac95363b077c68f Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 31 Jul 2026 22:13:57 -0700 Subject: [PATCH 020/123] refactor(proxy): resolve team member lookups in one query and cap the rejection message Resolve the requested member user_ids with a single find_many instead of one lookup per member, so a large member list no longer turns into that many round-trips before the permission check runs. Write the member-add audit entries concurrently rather than one after another, and list at most a few ids in the rejection message instead of echoing the whole request back. Update the team-admin member-add case that covered adding a user_id with no user row, which the endpoint now leaves to proxy admins. --- .../management_endpoints/team_endpoints.py | 49 +++++++++++----- tests/proxy_unit_tests/test_proxy_server.py | 2 +- .../test_team_endpoints.py | 57 ++++++++++++++++--- 3 files changed, 87 insertions(+), 21 deletions(-) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 7b5ce33cb8b..a8c5fcea4df 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -2431,12 +2431,23 @@ async def _resolve_existing_member_user_ids( members: Sequence[Member], prisma_client: PrismaClient, ) -> frozenset[str]: - """Return the caller-supplied user_ids that already have a user row.""" - user_repository = UserRepository(prisma_client) - found = await asyncio.gather( - *(user_repository.find_by_id(member.user_id) for member in members if member.user_id is not None) + """Return the caller-supplied user_ids that already have a user row. + + Resolved with a single query so the number of members in the request does + not translate into that many concurrent connections. + """ + requested_user_ids = frozenset(member.user_id for member in members if member.user_id is not None) + if not requested_user_ids: + return frozenset() + + found = await UserRepository(prisma_client).table.find_many( + where={ # mutable-ok: Prisma query filters are dict-shaped + "user_id": { # mutable-ok: Prisma query filters are dict-shaped + "in": sorted(requested_user_ids) + } + } ) - return frozenset(user.user_id for user in found if user is not None and user.user_id is not None) + return frozenset(user.user_id for user in found or () if user.user_id is not None) def _pre_existing_user_ids( @@ -2459,6 +2470,9 @@ def _pre_existing_user_ids( return existing_user_ids | populated_user_ids +_MAX_REPORTED_UNKNOWN_USER_IDS = 10 + + def _validate_member_user_id_provisioning( members: Sequence[Member], existing_user_ids: frozenset[str], @@ -2481,13 +2495,15 @@ def _validate_member_user_id_provisioning( if not unknown_user_ids: return + listed = ", ".join(unknown_user_ids[:_MAX_REPORTED_UNKNOWN_USER_IDS]) + remaining = len(unknown_user_ids) - _MAX_REPORTED_UNKNOWN_USER_IDS raise HTTPException( status_code=403, detail={ # mutable-ok: HTTPException detail must be a plain mapping to keep this route's {"error": ...} response shape "error": ( - "Only proxy admins can add a user_id that does not exist yet: {}. " + "Only proxy admins can add a user_id that does not exist yet: {}{}. " "Add the member by user_email to invite a new user, or ask a proxy admin " - "to create the user first.".format(", ".join(unknown_user_ids)) + "to create the user first.".format(listed, " and {} more".format(remaining) if remaining > 0 else "") ) }, ) @@ -2515,13 +2531,15 @@ async def _create_team_member_add_audit_logs( user_api_key_dict: UserAPIKeyAuth, litellm_proxy_admin_name: str, ) -> None: - """Record the membership change, and any user row it created, in the audit log.""" + """Record the membership change, and any user row it created, in the audit log. + + The entries are written concurrently so a request adding many members does + not pay for them one after another. + """ from litellm.proxy.management_helpers.audit_logs import create_object_audit_log - for user in updated_users: - if user.user_id is None or user.user_id in existing_user_ids: - continue - await create_object_audit_log( + created_user_entries = tuple( + create_object_audit_log( object_id=user.user_id, action="created", litellm_changed_by=None, @@ -2531,8 +2549,11 @@ async def _create_team_member_add_audit_logs( before_value=None, after_value=safe_dumps(user.model_dump(exclude_none=True)), ) + for user in updated_users + if user.user_id is not None and user.user_id not in existing_user_ids + ) - await create_object_audit_log( + membership_entry = create_object_audit_log( object_id=team_id, action="updated", litellm_changed_by=None, @@ -2543,6 +2564,8 @@ async def _create_team_member_add_audit_logs( after_value=_members_audit_value(after_members), ) + await asyncio.gather(*created_user_entries, membership_entry) + async def _validate_and_populate_member_user_info( member: Member, diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index bedd4dd1838..f64994cb3b1 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -1475,7 +1475,7 @@ async def test_create_team_member_add_team_admin( user_api_key_dict=valid_token, ) except HTTPException as e: - if user_role == "user": + if user_role == "user" or new_member_method == "user_id": assert e.status_code == 403 return else: diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index b1438447e4f..0de2c1ac71d 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -10440,20 +10440,18 @@ def test_validate_member_user_id_provisioning_reports_every_unknown_member(): @pytest.mark.asyncio async def test_resolve_existing_member_user_ids_matches_caller_supplied_user_ids(): - """Only caller-supplied user_ids are looked up; unknown ones resolve to nothing.""" + """Caller-supplied user_ids resolve in one query; unknown ones resolve to nothing.""" from litellm.proxy.management_endpoints.team_endpoints import ( _resolve_existing_member_user_ids, ) prisma_client = MagicMock() - - async def find_by_id(user_id): - if user_id == "by-id": - return LiteLLM_UserTable(user_id="by-id", max_budget=None, spend=0.0, user_email=None, models=[]) - return None + find_many = AsyncMock( + return_value=[LiteLLM_UserTable(user_id="by-id", max_budget=None, spend=0.0, user_email=None, models=[])] + ) with patch("litellm.proxy.management_endpoints.team_endpoints.UserRepository") as repo: - repo.return_value.find_by_id = AsyncMock(side_effect=find_by_id) + repo.return_value.table.find_many = find_many resolved = await _resolve_existing_member_user_ids( members=[ @@ -10465,6 +10463,28 @@ async def test_resolve_existing_member_user_ids_matches_caller_supplied_user_ids ) assert resolved == frozenset({"by-id"}) + # one round-trip, and email-only members contribute no id to look up + find_many.assert_awaited_once() + assert find_many.await_args.kwargs["where"] == {"user_id": {"in": ["by-id", "missing"]}} + + +@pytest.mark.asyncio +async def test_resolve_existing_member_user_ids_skips_the_query_when_no_user_ids(): + """An all-email payload must not hit the database at all.""" + from litellm.proxy.management_endpoints.team_endpoints import ( + _resolve_existing_member_user_ids, + ) + + with patch("litellm.proxy.management_endpoints.team_endpoints.UserRepository") as repo: + repo.return_value.table.find_many = AsyncMock() + + resolved = await _resolve_existing_member_user_ids( + members=[Member(user_email="a@example.com", role="user")], + prisma_client=MagicMock(), + ) + + assert resolved == frozenset() + repo.return_value.table.find_many.assert_not_awaited() def test_pre_existing_user_ids_counts_ids_filled_in_by_member_resolution(): @@ -10574,3 +10594,26 @@ async def test_team_member_add_audits_a_user_created_from_a_list_payload(monkeyp mock_audit.assert_called_once() assert created_user_id not in mock_audit.call_args.kwargs["existing_user_ids"] + + +def test_validate_member_user_id_provisioning_caps_the_ids_it_echoes_back(): + """A large member list must not echo every id back in the error body.""" + from litellm.proxy.management_endpoints.team_endpoints import ( + _MAX_REPORTED_UNKNOWN_USER_IDS, + _validate_member_user_id_provisioning, + ) + + members = [Member(user_id=f"u{i}", role="user") for i in range(500)] + + with pytest.raises(HTTPException) as exc_info: + _validate_member_user_id_provisioning( + members=members, + existing_user_ids=frozenset(), + user_api_key_dict=_provisioning_caller(LitellmUserRoles.INTERNAL_USER), + ) + + detail = str(exc_info.value.detail) + assert "u0" in detail + assert f"u{_MAX_REPORTED_UNKNOWN_USER_IDS}" not in detail + assert f"and {500 - _MAX_REPORTED_UNKNOWN_USER_IDS} more" in detail + assert len(detail) < 1000 From 77e490a69513c44fa2157d41d8a69ed5333b529f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 31 Jul 2026 22:52:00 -0700 Subject: [PATCH 021/123] fix(proxy): publish router_settings changes so peer pods apply them on resync add_deployment already reapplies DB router settings through _update_llm_router, so gating router_settings out of the pub/sub publish set left the push path covering less than the resync actually applies --- .../proxy/common_utils/config_sync_pubsub.py | 1 + .../common_utils/test_config_sync_pubsub.py | 10 +++--- .../proxy/proxy_server/test_proxy_config.py | 36 +++++++++++++++++++ 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/common_utils/config_sync_pubsub.py b/litellm/proxy/common_utils/config_sync_pubsub.py index e4521c397b7..76c3066ae83 100644 --- a/litellm/proxy/common_utils/config_sync_pubsub.py +++ b/litellm/proxy/common_utils/config_sync_pubsub.py @@ -60,6 +60,7 @@ _CONFIG_SYNCED_TABLE_NAMES: frozenset[str] = frozenset( _RESYNC_APPLIED_CONFIG_PARAM_NAMES: frozenset[str] = frozenset( { "general_settings", + "router_settings", "litellm_settings", "model_cost_map_reload_config", "anthropic_beta_headers_reload_config", diff --git a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py index f50eef4f1cc..6872407808c 100644 --- a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py +++ b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py @@ -56,10 +56,11 @@ _EXPECTED_RESYNC_APPLIED_CONFIG_PARAM_NAMES = frozenset( "general_settings", "litellm_settings", "model_cost_map_reload_config", + "router_settings", } ) -_STARTUP_ONLY_CONFIG_PARAM_NAMES = ("environment_variables", "router_settings") +_STARTUP_ONLY_CONFIG_PARAM_NAMES = ("environment_variables",) class _RecordingRedisClient(Redis): @@ -716,13 +717,14 @@ async def _publish_calls_for_invalidated_param(param_name: str) -> List[Tuple[st return client.published -async def test_invalidate_config_param_publishes_params_a_resync_applies() -> None: - published = await _publish_calls_for_invalidated_param("general_settings") +@pytest.mark.parametrize("param_name", sorted(_EXPECTED_RESYNC_APPLIED_CONFIG_PARAM_NAMES)) +async def test_invalidate_config_param_publishes_params_a_resync_applies(param_name: str) -> None: + published = await _publish_calls_for_invalidated_param(param_name) assert len(published) == 1 channel, message = published[0] assert channel == CONFIG_SYNC_CHANNEL - assert json.loads(message) == {"object_type": "general_settings"} + assert json.loads(message) == {"object_type": param_name} @pytest.mark.parametrize("param_name", _STARTUP_ONLY_CONFIG_PARAM_NAMES) diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 8d1d8185e4d..a79bd25b60b 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -2058,6 +2058,42 @@ async def test_ProxyConfig__add_router_settings_from_db_config_none_router_noop( await pc._add_router_settings_from_db_config() # type: ignore[call-arg] +# --------------------------------------------------------------------------- +# ProxyConfig.add_deployment +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_ProxyConfig_add_deployment_applies_db_router_settings(monkeypatch): + from litellm.proxy import proxy_server + + pc = ProxyConfig() + fake_router = MagicMock() + fake_router.get_model_list = MagicMock(return_value=[]) + fake_prisma = MagicMock() + fake_prisma.db.litellm_config.find_first = AsyncMock( + return_value=SimpleNamespace(param_value={"routing_strategy": "latency-based-routing"}) + ) + + async def fake_get_config(*args, **kwargs): + return {} + + monkeypatch.setattr(pc, "get_config", fake_get_config) + monkeypatch.setattr(pc, "_get_models_from_db", AsyncMock(return_value=[])) + monkeypatch.setattr(pc, "_init_non_llm_objects_in_db", AsyncMock()) + monkeypatch.setattr(proxy_server, "prefetch_config_params", AsyncMock()) + monkeypatch.setattr(proxy_server, "get_config_param", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_server, "llm_router", fake_router) + monkeypatch.setattr(proxy_server, "master_key", "sk-master") + monkeypatch.setattr(proxy_server, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "proxy_config", pc) + + await pc.add_deployment(prisma_client=fake_prisma, proxy_logging_obj=MagicMock()) + + fake_router.update_settings.assert_called_once_with(routing_strategy="latency-based-routing") + + # --------------------------------------------------------------------------- # ProxyConfig._add_general_settings_from_db_config # --------------------------------------------------------------------------- From add2a1ce3f6c5c7f2462091d97b70cb7c889bbc0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 1 Aug 2026 03:57:48 -0700 Subject: [PATCH 022/123] chore(typing): clear 2.4k basedpyright errors across 15 Any hotspot files Replace Any-typed seams with real types in the files carrying the highest reportAny/reportExplicitAny density. The dominant source was the repository layer: BaseRepository.table is declared Any, so every repository read poisoned its rows and every downstream call. Typed pass-through accessors under a _PrismaTableActions Protocol pay that crossing once per table, and TypedDicts and Protocols replace the remaining Any-typed request, row, and tool payloads across the team, key, SCIM, spend, MCP, guardrail, video, and websearch surfaces No casts, no type: ignore, no noqa, no new Any annotations, no behavior changes. Whole-tree basedpyright: reportAny 20,840 -> 19,397, reportExplicitAny 7,253 -> 6,518, all rules 151,424 -> 149,066, with no rule increased in any file. Budgets ratcheted: basedpyright -2,358, ruff-strict -300, type-discipline -68 --- basedpyright-code-budget.json | 18 +- .../websearch_interception/handler.py | 107 ++-- .../context_management/editors/compact.py | 134 +++-- litellm/llms/custom_httpx/llm_http_handler.py | 558 +++++++++--------- litellm/llms/openai/videos/transformation.py | 61 +- .../mcp_server/mcp_server_manager.py | 188 ++++-- .../mcp_server/sampling_handler.py | 209 ++++--- .../proxy/guardrails/guardrail_endpoints.py | 96 ++- .../internal_user_endpoints.py | 159 +++-- .../key_management_endpoints.py | 174 ++++-- .../model_management_endpoints.py | 17 +- .../management_endpoints/scim/scim_v2.py | 255 +++++--- .../management_endpoints/team_endpoints.py | 297 +++++++--- .../spend_management_endpoints.py | 368 +++++++++--- .../mcp/litellm_proxy_mcp_handler.py | 72 ++- litellm/videos/main.py | 204 +++---- ruff-strict-budget.json | 20 +- type-discipline-budget.json | 8 +- 18 files changed, 1903 insertions(+), 1042 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 43df27ea2e2..f6dd90077b1 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 31256 + "limit": 29813 }, "reportArgumentType": { "limit": 2645 @@ -24,7 +24,7 @@ "limit": 42 }, "reportExplicitAny": { - "limit": 10208 + "limit": 9473 }, "reportFunctionMemberAccess": { "limit": 11 @@ -54,10 +54,10 @@ "limit": 0 }, "reportMissingParameterType": { - "limit": 5869 + "limit": 5855 }, "reportMissingTypeArgument": { - "limit": 15861 + "limit": 15852 }, "reportMissingTypeStubs": { "limit": 41 @@ -99,19 +99,19 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 45357 + "limit": 45324 }, "reportUnknownLambdaType": { "limit": 113 }, "reportUnknownMemberType": { - "limit": 40477 + "limit": 40452 }, "reportUnknownParameterType": { - "limit": 20338 + "limit": 20309 }, "reportUnknownVariableType": { - "limit": 32047 + "limit": 31978 }, "reportUnnecessaryCast": { "limit": 177 @@ -123,7 +123,7 @@ "limit": 7 }, "reportUnnecessaryIsInstance": { - "limit": 1205 + "limit": 1204 }, "reportUntypedBaseClass": { "limit": 165 diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 21d990e8e60..531caf273f1 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -9,7 +9,8 @@ server-side using litellm router's search tools. import asyncio import math import uuid -from typing import Any, Dict, List, Optional, Tuple, Union, cast +from collections.abc import AsyncIterator, Mapping +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast import litellm from litellm._logging import verbose_logger @@ -29,19 +30,31 @@ from litellm.integrations.websearch_interception.transformation import ( WebSearchTransformation, ) from litellm.llms.base_llm.search.transformation import SearchResponse -from litellm.types.integrations.websearch_interception import ( - WebSearchInterceptionConfig, -) from litellm.types.integrations.custom_logger import ( CHAT_COMPLETION_AGENTIC_SURFACE, RESPONSES_AGENTIC_SURFACE, AgenticLoopPlan, AgenticLoopRequestPatch, ) +from litellm.types.integrations.websearch_interception import ( + WebSearchInterceptionConfig, +) from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import CallTypes, LlmProviders from litellm.utils import ProviderConfigManager +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.base_llm.anthropic_messages.transformation import ( + BaseAnthropicMessagesConfig, + ) + from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.llms.anthropic_messages.anthropic_response import ( + AnthropicMessagesResponse, + ) + from litellm.types.utils import ModelResponse + from litellm.utils import CustomStreamWrapper + # Key used to flag, on per-request kwargs, that the originating client sent # an Anthropic-native ``web_search_*`` tool — meaning the final response # should include ``web_search_tool_result`` content blocks so the client @@ -94,8 +107,8 @@ class WebSearchInterceptionLogger(CustomLogger): messages: List[Dict], tools: Optional[List[Dict]], custom_llm_provider: Optional[str], - kwargs: Optional[dict[str, Any]] = None, - ) -> Optional[Dict[str, Any]]: + kwargs: Mapping[str, object] | None = None, + ) -> dict[str, object] | None: """ Short-circuit web-search-only requests by executing the search directly. @@ -188,7 +201,7 @@ class WebSearchInterceptionLogger(CustomLogger): verbose_logger.error(f"WebSearchInterception: Short-circuit search failed: {e}") search_result_text, structured = f"Search failed: {e}", None - content: List[Dict[str, Any]] = [] + content: list[dict[str, object]] = [] if native_tool is not None: tool_use_id = f"srvtoolu_{uuid.uuid4().hex}" tool_name = native_tool.get("name") or "web_search" @@ -210,7 +223,7 @@ class WebSearchInterceptionLogger(CustomLogger): # github_copilot, etc.) see the same payload they always have. content.append({"type": "text", "text": search_result_text}) - response: Dict[str, Any] = { + response: dict[str, object] = { "id": f"msg_{str(uuid.uuid4())}", "type": "message", "role": "assistant", @@ -228,7 +241,9 @@ class WebSearchInterceptionLogger(CustomLogger): ) return response - async def async_pre_call_deployment_hook(self, kwargs: Dict[str, Any], call_type: Optional[Any]) -> Optional[dict]: + async def async_pre_call_deployment_hook( + self, kwargs: Dict[str, Any], call_type: Optional[CallTypes] + ) -> Optional[dict]: """ Pre-call hook to convert native Anthropic web_search tools to regular tools. @@ -297,7 +312,7 @@ class WebSearchInterceptionLogger(CustomLogger): return kwargs - def _convert_responses_tools(self, kwargs: dict[str, Any], tools: list[dict[str, Any]]) -> dict | None: + def _convert_responses_tools(self, kwargs: Mapping[str, object], tools: list[dict[str, object]]) -> dict | None: """Convert Responses API web search tools to the LiteLLM standard function tool.""" if not any(is_web_search_tool_responses(tool) for tool in tools): return None @@ -370,7 +385,7 @@ class WebSearchInterceptionLogger(CustomLogger): return tool.get("name") @classmethod - def _sync_forced_tool_choice(cls, tool_choice: Any, converted_tools: list[dict[str, Any]]) -> Any: + def _sync_forced_tool_choice(cls, tool_choice: Any, converted_tools: list[dict[str, object]]) -> object: """Repoint a forced ``tool_choice`` at ``litellm_web_search`` when it names a web-search tool that was just converted away. @@ -468,7 +483,7 @@ class WebSearchInterceptionLogger(CustomLogger): async def async_should_run_agentic_loop( self, - response: Any, + response: object, model: str, messages: List[Dict], tools: Optional[List[Dict]], @@ -578,7 +593,7 @@ class WebSearchInterceptionLogger(CustomLogger): async def async_should_run_chat_completion_agentic_loop( self, - response: Any, + response: object, model: str, messages: List[Dict], tools: Optional[List[Dict]], @@ -636,7 +651,7 @@ class WebSearchInterceptionLogger(CustomLogger): async def async_should_run_responses_agentic_loop( self, - response: Any, + response: object, model: str, messages: list[dict], tools: list[dict] | None, @@ -687,13 +702,13 @@ class WebSearchInterceptionLogger(CustomLogger): tools: Dict, model: str, messages: List[Dict], - response: Any, - anthropic_messages_provider_config: Any, + response: object, + anthropic_messages_provider_config: "BaseAnthropicMessagesConfig | None", anthropic_messages_optional_request_params: Dict, - logging_obj: Any, + logging_obj: "LiteLLMLoggingObj | None", stream: bool, kwargs: Dict, - ) -> Any: + ) -> "AnthropicMessagesResponse | AsyncIterator[object]": """ Execute agentic loop with WebSearch execution for Anthropic Messages API. @@ -721,10 +736,10 @@ class WebSearchInterceptionLogger(CustomLogger): tools: Dict, model: str, messages: List[Dict], - response: Any, - anthropic_messages_provider_config: Any, + response: object, + anthropic_messages_provider_config: "BaseAnthropicMessagesConfig | None", anthropic_messages_optional_request_params: Dict, - logging_obj: Any, + logging_obj: "LiteLLMLoggingObj | None", stream: bool, kwargs: Dict, ) -> AgenticLoopPlan: @@ -764,7 +779,7 @@ class WebSearchInterceptionLogger(CustomLogger): kwargs=kwargs, ) - metadata: Dict[str, Any] = { + metadata: dict[str, object] = { "tool_type": "websearch", "response_format": "anthropic", } @@ -787,10 +802,10 @@ class WebSearchInterceptionLogger(CustomLogger): async def async_post_agentic_loop_response_hook( self, - response: Any, + response: object, plan: AgenticLoopPlan, kwargs: Dict, - ) -> Any: + ) -> object: """ Inject Anthropic-native ``web_search_tool_result`` blocks into the final response when the originating client used a native @@ -810,9 +825,9 @@ class WebSearchInterceptionLogger(CustomLogger): def _build_native_result_blocks( tool_calls: List[Dict], structured_results: List[Optional[SearchResponse]], - ) -> List[Dict[str, Any]]: + ) -> list[dict[str, object]]: """Build one ``web_search_tool_result`` block per tool_call.""" - blocks: List[Dict[str, Any]] = [] + blocks: list[dict[str, object]] = [] for i, tool_call in enumerate(tool_calls): tool_use_id = tool_call.get("id") or "" structured = structured_results[i] if i < len(structured_results) else None @@ -825,7 +840,7 @@ class WebSearchInterceptionLogger(CustomLogger): return blocks @staticmethod - def _inject_native_blocks(response: Any, native_blocks: List[Dict[str, Any]]) -> Any: + def _inject_native_blocks(response: Any, native_blocks: list[dict[str, object]]) -> Any: """Prepend native blocks to response content, dict or object form.""" if not native_blocks: return response @@ -849,12 +864,12 @@ class WebSearchInterceptionLogger(CustomLogger): tools: Dict, model: str, messages: List[Dict], - response: Any, + response: object, optional_params: Dict, - logging_obj: Any, + logging_obj: "LiteLLMLoggingObj | None", stream: bool, kwargs: Dict, - ) -> Any: + ) -> "ModelResponse | CustomStreamWrapper": """ Execute agentic loop with WebSearch execution for Chat Completions API. @@ -884,9 +899,9 @@ class WebSearchInterceptionLogger(CustomLogger): tools: Dict, model: str, messages: List[Dict], - response: Any, + response: object, optional_params: Dict, - logging_obj: Any, + logging_obj: "LiteLLMLoggingObj | None", stream: bool, kwargs: Dict, ) -> AgenticLoopPlan: @@ -911,9 +926,9 @@ class WebSearchInterceptionLogger(CustomLogger): tools: dict, model: str, messages: list[dict], - response: Any, + response: object, optional_params: dict, - logging_obj: Any, + logging_obj: "LiteLLMLoggingObj | None", stream: bool, kwargs: dict, ) -> AgenticLoopPlan: @@ -1023,7 +1038,7 @@ class WebSearchInterceptionLogger(CustomLogger): return [] @staticmethod - def _extract_search_text(result: Any) -> str: + def _extract_search_text(result: object) -> str: if isinstance(result, Exception): verbose_logger.error(f"WebSearchInterception: Responses search failed with error: {str(result)}") return f"Search failed: {str(result)}" @@ -1091,10 +1106,10 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls: List[Dict], thinking_blocks: List[Dict], anthropic_messages_optional_request_params: Dict, - logging_obj: Any, + logging_obj: "LiteLLMLoggingObj | None", stream: bool, kwargs: Dict, - ) -> Any: + ) -> "AnthropicMessagesResponse | AsyncIterator[object]": """Legacy path: execute search + build patch + run follow-up call.""" request_patch, structured_results = await self._build_anthropic_request_patch( model=model, @@ -1118,7 +1133,7 @@ class WebSearchInterceptionLogger(CustomLogger): if max_tokens is None: max_tokens = cast(int, kwargs.get("max_tokens", 1024)) - response = await anthropic_messages.acreate( + response: AnthropicMessagesResponse | AsyncIterator[object] = await anthropic_messages.acreate( max_tokens=max_tokens, messages=request_patch.messages, model=request_patch.model or model, @@ -1145,7 +1160,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls: List[Dict], thinking_blocks: List[Dict], anthropic_messages_optional_request_params: Dict, - logging_obj: Any, + logging_obj: "LiteLLMLoggingObj | None", kwargs: Dict, ) -> Tuple[AgenticLoopRequestPatch, List[Optional[SearchResponse]]]: """ @@ -1238,7 +1253,7 @@ class WebSearchInterceptionLogger(CustomLogger): return patch, structured_results async def _execute_search( - self, query: str, kwargs: Optional[dict[str, Any]] = None + self, query: str, kwargs: Mapping[str, object] | None = None ) -> Tuple[str, Optional[SearchResponse]]: """ Execute a single web search using router's search tools. @@ -1300,8 +1315,8 @@ class WebSearchInterceptionLogger(CustomLogger): async def _authorize_search_tool( self, - search_tool: dict[str, Any], - kwargs: Optional[dict[str, Any]], + search_tool: Mapping[str, object], + kwargs: Mapping[str, object] | None, ) -> None: search_tool_name = search_tool.get("search_tool_name") if not isinstance(search_tool_name, str) or not search_tool_name: @@ -1343,7 +1358,7 @@ class WebSearchInterceptionLogger(CustomLogger): ) @staticmethod - def _get_user_api_key_auth_from_kwargs(kwargs: Optional[dict[str, Any]]) -> Any: + def _get_user_api_key_auth_from_kwargs(kwargs: Mapping[str, object] | None) -> "UserAPIKeyAuth | None": if not kwargs: return None @@ -1363,7 +1378,7 @@ class WebSearchInterceptionLogger(CustomLogger): return None - def _select_search_tool_from_router(self, llm_router: Any) -> Optional[dict[str, Any]]: + def _select_search_tool_from_router(self, llm_router: object) -> Optional[dict[str, Any]]: if llm_router is None or not hasattr(llm_router, "search_tools"): return None search_tools = list(getattr(llm_router, "search_tools") or []) @@ -1405,11 +1420,11 @@ class WebSearchInterceptionLogger(CustomLogger): messages: List[Dict], tool_calls: List[Dict], optional_params: Dict, - logging_obj: Any, + logging_obj: "LiteLLMLoggingObj | None", stream: bool, kwargs: Dict, response_format: str = "openai", - ) -> Any: + ) -> "ModelResponse | CustomStreamWrapper": """Legacy path: execute search + build patch + run follow-up call.""" request_patch = await self._build_chat_completion_request_patch( model=model, diff --git a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py index f18a9f41939..c87014ffc1e 100644 --- a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py +++ b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py @@ -13,7 +13,8 @@ Mirrors Anthropic's native ``compact_20260112`` for non-Anthropic providers: """ import re -from typing import Any, Dict, List, Literal, Optional, Tuple, Union, cast +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union, cast import litellm from litellm._logging import verbose_logger @@ -23,6 +24,18 @@ from litellm.types.llms.anthropic import ( UsageIteration, ) +if TYPE_CHECKING: + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.proxy._types import UserAPIKeyAuth + from litellm.router import Router + from litellm.types.llms.anthropic import ( + AllAnthropicToolsValues, + AnthopicMessagesAssistantMessageParam, + AnthropicMessagesUserMessageParam, + ) + from litellm.types.llms.openai import ChatCompletionToolParam + from litellm.types.utils import ModelResponse + from ..constants import ( COMPACT_DEFAULT_INSTRUCTIONS, COMPACT_DEFAULT_TRIGGER_TOKENS, @@ -98,9 +111,9 @@ def _read_summary_max_tokens_setting() -> int: async def _check_summary_model_access( - user_api_key_auth: Any, + user_api_key_auth: Optional["UserAPIKeyAuth"], summary_model: str, - llm_router: Any, + llm_router: Optional["Router"], ) -> bool: """Return True when every model-allowlist scope on the parent request is satisfied for ``summary_model``. @@ -294,7 +307,7 @@ async def _check_summary_model_access( async def _check_summary_model_budget( - user_api_key_auth: Any, + user_api_key_auth: Optional["UserAPIKeyAuth"], summary_model: str, ) -> bool: """Return True when the caller is within their per-model budget for @@ -357,7 +370,7 @@ async def _check_summary_model_budget( async def _check_summary_model_rate_limit( - user_api_key_auth: Any, + user_api_key_auth: Optional["UserAPIKeyAuth"], summary_model: str, ) -> bool: """Return True when the caller is within their configured RPM/TPM limits @@ -433,7 +446,7 @@ async def _check_summary_model_rate_limit( def _find_latest_compaction_index( - messages: List[Dict[str, Any]], + messages: List[Dict[str, object]], ) -> Tuple[Optional[int], Optional[int]]: """Return (message_index, block_index) of the most recent compaction block. @@ -453,7 +466,7 @@ def _find_latest_compaction_index( def _slice_around_compaction_block( messages: List[Dict[str, Any]], -) -> Tuple[List[Dict[str, Any]], Optional[Dict[str, Any]]]: +) -> Tuple[List[Dict[str, object]], Optional[Dict[str, object]]]: """Apply Anthropic's "drop everything before the compaction block" rule. Returns ``(sliced_messages_with_compaction_block, compaction_block_dict)`` @@ -468,27 +481,26 @@ def _slice_around_compaction_block( original_msg = messages[msg_idx] original_content = original_msg["content"] - compaction_block = cast(Dict[str, Any], original_content[blk_idx]) + compaction_block = cast(Dict[str, object], original_content[blk_idx]) # Per Anthropic's contract everything before the compaction block is # dropped, including earlier blocks within the same assistant message. sliced_content = list(original_content[blk_idx:]) - sliced_first_msg = {**original_msg, "content": sliced_content} - sliced_messages: List[Dict[str, Any]] = [sliced_first_msg] + sliced_messages: List[Dict[str, object]] = [{**original_msg, "content": sliced_content}] sliced_messages.extend(messages[msg_idx + 1 :]) return sliced_messages, compaction_block def _strip_compaction_blocks( - messages: List[Dict[str, Any]], -) -> List[Dict[str, Any]]: + messages: List[Dict[str, object]], +) -> List[Dict[str, object]]: """Drop any ``compaction`` content blocks from messages. Used to build the downstream-bound message list — the adapter has no concept of a compaction block, so it must not see one. """ - cleaned: List[Dict[str, Any]] = [] + cleaned: List[Dict[str, object]] = [] for msg in messages: content = msg.get("content") if not isinstance(content, list): @@ -503,9 +515,9 @@ def _strip_compaction_blocks( def _augment_system_with_summary( - system: Optional[Union[str, List[Dict[str, Any]]]], + system: Optional[Union[str, List[Dict[str, object]]]], summary_text: str, -) -> Union[str, List[Dict[str, Any]]]: +) -> Union[str, List[Dict[str, object]]]: """Prepend a "Previous conversation summary: ..." block to ``system``.""" prefix = f"{COMPACT_SUMMARY_SYSTEM_PREFIX}{summary_text}\n\n" if system is None: @@ -522,7 +534,7 @@ def _augment_system_with_summary( return [{"type": "text", "text": prefix.rstrip()}, *system] -def _resolve_trigger_tokens(edit_spec: Dict[str, Any]) -> Tuple[int, List[str]]: +def _resolve_trigger_tokens(edit_spec: Dict[str, object]) -> Tuple[int, List[str]]: """Validate and resolve ``trigger.value``. Raises ``AnthropicContextManagementError`` if the explicitly-supplied value @@ -556,7 +568,7 @@ def _resolve_trigger_tokens(edit_spec: Dict[str, Any]) -> Tuple[int, List[str]]: return value, warnings -def _build_summary_prompt(edit_spec: Dict[str, Any], tools: Optional[List[Dict[str, Any]]]) -> str: +def _build_summary_prompt(edit_spec: Dict[str, object], tools: Optional[List[Dict[str, object]]]) -> str: custom = edit_spec.get("instructions") if isinstance(custom, str) and custom.strip(): return custom @@ -567,8 +579,8 @@ def _build_summary_prompt(edit_spec: Dict[str, Any], tools: Optional[List[Dict[s def _propagate_metadata( - parent_litellm_metadata: Optional[Dict[str, Any]], -) -> Dict[str, Any]: + parent_litellm_metadata: Optional[Mapping[str, object]], +) -> Dict[str, object]: """Extract the parent request's auth/spend-attribution fields for the summary subcall. The proxy attaches ``user_api_key``, ``user_api_key_team_id`` etc. to @@ -579,7 +591,7 @@ def _propagate_metadata( """ if not parent_litellm_metadata: return {} - propagated: Dict[str, Any] = {} + propagated: Dict[str, object] = {} for key in _PROPAGATED_METADATA_KEYS: if key in parent_litellm_metadata: propagated[key] = parent_litellm_metadata[key] @@ -588,10 +600,10 @@ def _propagate_metadata( def _count_effective_tokens( model: str, - effective_messages: List[Dict[str, Any]], - compaction_block: Optional[Dict[str, Any]], - tools: Optional[List[Dict[str, Any]]], - system: Optional[Union[str, List[Dict[str, Any]]]] = None, + effective_messages: List[Dict[str, object]], + compaction_block: Optional[CompactionBlock], + tools: Optional[List[Dict[str, object]]], + system: Optional[Union[str, List[Dict[str, object]]]] = None, ) -> int: """Token-count the conversation as it will appear downstream. @@ -609,25 +621,32 @@ def _count_effective_tokens( messages_without_compaction = _strip_compaction_blocks(effective_messages) adapter = LiteLLMAnthropicMessagesAdapter() try: - openai_shape = adapter.translate_anthropic_messages_to_openai(messages=cast(Any, messages_without_compaction)) + openai_shape = adapter.translate_anthropic_messages_to_openai( + messages=cast( + "List[Union[AnthropicMessagesUserMessageParam, AnthopicMessagesAssistantMessageParam]]", + messages_without_compaction, + ) + ) except Exception as e: verbose_logger.debug( "compact_20260112: anthropic→openai translation failed during token " "count, falling back to raw messages: %s", e, ) - openai_shape = cast(Any, messages_without_compaction) + openai_shape = messages_without_compaction # Translate Anthropic-shaped tools (``input_schema``) to OpenAI-shaped # tools (``{"type": "function", "function": {...}}``) so ``token_counter`` # gets a consistent format regardless of which counting path it uses. # An inaccurate tool token count here could cause the polyfill to skip # needed compaction or trigger unnecessary summarization. - openai_tools: Optional[List[Dict[str, Any]]] = None + openai_tools: Optional[List[Dict[str, object]]] = None if tools: try: - translated_tools, _ = adapter.translate_anthropic_tools_to_openai(tools=cast(Any, tools)) - openai_tools = cast(List[Dict[str, Any]], translated_tools) + translated_tools, _ = adapter.translate_anthropic_tools_to_openai( + tools=cast("List[AllAnthropicToolsValues]", tools) + ) + openai_tools = cast(List[Dict[str, object]], translated_tools) except Exception as e: verbose_logger.debug( "compact_20260112: anthropic→openai tools translation failed " @@ -638,8 +657,8 @@ def _count_effective_tokens( total = litellm.token_counter( model=model, - messages=cast(Any, openai_shape), - tools=cast(Any, openai_tools), + messages=cast(List[Dict[str, object]], openai_shape), + tools=cast("Optional[List[ChatCompletionToolParam]]", openai_tools), ) if compaction_block is not None: content = compaction_block.get("content") or "" @@ -652,7 +671,7 @@ def _count_effective_tokens( def _system_to_text( - system: Optional[Union[str, List[Dict[str, Any]]]], + system: Optional[Union[str, List[Dict[str, object]]]], ) -> str: """Flatten an Anthropic-style ``system`` value into a single string for token counting. Returns ``""`` when ``system`` carries no text.""" @@ -670,8 +689,8 @@ def _system_to_text( def _select_last_user_question( - messages: List[Dict[str, Any]], -) -> List[Dict[str, Any]]: + messages: List[Dict[str, object]], +) -> List[Dict[str, object]]: """Pick the most recent ``user`` turn that is a real question. Returns a one-element message list with any ``tool_result`` blocks @@ -735,10 +754,10 @@ def _system_to_openai_message( def _build_summary_messages( - effective_messages: List[Dict[str, Any]], + effective_messages: List[Dict[str, object]], prompt: str, - system: Optional[Union[str, List[Dict[str, Any]]]] = None, -) -> List[Dict[str, Any]]: + system: Optional[Union[str, List[Dict[str, object]]]] = None, +) -> List[Dict[str, object]]: """Build the OpenAI-shape message list for the summary call. The caller's ``system`` prompt is prepended (the default summarization @@ -753,7 +772,10 @@ def _build_summary_messages( stripped = _strip_compaction_blocks(effective_messages) try: openai_messages = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( - messages=cast(Any, stripped) + messages=cast( + "List[Union[AnthropicMessagesUserMessageParam, AnthopicMessagesAssistantMessageParam]]", + stripped, + ) ) except Exception as e: verbose_logger.warning( @@ -761,9 +783,9 @@ def _build_summary_messages( "building summary call; falling back to raw shape: %s", e, ) - openai_messages = cast(Any, stripped) + openai_messages = stripped - summary_messages: List[Dict[str, Any]] = [] + summary_messages: List[Dict[str, object]] = [] system_message = _system_to_openai_message(system) if system_message is not None: summary_messages.append(system_message) @@ -783,7 +805,7 @@ def _build_summary_messages( return summary_messages -def _is_user_message(msg: Any) -> bool: +def _is_user_message(msg: object) -> bool: return isinstance(msg, dict) and msg.get("role") == "user" @@ -805,12 +827,12 @@ def _append_text_to_content(content: Any, extra_text: str) -> Any: async def _call_summary_model( *, summary_model: str, - summary_messages: List[Dict[str, Any]], - metadata: Dict[str, Any], + summary_messages: List[Dict[str, object]], + metadata: Mapping[str, object], llm_router: Any, allowed_model_region: Optional[str] = None, max_tokens: int = COMPACT_SUMMARY_MAX_TOKENS, -) -> Any: +) -> Union["ModelResponse", "CustomStreamWrapper"]: """Invoke the configured summary model. Prefers ``llm_router.acompletion`` so the model alias resolves against the @@ -877,7 +899,7 @@ def _extract_response_text(response: Any) -> Optional[str]: return None -def _extract_usage(response: Any) -> Tuple[int, int]: +def _extract_usage(response: object) -> Tuple[int, int]: usage = getattr(response, "usage", None) if usage is None: return 0, 0 @@ -889,8 +911,8 @@ def _extract_usage(response: Any) -> Tuple[int, int]: def apply_client_compaction_block_history( *, - messages: List[Dict[str, Any]], - system: Optional[Union[str, List[Dict[str, Any]]]], + messages: List[Dict[str, object]], + system: Optional[Union[str, List[Dict[str, object]]]], ) -> Optional[PolyfillResult]: """Honor client-sent compaction blocks without a ``compact_20260112`` edit. @@ -911,7 +933,7 @@ def apply_client_compaction_block_history( ) prior_summary_text = prior_compaction_block.get("content") or "" - augmented_system: Union[str, List[Dict[str, Any]], None] = system + augmented_system: Union[str, List[Dict[str, object]], None] = system if isinstance(prior_summary_text, str) and prior_summary_text: augmented_system = _augment_system_with_summary(system, prior_summary_text) verbose_logger.info( @@ -936,13 +958,13 @@ def apply_client_compaction_block_history( async def apply_compact_20260112( *, model: str, - messages: List[Dict[str, Any]], - tools: Optional[List[Dict[str, Any]]], - system: Optional[Union[str, List[Dict[str, Any]]]], - edit_spec: Dict[str, Any], - litellm_metadata: Optional[Dict[str, Any]] = None, - llm_router: Any = None, - user_api_key_auth: Any = None, + messages: List[Dict[str, object]], + tools: Optional[List[Dict[str, object]]], + system: Optional[Union[str, List[Dict[str, object]]]], + edit_spec: Dict[str, object], + litellm_metadata: Optional[Mapping[str, object]] = None, + llm_router: Optional["Router"] = None, + user_api_key_auth: Optional["UserAPIKeyAuth"] = None, ) -> PolyfillResult: """Apply ``compact_20260112``; return a ``PolyfillResult``. @@ -971,7 +993,7 @@ async def apply_compact_20260112( # non-Anthropic backends (which would reject them). effective_messages, prior_compaction_block = _slice_around_compaction_block(messages) prior_summary_text = prior_compaction_block.get("content") if prior_compaction_block else None - augmented_system: Union[str, List[Dict[str, Any]], None] = system + augmented_system: Union[str, List[Dict[str, object]], None] = system if isinstance(prior_summary_text, str) and prior_summary_text: augmented_system = _augment_system_with_summary(system, prior_summary_text) verbose_logger.info( diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index d6acbaae434..b64d8e3dc7e 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -13,8 +13,10 @@ from typing import ( Iterator, List, Literal, + Mapping, Optional, Tuple, + TypeVar, Union, cast, get_type_hints, @@ -162,7 +164,11 @@ from .http_handler import get_shared_realtime_ssl_context if TYPE_CHECKING: from aiohttp import ClientSession + from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import ( + FakeAnthropicMessagesStreamIterator, + ) from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import ( AnthropicMessagesStreamingResponse, ) @@ -182,6 +188,8 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_ResponseT = TypeVar("_ResponseT") + def _google_genai_streaming_hidden_params( *, @@ -189,11 +197,11 @@ def _google_genai_streaming_hidden_params( litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, response_headers: httpx.Headers, -) -> Dict[str, Any]: +) -> Dict[str, object]: """Pre-stream metadata for proxy response headers (mirrors CustomStreamWrapper._hidden_params).""" from litellm.litellm_core_utils.core_helpers import process_response_headers - _model_info: Dict[str, Any] = dict(getattr(litellm_params, "model_info", None) or {}) + _model_info: Mapping[str, object] = dict(getattr(litellm_params, "model_info", None) or {}) _raw_id = _model_info.get("id") or logging_obj.get_router_model_id() or "" _model_id = _raw_id if isinstance(_raw_id, str) else str(_raw_id) return { @@ -210,7 +218,7 @@ def _responses_api_optional_request_param_names() -> frozenset[str]: return frozenset(get_type_hints(ResponsesAPIOptionalRequestParams).keys()) -def _custom_logger_callbacks(logging_obj: Any) -> list[Any]: +def _custom_logger_callbacks(logging_obj: LiteLLMLoggingObj) -> list["CustomLogger"]: from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import ( get_custom_logger_compatible_class, @@ -221,7 +229,7 @@ def _custom_logger_callbacks(logging_obj: Any) -> list[Any]: if isinstance(dynamic_success_callbacks, (list, tuple)): callbacks.extend(dynamic_success_callbacks) - custom_loggers: list[Any] = [] + custom_loggers: list[CustomLogger] = [] for cb in callbacks: if isinstance(cb, str): resolved = get_custom_logger_compatible_class(cb) # type: ignore[arg-type] @@ -233,7 +241,7 @@ def _custom_logger_callbacks(logging_obj: Any) -> list[Any]: return custom_loggers -def _has_pre_call_deployment_hook(logging_obj: Any) -> bool: +def _has_pre_call_deployment_hook(logging_obj: LiteLLMLoggingObj) -> bool: from litellm.integrations.custom_logger import CustomLogger base_func = CustomLogger.async_pre_call_deployment_hook @@ -359,7 +367,7 @@ class BaseLLMHTTPHandler: messages: list, optional_params: dict, litellm_params: dict, - encoding: Any, + encoding: object, api_key: Optional[str] = None, client: Optional[AsyncHTTPHandler] = None, json_mode: bool = False, @@ -425,7 +433,7 @@ class BaseLLMHTTPHandler: api_base: Optional[str], custom_llm_provider: str, model_response: ModelResponse, - encoding, + encoding: object, logging_obj: LiteLLMLoggingObj, optional_params: dict, timeout: Union[float, httpx.Timeout], @@ -474,7 +482,7 @@ class BaseLLMHTTPHandler: litellm_params=litellm_params, ) - data = provider_config.transform_request( + data: Dict[str, object] = provider_config.transform_request( model=model, messages=messages, optional_params=optional_params, @@ -651,7 +659,7 @@ class BaseLLMHTTPHandler: fake_stream: bool = False, client: Optional[HTTPHandler] = None, json_mode: bool = False, - ) -> Tuple[Any, dict]: + ) -> Tuple[object, dict]: if client is None or not isinstance(client, HTTPHandler): sync_httpx_client = _get_httpx_client( { @@ -691,7 +699,7 @@ class BaseLLMHTTPHandler: json_mode=json_mode, ) - completion_stream: Any = MockResponseIterator(model_response=model_response, json_mode=json_mode) + completion_stream: object = MockResponseIterator(model_response=model_response, json_mode=json_mode) else: completion_stream = provider_config.get_model_response_iterator( streaming_response=response.iter_lines(), @@ -783,7 +791,7 @@ class BaseLLMHTTPHandler: client: Optional[AsyncHTTPHandler] = None, json_mode: Optional[bool] = None, signed_json_body: Optional[bytes] = None, - ) -> Tuple[Any, httpx.Headers]: + ) -> Tuple[object, httpx.Headers]: """ Helper function for making an async call with stream. @@ -827,7 +835,7 @@ class BaseLLMHTTPHandler: json_mode=json_mode, ) - completion_stream: Any = MockResponseIterator(model_response=model_response, json_mode=json_mode) + completion_stream: object = MockResponseIterator(model_response=model_response, json_mode=json_mode) else: completion_stream = provider_config.get_model_response_iterator( streaming_response=response.aiter_lines(), sync_stream=False @@ -846,10 +854,10 @@ class BaseLLMHTTPHandler: def _add_stream_param_to_request_body( self, - data: dict, + data: Dict[str, object], provider_config: BaseConfig, fake_stream: bool, - ) -> dict: + ) -> Dict[str, object]: """ Some providers like Bedrock invoke do not support the stream parameter in the request body, we only pass `stream` in the request body the provider supports it. """ @@ -1051,7 +1059,7 @@ class BaseLLMHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]], model_response: RerankResponse, _is_async: bool = False, - headers: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, object]] = None, api_key: Optional[str] = None, api_base: Optional[str] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, @@ -1177,7 +1185,7 @@ class BaseLLMHTTPHandler: logging_obj: LiteLLMLoggingObj, api_key: Optional[str], api_base: Optional[str], - headers: Optional[Dict[str, Any]], + headers: Optional[Dict[str, object]], provider_config: BaseAudioTranscriptionConfig, ) -> Tuple[dict, str, Union[dict, bytes, None], Optional[dict]]: """ @@ -1266,10 +1274,10 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, atranscription: bool = False, - headers: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, object]] = None, provider_config: Optional[BaseAudioTranscriptionConfig] = None, shared_session: Optional["ClientSession"] = None, - ) -> Union[TranscriptionResponse, Coroutine[Any, Any, TranscriptionResponse]]: + ) -> Union[TranscriptionResponse, Coroutine[object, object, TranscriptionResponse]]: if provider_config is None: raise ValueError(f"No provider config found for model: {model} and provider: {custom_llm_provider}") @@ -1351,7 +1359,7 @@ class BaseLLMHTTPHandler: api_base: Optional[str], custom_llm_provider: str, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, - headers: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, object]] = None, provider_config: Optional[BaseAudioTranscriptionConfig] = None, shared_session: Optional["ClientSession"] = None, ) -> TranscriptionResponse: @@ -1417,7 +1425,7 @@ class BaseLLMHTTPHandler: logging_obj: LiteLLMLoggingObj, api_key: Optional[str], api_base: Optional[str], - headers: Optional[Dict[str, Any]], + headers: Optional[Dict[str, object]], provider_config: BaseOCRConfig, litellm_params: dict, ) -> Tuple[Dict[str, Any], str, Dict[str, Any], None]: @@ -1483,7 +1491,7 @@ class BaseLLMHTTPHandler: logging_obj: LiteLLMLoggingObj, api_key: Optional[str], api_base: Optional[str], - headers: Optional[Dict[str, Any]], + headers: Optional[Dict[str, object]], provider_config: BaseOCRConfig, litellm_params: dict, ) -> Tuple[Dict[str, Any], str, Dict[str, Any], None]: @@ -1567,10 +1575,10 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, aocr: bool = False, - headers: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, object]] = None, provider_config: Optional[BaseOCRConfig] = None, litellm_params: Optional[dict] = None, - ) -> Union[OCRResponse, Coroutine[Any, Any, OCRResponse]]: + ) -> Union[OCRResponse, Coroutine[object, object, OCRResponse]]: """ Sync OCR handler. """ @@ -1641,7 +1649,7 @@ class BaseLLMHTTPHandler: api_base: Optional[str], custom_llm_provider: str, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, - headers: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, object]] = None, provider_config: Optional[BaseOCRConfig] = None, litellm_params: Optional[dict] = None, ) -> OCRResponse: @@ -1703,9 +1711,9 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, asearch: bool = False, - headers: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, object]] = None, provider_config: Optional[BaseSearchConfig] = None, - ) -> Union[SearchResponse, Coroutine[Any, Any, SearchResponse]]: + ) -> Union[SearchResponse, Coroutine[object, object, SearchResponse]]: """ Sync Search handler. """ @@ -1798,7 +1806,7 @@ class BaseLLMHTTPHandler: api_base: Optional[str], custom_llm_provider: str, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, - headers: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, object]] = None, provider_config: Optional[BaseSearchConfig] = None, ) -> SearchResponse: """ @@ -1976,7 +1984,7 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, client: Optional[AsyncHTTPHandler] = None, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, api_key: Optional[str] = None, api_base: Optional[str] = None, stream: Optional[bool] = False, @@ -2339,10 +2347,10 @@ class BaseLLMHTTPHandler: api_key: Optional[str] = None, api_base: Optional[str] = None, stream: Optional[bool] = False, - kwargs: Optional[Dict[str, Any]] = None, + kwargs: Optional[Dict[str, object]] = None, ) -> Union[ AnthropicMessagesResponse, - Coroutine[Any, Any, Union[AnthropicMessagesResponse, AsyncIterator]], + Coroutine[object, object, Union[AnthropicMessagesResponse, AsyncIterator]], ]: """ LLM HTTP Handler for Anthropic Messages @@ -2449,18 +2457,18 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Mapping[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, fake_stream: bool = False, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, shared_session: Optional["ClientSession"] = None, ) -> Union[ ResponsesAPIResponse, BaseResponsesAPIStreamingIterator, - Coroutine[Any, Any, Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]], + Coroutine[object, object, Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]], ]: """ Handles responses API requests. @@ -2543,7 +2551,7 @@ class BaseLLMHTTPHandler: # Preserve the OpenAI-style request context (not sent to the provider) for streaming # hooks/metadata; the streaming iterator now consumes this to run deployment hooks # with the same info as chat, including litellm_params. - request_context: Dict[str, Any] = {"input": input} + request_context: Dict[str, object] = {"input": input} try: request_context.update(response_api_optional_request_params) except Exception: @@ -2663,12 +2671,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Mapping[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, fake_stream: bool = False, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, shared_session: Optional["ClientSession"] = None, ) -> Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]: """ @@ -2720,7 +2728,7 @@ class BaseLLMHTTPHandler: # Preserve the OpenAI-style request context (not sent to the provider) for streaming # hooks/metadata; the streaming iterator now consumes this to run deployment hooks # with the same info as chat, including litellm_params. - request_context: Dict[str, Any] = {"input": input} + request_context: Dict[str, object] = {"input": input} try: request_context.update(response_api_optional_request_params) except Exception: @@ -2847,8 +2855,8 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, @@ -2931,13 +2939,13 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union[DeleteResponseResult, Coroutine[Any, Any, DeleteResponseResult]]: + ) -> Union[DeleteResponseResult, Coroutine[object, object, DeleteResponseResult]]: """ Async version of the responses API handler. Uses async HTTP client to make requests. @@ -3021,13 +3029,13 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union[ResponsesAPIResponse, Coroutine[Any, Any, ResponsesAPIResponse]]: + ) -> Union[ResponsesAPIResponse, Coroutine[object, object, ResponsesAPIResponse]]: """ Get a response by ID Uses GET /v1/responses/{response_id} endpoint in the responses API @@ -3102,8 +3110,8 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -3183,12 +3191,12 @@ class BaseLLMHTTPHandler: include: Optional[List[str]] = None, limit: int = 20, order: Literal["asc", "desc"] = "desc", - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union[Dict, Coroutine[Any, Any, Dict]]: + ) -> Union[Dict, Coroutine[object, object, Dict]]: if _is_async: return self.async_list_responses_input_items( response_id=response_id, @@ -3269,7 +3277,7 @@ class BaseLLMHTTPHandler: include: Optional[List[str]] = None, limit: int = 20, order: Literal["asc", "desc"] = "desc", - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -3375,7 +3383,7 @@ class BaseLLMHTTPHandler: _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, - ) -> Union[OpenAIFileObject, Coroutine[Any, Any, OpenAIFileObject]]: + ) -> Union[OpenAIFileObject, Coroutine[object, object, OpenAIFileObject]]: """ Creates a file using Gemini's two-step upload process """ @@ -3789,7 +3797,7 @@ class BaseLLMHTTPHandler: client: Optional[Union["HTTPHandler", "AsyncHTTPHandler"]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, model: Optional[str] = None, - ) -> Union["LiteLLMBatch", Coroutine[Any, Any, "LiteLLMBatch"]]: + ) -> Union["LiteLLMBatch", Coroutine[object, object, "LiteLLMBatch"]]: """ Creates a batch using provider-specific batch creation process """ @@ -3901,7 +3909,7 @@ class BaseLLMHTTPHandler: client: Optional[Union["HTTPHandler", "AsyncHTTPHandler"]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, model: Optional[str] = None, - ) -> Union["LiteLLMBatch", Coroutine[Any, Any, "LiteLLMBatch"]]: + ) -> Union["LiteLLMBatch", Coroutine[object, object, "LiteLLMBatch"]]: """ Retrieve a batch using provider-specific configuration. """ @@ -4138,13 +4146,13 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union[ResponsesAPIResponse, Coroutine[Any, Any, ResponsesAPIResponse]]: + ) -> Union[ResponsesAPIResponse, Coroutine[object, object, ResponsesAPIResponse]]: """ Async version of the responses API handler. Uses async HTTP client to make requests. @@ -4218,8 +4226,8 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, @@ -4294,13 +4302,13 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union[ResponsesAPIResponse, Coroutine[Any, Any, ResponsesAPIResponse]]: + ) -> Union[ResponsesAPIResponse, Coroutine[object, object, ResponsesAPIResponse]]: """ Handler for the compact responses API. """ @@ -4393,8 +4401,8 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, @@ -4485,7 +4493,7 @@ class BaseLLMHTTPHandler: _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, - ) -> Union[OpenAIFileObject, Coroutine[Any, Any, OpenAIFileObject]]: + ) -> Union[OpenAIFileObject, Coroutine[object, object, OpenAIFileObject]]: """ Retrieve file metadata by ID """ @@ -4609,7 +4617,7 @@ class BaseLLMHTTPHandler: _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, - ) -> Union["FileDeleted", Coroutine[Any, Any, "FileDeleted"]]: + ) -> Union["FileDeleted", Coroutine[object, object, "FileDeleted"]]: """ Delete a file by ID """ @@ -4733,7 +4741,7 @@ class BaseLLMHTTPHandler: _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, - ) -> Union[List[OpenAIFileObject], Coroutine[Any, Any, List[OpenAIFileObject]]]: + ) -> Union[List[OpenAIFileObject], Coroutine[object, object, List[OpenAIFileObject]]]: """ List all files """ @@ -4857,7 +4865,7 @@ class BaseLLMHTTPHandler: _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, - ) -> Union["HttpxBinaryResponseContent", Coroutine[Any, Any, "HttpxBinaryResponseContent"]]: + ) -> Union["HttpxBinaryResponseContent", Coroutine[object, object, "HttpxBinaryResponseContent"]]: """ Retrieve file content by ID """ @@ -5008,7 +5016,7 @@ class BaseLLMHTTPHandler: return depth, max(max_loops, 1), fingerprints @staticmethod - def _has_agentic_completion_hook(logging_obj: Any) -> bool: + def _has_agentic_completion_hook(logging_obj: LiteLLMLoggingObj) -> bool: """ True if any registered callback actually overrides ``async_should_run_agentic_loop`` (the gate every agentic hook goes @@ -5039,7 +5047,7 @@ class BaseLLMHTTPHandler: @staticmethod def _check_agentic_loop_safety( - tool_calls: Any, + tool_calls: object, fingerprints: List[str], depth: int, max_loops: int, @@ -5062,7 +5070,7 @@ class BaseLLMHTTPHandler: return fingerprint @staticmethod - def _fingerprint_agentic_tools(tools: Dict) -> str: + def _fingerprint_agentic_tools(tools: object) -> str: try: return json.dumps(tools, sort_keys=True, default=str) except Exception: @@ -5081,8 +5089,8 @@ class BaseLLMHTTPHandler: fingerprints: List[str], fingerprint: str, stream: bool = False, - callback: Optional[Any] = None, - ) -> Any: + callback: Optional["CustomLogger"] = None, + ) -> Union[AnthropicMessagesResponse, AsyncIterator[object]]: from litellm.anthropic_interface import messages as anthropic_messages patch = plan.request_patch or AgenticLoopRequestPatch() @@ -5091,7 +5099,7 @@ class BaseLLMHTTPHandler: full_model_name = model if logging_obj is not None: - agentic_params = logging_obj.model_call_details.get("agentic_loop_params", {}) + agentic_params: Mapping[str, object] = logging_obj.model_call_details.get("agentic_loop_params", {}) full_model_name = cast(str, agentic_params.get("model", model)) optional_params = dict(anthropic_messages_optional_request_params) @@ -5121,7 +5129,7 @@ class BaseLLMHTTPHandler: kwargs_for_followup["max_agentic_loops"] = max_loops kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint] - response = await anthropic_messages.acreate( + response: Union[AnthropicMessagesResponse, AsyncIterator[object]] = await anthropic_messages.acreate( **{ "max_tokens": max_tokens, "messages": patch.messages, @@ -5160,8 +5168,8 @@ class BaseLLMHTTPHandler: max_loops: int, fingerprints: list[str], fingerprint: str, - callback: Any | None = None, - ) -> Any: + callback: Optional["CustomLogger"] = None, + ) -> Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]: patch = plan.request_patch or AgenticLoopRequestPatch() if patch.messages is None: raise ValueError("Agentic loop plan missing patched responses input") @@ -5192,7 +5200,7 @@ class BaseLLMHTTPHandler: kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint] try: - response = await litellm.aresponses( + response: Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator] = await litellm.aresponses( model=patch.model or model, input=patch.messages, **optional_params, @@ -5227,7 +5235,7 @@ class BaseLLMHTTPHandler: @staticmethod async def _run_agentic_loop_cleanup( - callback: Any, + callback: "CustomLogger", plan: AgenticLoopPlan, kwargs: dict, logging_obj: "LiteLLMLoggingObj", @@ -5248,10 +5256,10 @@ class BaseLLMHTTPHandler: self, result: Any, model: str, - responses_api_provider_config: Any, + responses_api_provider_config: BaseResponsesAPIConfig, logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str, - ) -> Any: + ) -> MockResponsesAPIStreamingIterator: """ Wrap a completed responses result as a synthetic stream. @@ -5330,10 +5338,10 @@ class BaseLLMHTTPHandler: def _maybe_wrap_in_fake_stream( self, - response: Any, + response: _ResponseT, logging_obj: Optional["LiteLLMLoggingObj"], api_surface: str, - ) -> Any: + ) -> Union[_ResponseT, "FakeAnthropicMessagesStreamIterator"]: """ If the original request was streaming but converted to non-streaming for WebSearch interception, wrap the dict response in a FakeAnthropicMessagesStreamIterator. @@ -5402,7 +5410,7 @@ class BaseLLMHTTPHandler: continue should_run: bool = False - tool_calls: Any = None + tool_calls: object = None try: # First: Check if agentic loop should run. Wrap in try/except # to shield from buggy user callbacks — a callback crash should @@ -5449,7 +5457,7 @@ class BaseLLMHTTPHandler: callback.__class__.async_build_agentic_loop_plan is not CustomLogger.async_build_agentic_loop_plan ) if not build_plan_overridden: - agentic_result = await callback.async_run_agentic_loop( + agentic_result: object = await callback.async_run_agentic_loop( tools=tool_calls, model=model, messages=messages, @@ -5571,7 +5579,7 @@ class BaseLLMHTTPHandler: continue should_run: bool = False - tool_calls: Any = None + tool_calls: object = None try: ( should_run, @@ -5828,7 +5836,7 @@ class BaseLLMHTTPHandler: client: Optional[Any] = None, timeout: Optional[float] = None, user_api_key_dict: Optional[Any] = None, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, query_params: Optional[RealtimeQueryParams] = None, ): import websockets @@ -5927,7 +5935,7 @@ class BaseLLMHTTPHandler: timeout: Union[float, httpx.Timeout], provider_config: Optional[Any] = None, model: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, api_version: Optional[str] = None, ) -> httpx.Response: @@ -5960,7 +5968,7 @@ class BaseLLMHTTPHandler: timeout: Union[float, httpx.Timeout], provider_config: Optional[Any] = None, model: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, api_version: Optional[str] = None, ) -> httpx.Response: @@ -5989,7 +5997,7 @@ class BaseLLMHTTPHandler: timeout: Union[float, httpx.Timeout], provider_config: Optional[Any] = None, model: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, api_version: Optional[str] = None, ) -> httpx.Response: @@ -6061,8 +6069,8 @@ class BaseLLMHTTPHandler: timeout: Union[float, httpx.Timeout], provider_config: Optional[Any] = None, model: Optional[str] = None, - session_config: Optional[Dict[str, Any]] = None, - extra_headers: Optional[Dict[str, Any]] = None, + session_config: Optional[Dict[str, object]] = None, + extra_headers: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, api_version: Optional[str] = None, ) -> httpx.Response: @@ -6144,7 +6152,7 @@ class BaseLLMHTTPHandler: api_key: Optional[str] = None, timeout: Optional[float] = None, user_api_key_dict: Optional[Any] = None, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, custom_llm_provider: Optional[str] = None, first_message: Optional[str] = None, **kwargs: Any, @@ -6324,15 +6332,15 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, fake_stream: bool = False, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, ) -> Union[ ImageResponse, - Coroutine[Any, Any, ImageResponse], + Coroutine[object, object, ImageResponse], ]: """ @@ -6444,11 +6452,11 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, fake_stream: bool = False, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, ) -> ImageResponse: """ Async version of the image edit handler. @@ -6542,16 +6550,16 @@ class BaseLLMHTTPHandler: litellm_params: Dict, logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, fake_stream: bool = False, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, api_key: Optional[str] = None, ) -> Union[ ImageResponse, - Coroutine[Any, Any, ImageResponse], + Coroutine[object, object, ImageResponse], ]: """ Handles image generation requests. @@ -6669,11 +6677,11 @@ class BaseLLMHTTPHandler: litellm_params: Dict, logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, fake_stream: bool = False, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, api_key: Optional[str] = None, ) -> ImageResponse: """ @@ -6777,16 +6785,16 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, fake_stream: bool = False, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, api_key: Optional[str] = None, ) -> Union[ VideoObject, - Coroutine[Any, Any, VideoObject], + Coroutine[object, object, VideoObject], ]: """ Handles video generation requests. @@ -6901,11 +6909,11 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, fake_stream: bool = False, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, api_key: Optional[str] = None, ) -> VideoObject: """ @@ -7001,12 +7009,12 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, api_key: Optional[str] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, variant: Optional[str] = None, - ) -> Union[bytes, Coroutine[Any, Any, bytes]]: + ) -> Union[bytes, Coroutine[object, object, bytes]]: """ Handle video content download requests. """ @@ -7091,7 +7099,7 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, api_key: Optional[str] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, variant: Optional[str] = None, @@ -7169,8 +7177,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, _is_async: bool = False, client=None, @@ -7268,8 +7276,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, client=None, api_key: Optional[str] = None, @@ -7351,7 +7359,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, _is_async: bool = False, client=None, @@ -7435,7 +7443,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, client=None, api_key: Optional[str] = None, @@ -7506,7 +7514,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, _is_async: bool = False, client=None, @@ -7575,7 +7583,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, client=None, api_key: Optional[str] = None, @@ -7634,8 +7642,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, _is_async: bool = False, client=None, @@ -7743,8 +7751,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, client=None, api_key: Optional[str] = None, @@ -7840,8 +7848,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, _is_async: bool = False, client=None, @@ -7929,8 +7937,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, client=None, api_key: Optional[str] = None, @@ -8004,8 +8012,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, _is_async: bool = False, client=None, @@ -8058,8 +8066,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, client=None, api_key: Optional[str] = None, @@ -8139,7 +8147,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, client=None, api_key: Optional[str] = None, @@ -8215,8 +8223,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, _is_async: bool = False, client=None, @@ -8320,8 +8328,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, client=None, api_key: Optional[str] = None, @@ -8410,11 +8418,11 @@ class BaseLLMHTTPHandler: container_provider_config: "BaseContainerConfig", litellm_params: GenericLiteLLMParams, logging_obj: "LiteLLMLoggingObj", - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, - ) -> Union["ContainerObject", Coroutine[Any, Any, "ContainerObject"]]: + ) -> Union["ContainerObject", Coroutine[object, object, "ContainerObject"]]: if _is_async: # Return the async coroutine if called with _is_async=True return self.async_container_create_handler( @@ -8497,7 +8505,7 @@ class BaseLLMHTTPHandler: container_provider_config: "BaseContainerConfig", litellm_params: GenericLiteLLMParams, logging_obj: "LiteLLMLoggingObj", - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> "ContainerObject": @@ -8574,12 +8582,12 @@ class BaseLLMHTTPHandler: after: Optional[str] = None, limit: Optional[int] = None, order: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, - ) -> Union["ContainerListResponse", Coroutine[Any, Any, "ContainerListResponse"]]: + ) -> Union["ContainerListResponse", Coroutine[object, object, "ContainerListResponse"]]: if _is_async: # Return the async coroutine if called with _is_async=True return self.async_container_list_handler( @@ -8664,8 +8672,8 @@ class BaseLLMHTTPHandler: after: Optional[str] = None, limit: Optional[int] = None, order: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> "ContainerListResponse": @@ -8739,12 +8747,12 @@ class BaseLLMHTTPHandler: container_provider_config: "BaseContainerConfig", litellm_params: GenericLiteLLMParams, logging_obj: "LiteLLMLoggingObj", - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, - ) -> Union["ContainerObject", Coroutine[Any, Any, "ContainerObject"]]: + ) -> Union["ContainerObject", Coroutine[object, object, "ContainerObject"]]: if _is_async: # Return the async coroutine if called with _is_async=True return self.async_container_retrieve_handler( @@ -8827,8 +8835,8 @@ class BaseLLMHTTPHandler: container_provider_config: "BaseContainerConfig", litellm_params: GenericLiteLLMParams, logging_obj: "LiteLLMLoggingObj", - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> "ContainerObject": @@ -8904,12 +8912,12 @@ class BaseLLMHTTPHandler: container_provider_config: "BaseContainerConfig", litellm_params: GenericLiteLLMParams, logging_obj: "LiteLLMLoggingObj", - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, - ) -> Union["DeleteContainerResult", Coroutine[Any, Any, "DeleteContainerResult"]]: + ) -> Union["DeleteContainerResult", Coroutine[object, object, "DeleteContainerResult"]]: if _is_async: # Return the async coroutine if called with _is_async=True return self.async_container_delete_handler( @@ -8992,8 +9000,8 @@ class BaseLLMHTTPHandler: container_provider_config: "BaseContainerConfig", litellm_params: GenericLiteLLMParams, logging_obj: "LiteLLMLoggingObj", - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> "DeleteContainerResult": @@ -9072,12 +9080,12 @@ class BaseLLMHTTPHandler: after: Optional[str] = None, limit: Optional[int] = None, order: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, - ) -> Union["ContainerFileListResponse", Coroutine[Any, Any, "ContainerFileListResponse"]]: + ) -> Union["ContainerFileListResponse", Coroutine[object, object, "ContainerFileListResponse"]]: if _is_async: return self.async_container_file_list_handler( container_id=container_id, @@ -9164,8 +9172,8 @@ class BaseLLMHTTPHandler: after: Optional[str] = None, limit: Optional[int] = None, order: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> "ContainerFileListResponse": @@ -9241,11 +9249,11 @@ class BaseLLMHTTPHandler: container_provider_config: "BaseContainerConfig", litellm_params: GenericLiteLLMParams, logging_obj: "LiteLLMLoggingObj", - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, - ) -> Union[bytes, Coroutine[Any, Any, bytes]]: + ) -> Union[bytes, Coroutine[object, object, bytes]]: if _is_async: return self.async_container_file_content_handler( container_id=container_id, @@ -9327,7 +9335,7 @@ class BaseLLMHTTPHandler: container_provider_config: "BaseContainerConfig", litellm_params: GenericLiteLLMParams, logging_obj: "LiteLLMLoggingObj", - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> bytes: @@ -9406,8 +9414,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, @@ -9504,12 +9512,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, - ) -> Union[VectorStoreSearchResponse, Coroutine[Any, Any, VectorStoreSearchResponse]]: + ) -> Union[VectorStoreSearchResponse, Coroutine[object, object, VectorStoreSearchResponse]]: if _is_async: return self.async_vector_store_search_handler( vector_store_id=vector_store_id, @@ -9598,8 +9606,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, @@ -9658,12 +9666,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, - ) -> Union[VectorStoreCreateResponse, Coroutine[Any, Any, VectorStoreCreateResponse]]: + ) -> Union[VectorStoreCreateResponse, Coroutine[object, object, VectorStoreCreateResponse]]: if _is_async: return self.async_vector_store_create_handler( vector_store_create_optional_params=vector_store_create_optional_params, @@ -9728,8 +9736,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> VectorStoreCreateResponse: @@ -9781,12 +9789,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, - ) -> Union[VectorStoreCreateResponse, Coroutine[Any, Any, VectorStoreCreateResponse]]: + ) -> Union[VectorStoreCreateResponse, Coroutine[object, object, VectorStoreCreateResponse]]: if _is_async: return self.async_vector_store_retrieve_handler( vector_store_id=vector_store_id, @@ -9848,8 +9856,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ): @@ -9912,8 +9920,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, @@ -9988,8 +9996,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> VectorStoreCreateResponse: @@ -10054,12 +10062,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, - ) -> Union[VectorStoreCreateResponse, Coroutine[Any, Any, VectorStoreCreateResponse]]: + ) -> Union[VectorStoreCreateResponse, Coroutine[object, object, VectorStoreCreateResponse]]: if _is_async: return self.async_vector_store_update_handler( vector_store_id=vector_store_id, @@ -10131,8 +10139,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ): @@ -10182,8 +10190,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, @@ -10249,8 +10257,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> VectorStoreFileObject: @@ -10314,12 +10322,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, - ) -> Union[VectorStoreFileObject, Coroutine[Any, Any, VectorStoreFileObject]]: + ) -> Union[VectorStoreFileObject, Coroutine[object, object, VectorStoreFileObject]]: if _is_async: return self.async_vector_store_file_create_handler( vector_store_id=vector_store_id, @@ -10391,8 +10399,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> VectorStoreFileListResponse: @@ -10455,12 +10463,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, - ) -> Union[VectorStoreFileListResponse, Coroutine[Any, Any, VectorStoreFileListResponse]]: + ) -> Union[VectorStoreFileListResponse, Coroutine[object, object, VectorStoreFileListResponse]]: if _is_async: return self.async_vector_store_file_list_handler( vector_store_id=vector_store_id, @@ -10531,7 +10539,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> VectorStoreFileObject: @@ -10590,11 +10598,11 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, - ) -> Union[VectorStoreFileObject, Coroutine[Any, Any, VectorStoreFileObject]]: + ) -> Union[VectorStoreFileObject, Coroutine[object, object, VectorStoreFileObject]]: if _is_async: return self.async_vector_store_file_retrieve_handler( vector_store_id=vector_store_id, @@ -10660,7 +10668,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> VectorStoreFileContentResponse: @@ -10721,13 +10729,13 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, ) -> Union[ VectorStoreFileContentResponse, - Coroutine[Any, Any, VectorStoreFileContentResponse], + Coroutine[object, object, VectorStoreFileContentResponse], ]: if _is_async: return self.async_vector_store_file_content_handler( @@ -10797,8 +10805,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> VectorStoreFileObject: @@ -10863,12 +10871,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, - ) -> Union[VectorStoreFileObject, Coroutine[Any, Any, VectorStoreFileObject]]: + ) -> Union[VectorStoreFileObject, Coroutine[object, object, VectorStoreFileObject]]: if _is_async: return self.async_vector_store_file_update_handler( vector_store_id=vector_store_id, @@ -10941,7 +10949,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> VectorStoreFileDeleteResponse: @@ -11000,13 +11008,13 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, ) -> Union[ VectorStoreFileDeleteResponse, - Coroutine[Any, Any, VectorStoreFileDeleteResponse], + Coroutine[object, object, VectorStoreFileDeleteResponse], ]: if _is_async: return self.async_vector_store_file_delete_handler( @@ -11077,13 +11085,13 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, stream: bool = False, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, system_instruction: Optional[Any] = None, ) -> Any: """ @@ -11209,12 +11217,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[AsyncHTTPHandler] = None, stream: bool = False, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, system_instruction: Optional[Any] = None, ) -> Any: """ @@ -11328,12 +11336,12 @@ class BaseLLMHTTPHandler: litellm_params: Dict, logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, ) -> Union[ "HttpxBinaryResponseContent", - Coroutine[Any, Any, "HttpxBinaryResponseContent"], + Coroutine[object, object, "HttpxBinaryResponseContent"], ]: """ Handles text-to-speech requests. @@ -11443,7 +11451,7 @@ class BaseLLMHTTPHandler: litellm_params: Dict, logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> "HttpxBinaryResponseContent": """ @@ -11575,12 +11583,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["Skill", Coroutine[Any, Any, "Skill"]]: + ) -> Union["Skill", Coroutine[object, object, "Skill"]]: """Create a skill""" if _is_async: return self.async_create_skill_handler( @@ -11641,7 +11649,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -11697,12 +11705,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["ListSkillsResponse", Coroutine[Any, Any, "ListSkillsResponse"]]: + ) -> Union["ListSkillsResponse", Coroutine[object, object, "ListSkillsResponse"]]: """List skills""" if _is_async: return self.async_list_skills_handler( @@ -11756,7 +11764,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -11802,12 +11810,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["Skill", Coroutine[Any, Any, "Skill"]]: + ) -> Union["Skill", Coroutine[object, object, "Skill"]]: """Get a skill""" if _is_async: return self.async_get_skill_handler( @@ -11858,7 +11866,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -11903,12 +11911,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["DeleteSkillResponse", Coroutine[Any, Any, "DeleteSkillResponse"]]: + ) -> Union["DeleteSkillResponse", Coroutine[object, object, "DeleteSkillResponse"]]: """Delete a skill""" if _is_async: return self.async_delete_skill_handler( @@ -11959,7 +11967,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -12009,12 +12017,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["Eval", Coroutine[Any, Any, "Eval"]]: + ) -> Union["Eval", Coroutine[object, object, "Eval"]]: """Create an eval""" if _is_async: return self.async_create_eval_handler( @@ -12068,7 +12076,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -12115,12 +12123,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["ListEvalsResponse", Coroutine[Any, Any, "ListEvalsResponse"]]: + ) -> Union["ListEvalsResponse", Coroutine[object, object, "ListEvalsResponse"]]: """List evals""" if _is_async: return self.async_list_evals_handler( @@ -12174,7 +12182,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -12220,12 +12228,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["Eval", Coroutine[Any, Any, "Eval"]]: + ) -> Union["Eval", Coroutine[object, object, "Eval"]]: """Get an eval""" if _is_async: return self.async_get_eval_handler( @@ -12276,7 +12284,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -12322,12 +12330,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["Eval", Coroutine[Any, Any, "Eval"]]: + ) -> Union["Eval", Coroutine[object, object, "Eval"]]: """Update an eval""" if _is_async: return self.async_update_eval_handler( @@ -12381,7 +12389,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -12427,12 +12435,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["DeleteEvalResponse", Coroutine[Any, Any, "DeleteEvalResponse"]]: + ) -> Union["DeleteEvalResponse", Coroutine[object, object, "DeleteEvalResponse"]]: """Delete an eval""" if _is_async: return self.async_delete_eval_handler( @@ -12483,7 +12491,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -12528,12 +12536,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["CancelEvalResponse", Coroutine[Any, Any, "CancelEvalResponse"]]: + ) -> Union["CancelEvalResponse", Coroutine[object, object, "CancelEvalResponse"]]: """Cancel an eval""" if _is_async: return self.async_cancel_eval_handler( @@ -12584,7 +12592,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -12634,12 +12642,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["Run", Coroutine[Any, Any, "Run"]]: + ) -> Union["Run", Coroutine[object, object, "Run"]]: """Create a run""" if _is_async: return self.async_create_run_handler( @@ -12693,7 +12701,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -12740,12 +12748,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["ListRunsResponse", Coroutine[Any, Any, "ListRunsResponse"]]: + ) -> Union["ListRunsResponse", Coroutine[object, object, "ListRunsResponse"]]: """List runs""" if _is_async: return self.async_list_runs_handler( @@ -12799,7 +12807,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -12845,12 +12853,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["Run", Coroutine[Any, Any, "Run"]]: + ) -> Union["Run", Coroutine[object, object, "Run"]]: """Get a run""" if _is_async: return self.async_get_run_handler( @@ -12901,7 +12909,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -12946,12 +12954,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["CancelRunResponse", Coroutine[Any, Any, "CancelRunResponse"]]: + ) -> Union["CancelRunResponse", Coroutine[object, object, "CancelRunResponse"]]: """Cancel a run""" if _is_async: return self.async_cancel_run_handler( @@ -13002,7 +13010,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -13047,12 +13055,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["RunDeleteResponse", Coroutine[Any, Any, "RunDeleteResponse"]]: + ) -> Union["RunDeleteResponse", Coroutine[object, object, "RunDeleteResponse"]]: """Delete a run""" if _is_async: return self.async_delete_run_handler( @@ -13103,7 +13111,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index 684601367b6..855bc410cea 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast from urllib.parse import quote import httpx -from httpx._types import RequestFiles +from httpx._types import FileContent, FileTypes, RequestFiles import litellm from litellm.litellm_core_utils.url_utils import encode_url_path_segment @@ -128,7 +128,7 @@ class OpenAIVideoConfig(BaseVideoConfig): # Handle input_reference parameter if provided _input_reference = video_create_optional_request_params.get("input_reference") data_without_files = {k: v for k, v in request_dict.items() if k not in ["input_reference"]} - files_list: List[Tuple[str, Any]] = [] + files_list: List[Tuple[str, FileTypes]] = [] # Handle input_reference parameter if _input_reference is not None: @@ -177,9 +177,7 @@ class OpenAIVideoConfig(BaseVideoConfig): request_data: Optional[Dict] = None, ) -> VideoObject: """Transform the OpenAI video creation response.""" - response_data = raw_response.json() - - video_obj = VideoObject(**response_data) # type: ignore[arg-type] + video_obj = VideoObject.model_validate(raw_response.json()) if custom_llm_provider and video_obj.id: video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, model) @@ -223,7 +221,7 @@ class OpenAIVideoConfig(BaseVideoConfig): url = f"{url}?variant={quote(variant, safe='')}" # No additional data needed for GET content request - data: Dict[str, Any] = {} + data: Dict[str, object] = {} return url, data @@ -274,10 +272,8 @@ class OpenAIVideoConfig(BaseVideoConfig): """ Transform the OpenAI video remix response. """ - response_data = raw_response.json() - # Transform the response data - video_obj = VideoObject(**response_data) # type: ignore[arg-type] + video_obj = VideoObject.model_validate(raw_response.json()) if custom_llm_provider and video_obj.id: video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, None) @@ -392,7 +388,7 @@ class OpenAIVideoConfig(BaseVideoConfig): url = f"{api_base.rstrip('/')}/{encoded_video_id}" # No data needed for DELETE request - data: Dict[str, Any] = {} + data: Dict[str, object] = {} return url, data @@ -404,10 +400,8 @@ class OpenAIVideoConfig(BaseVideoConfig): """ Transform the OpenAI video delete response. """ - response_data = raw_response.json() - # Transform the response data - video_obj = VideoObject(**response_data) # type: ignore[arg-type] # type: ignore[arg-type] + video_obj = VideoObject.model_validate(raw_response.json()) return video_obj @@ -429,7 +423,7 @@ class OpenAIVideoConfig(BaseVideoConfig): url = f"{api_base.rstrip('/')}/{encoded_video_id}" # No additional data needed for GET request - data: Dict[str, Any] = {} + data: Dict[str, object] = {} return url, data @@ -442,9 +436,8 @@ class OpenAIVideoConfig(BaseVideoConfig): """ Transform the OpenAI video retrieve response. """ - response_data = raw_response.json() # Transform the response data - video_obj = VideoObject(**response_data) # type: ignore[arg-type] + video_obj = VideoObject.model_validate(raw_response.json()) if custom_llm_provider and video_obj.id: video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, None) @@ -465,22 +458,22 @@ class OpenAIVideoConfig(BaseVideoConfig): def transform_video_create_character_request( self, name: str, - video: Any, + video: FileContent, api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, ) -> Tuple[str, list]: url = f"{api_base.rstrip('/')}/characters" - files_list: List[Tuple[str, Any]] = [("name", (None, name))] + files_list: List[Tuple[str, FileTypes]] = [("name", (None, name))] self._add_video_to_files(files_list, video, "video") return url, files_list def transform_video_create_character_response( self, raw_response: httpx.Response, - logging_obj: Any, + logging_obj: LiteLLMLoggingObj, ) -> CharacterObject: - return CharacterObject(**raw_response.json()) + return CharacterObject.model_validate(raw_response.json()) def transform_video_get_character_request( self, @@ -497,9 +490,9 @@ class OpenAIVideoConfig(BaseVideoConfig): def transform_video_get_character_response( self, raw_response: httpx.Response, - logging_obj: Any, + logging_obj: LiteLLMLoggingObj, ) -> CharacterObject: - return CharacterObject(**raw_response.json()) + return CharacterObject.model_validate(raw_response.json()) def transform_video_edit_request( self, @@ -508,12 +501,12 @@ class OpenAIVideoConfig(BaseVideoConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - extra_body: Optional[Dict[str, Any]] = None, - prefetched_source_data: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, object]] = None, + prefetched_source_data: Optional[Dict[str, object]] = None, ) -> Tuple[str, Dict]: original_video_id = extract_original_video_id(video_id) url = f"{api_base.rstrip('/')}/edits" - data: Dict[str, Any] = {"prompt": prompt, "video": {"id": original_video_id}} + data: Dict[str, object] = {"prompt": prompt, "video": {"id": original_video_id}} if extra_body: data.update(extra_body) return url, data @@ -521,11 +514,11 @@ class OpenAIVideoConfig(BaseVideoConfig): def transform_video_edit_response( self, raw_response: httpx.Response, - logging_obj: Any, + logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str] = None, request_data: Optional[Dict] = None, ) -> VideoObject: - video_obj = VideoObject(**raw_response.json()) + video_obj = VideoObject.model_validate(raw_response.json()) if custom_llm_provider and video_obj.id: video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, None) return video_obj @@ -538,11 +531,11 @@ class OpenAIVideoConfig(BaseVideoConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - extra_body: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, object]] = None, ) -> Tuple[str, Dict]: original_video_id = extract_original_video_id(video_id) url = f"{api_base.rstrip('/')}/extensions" - data: Dict[str, Any] = { + data: Dict[str, object] = { "prompt": prompt, "seconds": seconds, "video": {"id": original_video_id}, @@ -554,10 +547,10 @@ class OpenAIVideoConfig(BaseVideoConfig): def transform_video_extension_response( self, raw_response: httpx.Response, - logging_obj: Any, + logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str] = None, ) -> VideoObject: - video_obj = VideoObject(**raw_response.json()) + video_obj = VideoObject.model_validate(raw_response.json()) if custom_llm_provider and video_obj.id: video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, None) return video_obj @@ -578,8 +571,8 @@ class OpenAIVideoConfig(BaseVideoConfig): def _add_video_to_files( self, - files_list: List[Tuple[str, Any]], - video: Any, + files_list: List[Tuple[str, FileTypes]], + video: FileContent, field_name: str, ) -> None: """ @@ -592,7 +585,7 @@ class OpenAIVideoConfig(BaseVideoConfig): content_type = self._get_video_content_type(video=video, filename=filename) files_list.append((field_name, (filename, video, content_type))) - def _get_video_content_type(self, video: Any, filename: str) -> str: + def _get_video_content_type(self, video: FileContent, filename: str) -> str: guessed_content_type, _ = mimetypes.guess_type(filename) if guessed_content_type and guessed_content_type.startswith("video/"): return guessed_content_type diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index e3f7352e8ca..0a6d72881be 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -15,8 +15,19 @@ import re import time from collections.abc import Sequence from contextlib import asynccontextmanager -from typing import Any, AsyncIterator, Callable, Literal, Optional, Union, cast -from urllib.parse import urlparse +from typing import ( + TYPE_CHECKING, + Any, + AsyncIterator, + Callable, + Literal, + Optional, + TypeAlias, + TypedDict, + Union, + cast, +) +from urllib.parse import ParseResult, urlparse import anyio import httpx @@ -32,7 +43,7 @@ from mcp.types import ( ResourceTemplate, ) from mcp.types import Tool as MCPTool -from pydantic import AnyUrl +from pydantic import AnyUrl, BaseModel import litellm from litellm._logging import verbose_logger @@ -139,10 +150,15 @@ from litellm.proxy._types import ( from litellm.proxy.auth.ip_address_utils import IPAddressUtils from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper from litellm.proxy.common_utils.user_api_key_cache import get_management_object_ttl -from litellm.proxy.utils import ProxyLogging, get_server_root_path +from litellm.proxy.utils import PrismaClient, ProxyLogging, get_server_root_path from litellm.repositories.table_repositories import MCPServerRepository from litellm.types.llms.custom_http import httpxSpecialProvider -from litellm.types.mcp import DEFAULT_SUBJECT_TOKEN_TYPE, MCPAuth, MCPStdioConfig +from litellm.types.mcp import ( + DEFAULT_SUBJECT_TOKEN_TYPE, + MCPAuth, + MCPStdioConfig, + MCPTokenEndpointAuthMethod, +) from litellm.types.mcp_server.mcp_server_manager import ( MCPInfo, MCPOAuthMetadata, @@ -150,6 +166,14 @@ from litellm.types.mcp_server.mcp_server_manager import ( ) from litellm.types.utils import CallTypes +if TYPE_CHECKING: + from mcp.client.session import ClientSession + from mcp.shared.context import RequestContext + from mcp.types import CreateMessageRequestParams + + from litellm.caching.caching import InMemoryCache + from litellm.types.mcp_server.mcp_toolset import MCPToolset + try: from mcp.shared.tool_name_validation import ( SEP_986_URL, @@ -209,6 +233,95 @@ _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES: tuple[MCPAuth, ...] = ( _OAUTH_DISCOVERY_RETRY_BASE_SECONDS = 30.0 _OAUTH_DISCOVERY_RETRY_MAX_SECONDS = 900.0 +_StringList: TypeAlias = list[str] +_StringMap: TypeAlias = dict[str, str] +_ToolParamMap: TypeAlias = dict[str, list[str]] +_EnvVarList: TypeAlias = list[dict[str, object]] +_InMemoryCacheDict: TypeAlias = dict[str, object] +_ToolArguments: TypeAlias = dict[str, object] + + +class MCPServerConfig(TypedDict, total=False): + """Shape of a single ``mcp_servers`` entry in config.yaml, as consumed by + :meth:`MCPServerManager.load_servers_from_config`. Every key is optional: YAML supplies + whatever the admin wrote, and each read applies its own default.""" + + alias: str + description: str + mcp_info: MCPInfo + url: str + spec_path: str + transport: MCPTransportType + auth_type: MCPAuthType + authentication_token: str + auth_value: str + instructions: str + command: str + args: _StringList + env: _StringMap + client_id: str + client_secret: str + oauth2_flow: str + issuer: str + authorization_url: str + token_url: str + registration_url: str + token_endpoint_auth_method: MCPTokenEndpointAuthMethod + scopes: str | Sequence[str] + dcr_bridge: object + extra_headers: _StringList + allowed_tools: _StringList + disallowed_tools: _StringList + allowed_params: _ToolParamMap + access_groups: _StringList + static_headers: _StringMap + env_vars: _EnvVarList + allow_all_keys: bool + available_on_public_internet: bool + delegate_auth_to_upstream: bool + oauth_passthrough: bool + allow_sampling: bool + allow_elicitation: bool + aws_access_key_id: str + aws_secret_access_key: str + aws_session_token: str + aws_region_name: str + aws_service_name: str + aws_role_name: str + aws_session_name: str + token_exchange_endpoint: str + token_exchange_profile: str + audience: str + subject_token_type: str + upstream_resource: str + id_jag_resource_token_endpoint: str + id_jag_resource: str + client_private_key: str + client_private_key_id: str + client_assertion_signing_alg: str + timeout: float + max_concurrent_requests: int + + +class _ProtectedResourceMetadataPayload(TypedDict, total=False): + """The RFC 9728 protected-resource metadata document fields this gateway reads.""" + + authorization_servers: Sequence[object] + scopes_supported: Sequence[str] + scopes: Sequence[str] + + +class _AuthorizationServerMetadataPayload(TypedDict, total=False): + """The RFC 8414 / OpenID Discovery authorization-server metadata fields this gateway reads.""" + + issuer: str + authorization_endpoint: str + token_endpoint: str + registration_endpoint: str + scopes_supported: Sequence[str] + grant_types_supported: Sequence[str] + token_endpoint_auth_methods_supported: Sequence[str] + def _blank_to_none(value: str | None) -> str | None: """Collapse an absent, empty, or whitespace-only string to ``None``. @@ -968,7 +1081,7 @@ def _warn_internal_delegate_pkce_if_applicable(server: MCPServer, *, source: str ) -def _deserialize_json_dict(data: Any) -> Optional[dict[str, str]]: +def _deserialize_json_dict(data: str | _StringMap | None) -> Optional[dict[str, str]]: """ Deserialize optional JSON mappings stored in the database. @@ -1057,7 +1170,7 @@ def _normalize_mcp_server_cost_info(mcp_info: MCPInfo) -> None: mcp_info["mcp_server_cost_info"] = normalized -def _create_sampling_callback(user_api_key_auth: Optional[Any] = None): +def _create_sampling_callback(user_api_key_auth: Optional[UserAPIKeyAuth] = None): """ Create a sampling callback for MCP ClientSession. Returns a callable that handles sampling/createMessage requests from @@ -1066,7 +1179,10 @@ def _create_sampling_callback(user_api_key_auth: Optional[Any] = None): if not MCP_SAMPLING_AVAILABLE: return None - async def _sampling_callback(context, params): + async def _sampling_callback( + context: "RequestContext[ClientSession, object]", + params: "CreateMessageRequestParams", + ): import litellm from litellm.proxy._experimental.mcp_server.sampling_handler import ( handle_sampling_create_message, @@ -1309,8 +1425,9 @@ class MCPServerManager: if state is None: return True failures, attempted_at = state + backoff_multiplier: int = 2 ** max(failures - 1, 0) delay = min( - _OAUTH_DISCOVERY_RETRY_BASE_SECONDS * (2 ** max(failures - 1, 0)), + _OAUTH_DISCOVERY_RETRY_BASE_SECONDS * backoff_multiplier, _OAUTH_DISCOVERY_RETRY_MAX_SECONDS, ) return (time.monotonic() - attempted_at) >= delay @@ -1324,7 +1441,7 @@ class MCPServerManager: self._oauth_discovery_retry_state[server.server_id] = (failures + 1, time.monotonic()) def _remember_upstream_initialize_instructions(self, server: MCPServer, client: MCPClient) -> None: - raw = getattr(client, "_last_initialize_instructions", None) + raw: str | None = getattr(client, "_last_initialize_instructions", None) if raw and str(raw).strip(): self._upstream_initialize_instructions_by_server_id[server.server_id] = str(raw).strip() @@ -1430,9 +1547,10 @@ class MCPServerManager: # Track which aliases have been used to ensure only first occurrence is used used_aliases = set() - for server_name, server_config in mcp_servers_config.items(): + for server_name, raw_server_config in mcp_servers_config.items(): + server_config: MCPServerConfig = raw_server_config validate_mcp_server_name(server_name) - _mcp_info: dict[str, Any] = server_config.get("mcp_info", None) or {} + _mcp_info: MCPInfo = server_config.get("mcp_info", None) or {} # Preserve all custom fields from config while setting defaults for core fields mcp_info: MCPInfo = _mcp_info.copy() # Set default values for core fields if not present @@ -1895,7 +2013,7 @@ class MCPServerManager: mcp_server: LiteLLM_MCPServerTable, *, env_vars_are_encrypted: bool, - ) -> Optional[list[dict[str, Any]]]: + ) -> Optional[_EnvVarList]: env_vars_list = _deserialize_json_list(getattr(mcp_server, "env_vars", None)) if env_vars_are_encrypted: from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 @@ -2279,7 +2397,7 @@ class MCPServerManager: async def _get_active_submitted_mcp_server_ids_for_user( self, user_api_key_auth: UserAPIKeyAuth | None ) -> list[str]: - submitter_user_id = getattr(user_api_key_auth, "user_id", None) if user_api_key_auth else None + submitter_user_id: str | None = getattr(user_api_key_auth, "user_id", None) if user_api_key_auth else None if not submitter_user_id: return [] @@ -2551,10 +2669,10 @@ class MCPServerManager: try: from litellm.proxy.proxy_server import user_api_key_cache - in_mem = getattr(user_api_key_cache, "in_memory_cache", None) + in_mem: InMemoryCache | None = getattr(user_api_key_cache, "in_memory_cache", None) if in_mem is None: return - cache_dict = getattr(in_mem, "cache_dict", {}) + cache_dict: _InMemoryCacheDict = getattr(in_mem, "cache_dict", {}) if toolset_id is None: keys_to_remove = [k for k in cache_dict if k.startswith("toolset_")] else: @@ -2574,9 +2692,9 @@ class MCPServerManager: async def get_toolset_by_name_cached( self, - prisma_client: Any, + prisma_client: PrismaClient, toolset_name: str, - ) -> Optional[Any]: + ) -> "Optional[MCPToolset]": """Return a toolset by name, cached in ``user_api_key_cache`` (Redis-backed ``DualCache`` in production) to avoid a DB hit on every routed request. @@ -2803,7 +2921,7 @@ class MCPServerManager: and report ``unknown`` instead of a misleading ``unhealthy``. """ static_headers = server.static_headers - env_vars = getattr(server, "env_vars", None) + env_vars: _EnvVarList | None = getattr(server, "env_vars", None) if not static_headers or not env_vars: return False _global_values, user_specs = parse_admin_env_vars(env_vars) @@ -2929,7 +3047,7 @@ class MCPServerManager: """ if user_api_key_auth is None: return {} - user_id = getattr(user_api_key_auth, "user_id", None) + user_id: str | None = getattr(user_api_key_auth, "user_id", None) if not user_id: return {} @@ -2976,7 +3094,7 @@ class MCPServerManager: match await provider.resolve_credentials(to_subject(user_api_key_auth, subject_token), spec): case Ok(auth): # NoOpAuth has no header_name and so never conflicts. - header_name = getattr(auth, "header_name", None) + header_name: str | None = getattr(auth, "header_name", None) conflicts = bool( header_name and extra_headers and any(key.lower() == header_name.lower() for key in extra_headers) ) @@ -3546,7 +3664,7 @@ class MCPServerManager: self, server: MCPServer, prompt_name: str, - arguments: Optional[dict[str, Any]] = None, + arguments: Optional[dict[str, str]] = None, mcp_auth_header: Optional[Union[str, dict[str, str]]] = None, extra_headers: Optional[dict[str, str]] = None, raw_headers: Optional[dict[str, str]] = None, @@ -3606,7 +3724,7 @@ class MCPServerManager: and base_port == target_port ) - async def _fetch_oauth_discovery_url(self, url: str, server_url: str) -> Any: + async def _fetch_oauth_discovery_url(self, url: str, server_url: str) -> httpx.Response: client = get_async_httpx_client( llm_provider=httpxSpecialProvider.MCP, params={"timeout": MCP_METADATA_TIMEOUT}, @@ -3807,7 +3925,7 @@ class MCPServerManager: try: response = await self._fetch_oauth_discovery_url(resource_metadata_url, server_url) response.raise_for_status() - data = response.json() + data: _ProtectedResourceMetadataPayload = response.json() except SSRFError as exc: verbose_logger.warning( "MCP OAuth discovery: refusing to fetch resource metadata from %s " @@ -3932,7 +4050,7 @@ class MCPServerManager: try: response = await self._fetch_oauth_discovery_url(url, server_url) response.raise_for_status() - data = response.json() + data: _AuthorizationServerMetadataPayload = response.json() except SSRFError as exc: verbose_logger.warning( "MCP OAuth discovery: refusing to fetch authorization-server " @@ -3993,7 +4111,7 @@ class MCPServerManager: @staticmethod def _build_azure_authorization_server_metadata( - parsed_issuer_url: Any, + parsed_issuer_url: ParseResult, ) -> Optional[MCPOAuthMetadata]: path_parts = [part for part in (parsed_issuer_url.path or "").split("/") if part] if parsed_issuer_url.netloc not in _AZURE_ENTRA_HOSTS or len(path_parts) != 2 or path_parts[1] != "v2.0": @@ -4054,7 +4172,7 @@ class MCPServerManager: "aws_session_name": credentials_dict.get("aws_session_name"), } - def _extract_scopes(self, scopes_value: Any) -> Optional[list[str]]: + def _extract_scopes(self, scopes_value: str | Sequence[object] | None) -> Optional[list[str]]: if isinstance(scopes_value, str): scopes = [s.strip() for s in scopes_value.split() if s.strip()] return scopes or None @@ -4292,7 +4410,7 @@ class MCPServerManager: return match_known_tool_name(tool_name, server, server.allowed_tools or ()) is not None return match_known_tool_name(tool_name, server, server.disallowed_tools or ()) is None - def validate_allowed_params(self, tool_name: str, arguments: dict[str, Any], server: MCPServer) -> None: + def validate_allowed_params(self, tool_name: str, arguments: _ToolArguments, server: MCPServer) -> None: """ Filter arguments to only include allowed parameters for the given tool. @@ -4373,7 +4491,7 @@ class MCPServerManager: self, server: MCPServer, tool_name: str, - arguments: dict[str, Any], + arguments: _ToolArguments, ) -> CallToolResult: """ Call an OpenAPI tool handler directly. @@ -4537,7 +4655,7 @@ class MCPServerManager: def _create_during_hook_task( self, name: str, - arguments: dict[str, Any], + arguments: _ToolArguments, server_name_from_prefix: Optional[str], user_api_key_auth: Optional[UserAPIKeyAuth], proxy_logging_obj: ProxyLogging, @@ -4636,7 +4754,7 @@ class MCPServerManager: self, mcp_server: MCPServer, original_tool_name: str, - arguments: dict[str, Any], + arguments: _ToolArguments, tasks: list, mcp_auth_header: Optional[str], mcp_server_auth_headers: Optional[dict[str, dict[str, str]]], @@ -4990,7 +5108,7 @@ class MCPServerManager: # shadow the resolver, double-resolving and hiding the per-server challenge. return oauth2_headers - user_id = getattr(user_api_key_auth, "user_id", None) + user_id: str | None = getattr(user_api_key_auth, "user_id", None) if not user_id: return oauth2_headers @@ -5091,7 +5209,7 @@ class MCPServerManager: self, server_name: str, name: str, - arguments: dict[str, Any], + arguments: _ToolArguments, user_api_key_auth: Optional[UserAPIKeyAuth] = None, mcp_auth_header: Optional[str] = None, mcp_server_auth_headers: Optional[dict[str, dict[str, str]]] = None, @@ -5330,7 +5448,7 @@ class MCPServerManager: # Pending/rejected servers are excluded at the DB level so we never load them. from litellm.proxy._experimental.mcp_server.db import LiteLLM_MCPServerTable - raw_rows = await MCPServerRepository(prisma_client).table.find_many( + raw_rows: Sequence[BaseModel] = await MCPServerRepository(prisma_client).table.find_many( where={ "OR": [ {"approval_status": None}, @@ -5836,7 +5954,7 @@ class MCPServerManager: @staticmethod def _env_vars_to_models( - env_vars: Optional[list[dict[str, Any]]], + env_vars: Optional[_EnvVarList], ) -> Optional[list[MCPEnvVar]]: if env_vars is None: return None diff --git a/litellm/proxy/_experimental/mcp_server/sampling_handler.py b/litellm/proxy/_experimental/mcp_server/sampling_handler.py index 65630f74e90..5bad530f37b 100644 --- a/litellm/proxy/_experimental/mcp_server/sampling_handler.py +++ b/litellm/proxy/_experimental/mcp_server/sampling_handler.py @@ -10,16 +10,23 @@ MCP Spec Reference: https://modelcontextprotocol.io/specification/2025-11-25/client/sampling """ -from typing import Any, Dict, List, Optional, Union import typing +from collections.abc import Mapping, Sequence +from typing import Any, Dict, List, NamedTuple, Optional, Protocol, Union if typing.TYPE_CHECKING: + from fastapi import Request + from mcp.client.session import ClientSession + from mcp.shared.context import RequestContext + from mcp.types import ContentBlock, SamplingMessageContentBlock + + from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging -from litellm._logging import verbose_logger - from fastapi import HTTPException +from litellm._logging import verbose_logger + # Guard imports that require the mcp package try: from mcp.types import ( @@ -65,7 +72,7 @@ def _resolve_model_from_preferences( import litellm # Build list of available model names from proxy Router or litellm.model_list - available_model_names: list = [] + available_model_names: list[str] = [] try: from litellm.proxy.proxy_server import llm_router @@ -83,7 +90,7 @@ def _resolve_model_from_preferences( available_model_names.append(entry) if model_preferences and model_preferences.hints: for hint in model_preferences.hints: - hint_name = getattr(hint, "name", None) + hint_name: str | None = getattr(hint, "name", None) if not hint_name: continue # Try direct match first @@ -133,7 +140,7 @@ def _resolve_model_from_preferences( ) return available_model_names[0] # Last resort - use LiteLLM default or raise error - default_sampling_model = getattr(litellm, "default_mcp_sampling_model", None) + default_sampling_model: str | None = getattr(litellm, "default_mcp_sampling_model", None) if default_sampling_model: verbose_logger.debug( "MCP sampling model resolution: using litellm.default_mcp_sampling_model='%s'", @@ -153,6 +160,13 @@ def _has_priorities(model_preferences: "ModelPreferences") -> bool: ) +class _ScoredModel(NamedTuple): + name: str + cost: float + max_output: float + output_tps: float + + def _select_model_by_priority( model_names: List[str], model_preferences: "ModelPreferences", @@ -183,12 +197,12 @@ def _select_model_by_priority( """ import litellm as _litellm - cost_weight = getattr(model_preferences, "costPriority", None) or 0.0 - speed_weight = getattr(model_preferences, "speedPriority", None) or 0.0 - intel_weight = getattr(model_preferences, "intelligencePriority", None) or 0.0 + cost_weight: float = getattr(model_preferences, "costPriority", None) or 0.0 + speed_weight: float = getattr(model_preferences, "speedPriority", None) or 0.0 + intel_weight: float = getattr(model_preferences, "intelligencePriority", None) or 0.0 # Gather raw metrics for each model - scored: List[Dict[str, Any]] = [] + scored: list[_ScoredModel] = [] for name in model_names: try: info = _litellm.get_model_info(name) @@ -200,12 +214,12 @@ def _select_model_by_priority( max_output = info.get("max_output_tokens") or info.get("max_tokens") or 0 output_tps = info.get("output_tokens_per_second") or 0.0 scored.append( - { - "name": name, - "cost": total_cost, - "max_output": max_output, - "output_tps": output_tps, - } + _ScoredModel( + name=name, + cost=total_cost, + max_output=max_output, + output_tps=output_tps, + ) ) if not scored: @@ -222,9 +236,9 @@ def _select_model_by_priority( normed = [1.0 - n for n in normed] return normed - costs = [s["cost"] for s in scored] - max_outputs = [float(s["max_output"]) for s in scored] - output_tps_values = [s["output_tps"] for s in scored] + costs = [s.cost for s in scored] + max_outputs = [float(s.max_output) for s in scored] + output_tps_values = [s.output_tps for s in scored] # costPriority: lower cost → higher score (invert) cost_scores = _normalise(costs, invert=True) @@ -243,7 +257,7 @@ def _select_model_by_priority( score = cost_weight * cost_scores[i] + speed_weight * speed_scores[i] + intel_weight * intel_scores[i] verbose_logger.debug( "MCP priority scoring: model=%s cost_score=%.3f speed_score=%.3f intel_score=%.3f → weighted=%.3f", - entry["name"], + entry.name, cost_scores[i], speed_scores[i], intel_scores[i], @@ -251,14 +265,14 @@ def _select_model_by_priority( ) if score > best_score: best_score = score - best_name = entry["name"] + best_name = entry.name return best_name def _convert_mcp_content_to_openai( - content: Any, -) -> Union[str, Dict[str, Any], List[Dict[str, Any]]]: + content: "SamplingMessageContentBlock | Sequence[SamplingMessageContentBlock]", +) -> "str | dict[str, object] | list[dict[str, object]]": """ Convert MCP SamplingMessage content to OpenAI message content format. Handles: @@ -283,7 +297,7 @@ def _convert_mcp_content_to_openai( def _convert_single_content( content: Any, -) -> Union[Dict[str, Any], List[Dict[str, Any]]]: +) -> "dict[str, object] | list[dict[str, object]]": """Convert a single MCP content item to OpenAI format. For text/image/audio content, returns a single content-part dict. @@ -339,7 +353,7 @@ def _convert_single_content( # Marked so the message-level converter can emit it as a # separate ``{"role": "tool", ...}`` message. tool_use_id = getattr(content, "toolUseId", "") - nested_content = getattr(content, "content", []) + nested_content: Sequence[ContentBlock] = getattr(content, "content", []) if isinstance(nested_content, list): text_parts = [getattr(c, "text", str(c)) for c in nested_content if getattr(c, "type", None) == "text"] result_text = "\n".join(text_parts) if text_parts else "" @@ -358,7 +372,7 @@ def _convert_single_content( def _convert_mcp_messages_to_openai( messages: List["SamplingMessage"], system_prompt: Optional[str] = None, -) -> List[Dict[str, Any]]: +) -> "Sequence[Mapping[str, object]]": """ Convert MCP SamplingMessage list to OpenAI messages format. MCP messages use: @@ -369,7 +383,7 @@ def _convert_mcp_messages_to_openai( - role: "system" | "user" | "assistant" | "tool" - content: str | list[content_part] """ - openai_messages: List[Dict[str, Any]] = [] + openai_messages: list[Mapping[str, object]] = [] # Add system prompt if provided if system_prompt: openai_messages.append({"role": "system", "content": system_prompt}) @@ -380,7 +394,7 @@ def _convert_mcp_messages_to_openai( if role == "assistant" and _has_tool_use(content): tool_calls = _extract_tool_calls(content) if tool_calls: - openai_msg: Dict[str, Any] = { + openai_msg: dict[str, object] = { "role": "assistant", "tool_calls": tool_calls, } @@ -400,7 +414,7 @@ def _convert_mcp_messages_to_openai( # tool_use / tool_result that slipped past the fast-path checks # above (e.g. unexpected role, single non-list content). converted = _convert_mcp_content_to_openai(content) - converted_parts = ( + converted_parts: Sequence[Mapping[str, object]] = ( converted if isinstance(converted, list) else ([converted] if isinstance(converted, dict) else []) ) @@ -422,7 +436,7 @@ def _convert_mcp_messages_to_openai( # Emit assistant message with tool_calls if any were found if tool_call_markers: - openai_msg_tc: Dict[str, Any] = { + openai_msg_tc: dict[str, object] = { "role": "assistant", "tool_calls": tool_call_markers, } @@ -442,21 +456,25 @@ def _convert_mcp_messages_to_openai( return openai_messages -def _has_tool_use(content: Any) -> bool: +def _has_tool_use(content: "SamplingMessageContentBlock | Sequence[SamplingMessageContentBlock]") -> bool: """Check if content contains ToolUseContent.""" if isinstance(content, list): return any(getattr(c, "type", None) == "tool_use" for c in content) - return getattr(content, "type", None) == "tool_use" + content_type: str | None = getattr(content, "type", None) + return content_type == "tool_use" -def _has_tool_result(content: Any) -> bool: +def _has_tool_result(content: "SamplingMessageContentBlock | Sequence[SamplingMessageContentBlock]") -> bool: """Check if content contains ToolResultContent.""" if isinstance(content, list): return any(getattr(c, "type", None) == "tool_result" for c in content) - return getattr(content, "type", None) == "tool_result" + content_type: str | None = getattr(content, "type", None) + return content_type == "tool_result" -def _extract_tool_calls(content: Any) -> List[Dict[str, Any]]: +def _extract_tool_calls( + content: "SamplingMessageContentBlock | Sequence[SamplingMessageContentBlock]", +) -> "Sequence[Mapping[str, object]]": """Extract OpenAI-format tool_calls from MCP ToolUseContent.""" import json @@ -477,7 +495,9 @@ def _extract_tool_calls(content: Any) -> List[Dict[str, Any]]: return tool_calls -def _extract_text_parts(content: Any) -> Optional[str]: +def _extract_text_parts( + content: "SamplingMessageContentBlock | Sequence[SamplingMessageContentBlock]", +) -> Optional[str]: """Extract text parts from mixed content.""" items = content if isinstance(content, list) else [content] texts = [] @@ -487,7 +507,9 @@ def _extract_text_parts(content: Any) -> Optional[str]: return "\n".join(texts) if texts else None -def _extract_tool_results(content: Any) -> List[Dict[str, Any]]: +def _extract_tool_results( + content: "SamplingMessageContentBlock | Sequence[SamplingMessageContentBlock]", +) -> "Sequence[Mapping[str, object]]": """Extract OpenAI-format tool messages from MCP ToolResultContent.""" items = content if isinstance(content, list) else [content] results = [] @@ -495,7 +517,7 @@ def _extract_tool_results(content: Any) -> List[Dict[str, Any]]: if getattr(item, "type", None) == "tool_result": tool_use_id = getattr(item, "toolUseId", "") # Extract text from nested content - nested_content = getattr(item, "content", []) + nested_content: Sequence[ContentBlock] = getattr(item, "content", []) if isinstance(nested_content, list): text_parts = [getattr(c, "text", str(c)) for c in nested_content if getattr(c, "type", None) == "text"] result_text = "\n".join(text_parts) if text_parts else "" @@ -513,7 +535,7 @@ def _extract_tool_results(content: Any) -> List[Dict[str, Any]]: def _convert_mcp_tools_to_openai( tools: Optional[List["Tool"]], -) -> Optional[List[Dict[str, Any]]]: +) -> "Sequence[Mapping[str, object]] | None": """ Convert MCP Tool definitions to OpenAI function calling format. MCP Tool: {name, description, inputSchema} @@ -541,7 +563,7 @@ def _convert_mcp_tools_to_openai( def _convert_mcp_tool_choice_to_openai( tool_choice: Optional["ToolChoice"], -) -> Optional[Union[str, Dict[str, Any]]]: +) -> "str | None": """ Convert MCP ToolChoice to OpenAI tool_choice format. MCP: {mode: "auto"} | {mode: "required"} | {mode: "none"} @@ -559,8 +581,32 @@ def _convert_mcp_tool_choice_to_openai( return "auto" +class _SamplingResponseMessage(Protocol): + @property + def content(self) -> str | None: ... + + @property + def tool_calls(self) -> Sequence[object] | None: ... + + +class _SamplingResponseChoice(Protocol): + @property + def message(self) -> _SamplingResponseMessage: ... + + @property + def finish_reason(self) -> str | None: ... + + +class _SamplingCompletionResponse(Protocol): + @property + def choices(self) -> Sequence[_SamplingResponseChoice]: ... + + @property + def model(self) -> str | None: ... + + def _convert_openai_response_to_mcp_result( - response: Any, + response: _SamplingCompletionResponse, model_name: str, ) -> Union["CreateMessageResult", "CreateMessageResultWithTools", "ErrorData"]: """ @@ -593,12 +639,12 @@ def _convert_openai_response_to_mcp_result( stop_reason = "maxTokens" else: stop_reason = "endTurn" - actual_model = getattr(response, "model", model_name) or model_name + actual_model: str = getattr(response, "model", model_name) or model_name # Check if response has tool calls tool_calls = getattr(message, "tool_calls", None) if tool_calls: # Build ToolUseContent items - content_parts: "List[Any]" = [] + content_parts: list[SamplingMessageContentBlock] = [] # Include text content if present if message.content: content_parts.append(TextContent(type="text", text=message.content)) @@ -636,7 +682,7 @@ def _convert_openai_response_to_mcp_result( ) -async def _check_model_access(model: str, user_api_key_auth: Any) -> Optional["ErrorData"]: +async def _check_model_access(model: str, user_api_key_auth: "UserAPIKeyAuth | None") -> Optional["ErrorData"]: """Enforce model-permission checks for MCP sampling requests. Runs the same authorization checks as ``/chat/completions``: @@ -678,14 +724,14 @@ async def _check_model_access(model: str, user_api_key_auth: Any) -> Optional["E try: import litellm from litellm.proxy.auth.auth_checks import ( + _check_team_member_model_access, can_key_call_model, + can_project_access_model, can_team_access_model, can_user_call_model, - can_project_access_model, - _check_team_member_model_access, + get_project_object, get_team_object, get_user_object, - get_project_object, ) try: @@ -700,16 +746,20 @@ async def _check_model_access(model: str, user_api_key_auth: Any) -> Optional["E llm_router=_llm_router, ) - _team_id = getattr(user_api_key_auth, "team_id", None) - _user_id = getattr(user_api_key_auth, "user_id", None) - _project_id = getattr(user_api_key_auth, "project_id", None) + _team_id: str | None = getattr(user_api_key_auth, "team_id", None) + _user_id: str | None = getattr(user_api_key_auth, "user_id", None) + _project_id: str | None = getattr(user_api_key_auth, "project_id", None) try: from litellm.proxy.proxy_server import ( prisma_client as _prisma_client, - user_api_key_cache as _user_api_key_cache, + ) + from litellm.proxy.proxy_server import ( proxy_logging_obj as _proxy_logging_obj, ) + from litellm.proxy.proxy_server import ( + user_api_key_cache as _user_api_key_cache, + ) except ImportError: _prisma_client = None _user_api_key_cache = None # type: ignore[assignment] @@ -799,7 +849,7 @@ async def _check_model_access(model: str, user_api_key_auth: Any) -> Optional["E async def _run_budget_checks( model: str, - user_api_key_auth: Any, + user_api_key_auth: "UserAPIKeyAuth", raw_headers: Optional[Dict[str, str]] = None, client_ip: Optional[str] = None, ) -> Optional["ErrorData"]: @@ -811,25 +861,33 @@ async def _run_budget_checks( Returns None if all checks pass, or an ErrorData describing the denial. """ try: - from litellm.proxy.auth.auth_checks import common_checks - from litellm.proxy.proxy_server import ( - general_settings, - llm_router as _llm_router, - prisma_client as _prisma_client, - proxy_logging_obj as _proxy_logging_obj, - user_api_key_cache as _user_api_key_cache, - ) + import litellm from litellm.proxy.auth.auth_checks import ( + common_checks, get_team_object, get_user_object, ) - import litellm + from litellm.proxy.proxy_server import ( + general_settings, + ) + from litellm.proxy.proxy_server import ( + llm_router as _llm_router, + ) + from litellm.proxy.proxy_server import ( + prisma_client as _prisma_client, + ) + from litellm.proxy.proxy_server import ( + proxy_logging_obj as _proxy_logging_obj, + ) + from litellm.proxy.proxy_server import ( + user_api_key_cache as _user_api_key_cache, + ) except ImportError as import_err: verbose_logger.warning("MCP sampling: budget check imports unavailable: %s", import_err) return None # Can't enforce budgets without the modules - _team_id = getattr(user_api_key_auth, "team_id", None) - _user_id = getattr(user_api_key_auth, "user_id", None) + _team_id: str | None = getattr(user_api_key_auth, "team_id", None) + _user_id: str | None = getattr(user_api_key_auth, "user_id", None) team_obj = None if _team_id and _prisma_client and _user_api_key_cache: @@ -889,7 +947,7 @@ async def _run_budget_checks( # common_checks runs. _tag_max_budget_check inside common_checks only # inspects request_body; without this pre-merge, header-supplied tags # bypass per-tag budget enforcement (mirroring the regular auth path). - request_body: Dict[str, Any] = {"model": model} + request_body: dict[str, object] = {"model": model} try: from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup @@ -935,7 +993,7 @@ async def _run_budget_checks( def _build_sampling_request( raw_headers: Optional[Dict[str, str]] = None, client_ip: Optional[str] = None, -) -> Any: +) -> "Request": """Build a synthetic FastAPI Request for sampling sub-calls. Converts the original MCP connection's HTTP headers into ASGI @@ -961,7 +1019,7 @@ def _build_sampling_request( from fastapi import Request # --- Build ASGI headers --- - _scope_headers: list = [(b"content-type", b"application/json")] + _scope_headers: list[tuple[bytes, bytes]] = [(b"content-type", b"application/json")] # Hop-by-hop headers that must NOT be forwarded into the # synthetic request (they describe the original HTTP framing, # not the logical request). @@ -1001,8 +1059,8 @@ def _build_sampling_request( try: import litellm.proxy.proxy_server as proxy_server - _proxy_host = getattr(proxy_server, "server_host", None) - _proxy_port = getattr(proxy_server, "server_port", None) + _proxy_host: str | None = getattr(proxy_server, "server_host", None) + _proxy_port: str | int | None = getattr(proxy_server, "server_port", None) if _proxy_host: _server_host = str(_proxy_host) @@ -1016,7 +1074,7 @@ def _build_sampling_request( if client_ip: _client_tuple = (client_ip, 0) - scope: Dict[str, Any] = { + scope: dict[str, object] = { "type": "http", "method": "POST", "path": "/mcp/sampling/createMessage", @@ -1035,7 +1093,7 @@ def _build_sampling_request( async def _build_completion_kwargs( params: "CreateMessageRequestParams", model: str, - user_api_key_auth: Any, + user_api_key_auth: "UserAPIKeyAuth", raw_headers: Optional[Dict[str, str]], client_ip: Optional[str], ) -> Dict[str, Any]: @@ -1078,7 +1136,7 @@ async def _build_completion_kwargs( async def _run_guardrails_and_call_llm( completion_kwargs: Dict[str, Any], - user_api_key_auth: Any, + user_api_key_auth: "UserAPIKeyAuth", ) -> Any: try: from litellm.proxy.proxy_server import proxy_logging_obj as _plo @@ -1111,10 +1169,10 @@ async def _run_guardrails_and_call_llm( async def handle_sampling_create_message( - context: Any, + context: "RequestContext[ClientSession, object]", params: "CreateMessageRequestParams", default_model: Optional[str] = None, - user_api_key_auth: Optional[Any] = None, + user_api_key_auth: "UserAPIKeyAuth | None" = None, raw_headers: Optional[Dict[str, str]] = None, client_ip: Optional[str] = None, ) -> Union["CreateMessageResult", "CreateMessageResultWithTools", "ErrorData"]: @@ -1184,7 +1242,7 @@ async def handle_sampling_create_message( client_ip=client_ip, ) - openai_messages = completion_kwargs["messages"] + openai_messages: Sequence[Mapping[str, object]] = completion_kwargs["messages"] openai_tools = completion_kwargs.get("tools") verbose_logger.debug( "MCP sampling: calling litellm.acompletion with model=%s, num_messages=%d, has_tools=%s", @@ -1193,7 +1251,7 @@ async def handle_sampling_create_message( bool(openai_tools), ) - response = await _run_guardrails_and_call_llm( + response: _SamplingCompletionResponse = await _run_guardrails_and_call_llm( completion_kwargs=completion_kwargs, user_api_key_auth=user_api_key_auth, ) @@ -1214,7 +1272,6 @@ async def handle_sampling_create_message( RateLimitError, ServiceUnavailableError, ) - from litellm.proxy._types import ProxyException if isinstance( diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index da7a72e1cff..a29899a3965 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -6,8 +6,20 @@ import concurrent.futures import inspect import json import os +from collections.abc import Mapping, Sequence from datetime import datetime, timezone -from typing import Any, Dict, List, Literal, Optional, Type, TypeVar, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Dict, + List, + Literal, + Optional, + Type, + TypeVar, + Union, + cast, +) from urllib.parse import urlparse from fastapi import APIRouter, Depends, HTTPException, Request @@ -49,12 +61,44 @@ from litellm.types.guardrails import ( ToolPermissionGuardrailConfigModel, ) +if TYPE_CHECKING: + from types import CodeType + + from prisma.actions import LiteLLM_GuardrailsTableActions + from prisma.models import LiteLLM_GuardrailsTable + + from litellm.proxy.utils import PrismaClient + #### GUARDRAILS ENDPOINTS #### router = APIRouter() GUARDRAIL_REGISTRY = GuardrailRegistry() +def _guardrails_table(prisma_client: "PrismaClient") -> "LiteLLM_GuardrailsTableActions[LiteLLM_GuardrailsTable]": + table: LiteLLM_GuardrailsTableActions[LiteLLM_GuardrailsTable] = GuardrailsRepository(prisma_client).table + return table + + +async def _create_guardrail_row(prisma_client: "PrismaClient", data: Mapping[str, object]) -> "LiteLLM_GuardrailsTable": + row: LiteLLM_GuardrailsTable = await GuardrailsRepository(prisma_client).table.create(data=data) + return row + + +async def _delete_guardrail_row(prisma_client: "PrismaClient", where: Mapping[str, object]) -> None: + await GuardrailsRepository(prisma_client).table.delete(where=where) + + +async def _find_team_guardrail_rows( + prisma_client: "PrismaClient", where: Mapping[str, object] +) -> "Sequence[LiteLLM_GuardrailsTable]": + rows: Sequence[LiteLLM_GuardrailsTable] = await GuardrailsRepository(prisma_client).table.find_many( + where=where, + order={"created_at": "desc"}, + ) + return rows + + def _get_guardrails_list_response( guardrails_config: List[Dict], ) -> ListGuardrailsResponse: @@ -363,7 +407,7 @@ async def create_guardrail( # Configuration error — roll back the DB write so the guardrail isn't orphaned if prisma_client is not None: try: - await GuardrailsRepository(prisma_client).table.delete(where={"guardrail_id": guardrail_id}) + await _delete_guardrail_row(prisma_client, where={"guardrail_id": guardrail_id}) except Exception as rollback_err: verbose_proxy_logger.warning(f"Rollback failed for guardrail '{guardrail_id}': {rollback_err}") raise HTTPException( @@ -571,7 +615,7 @@ class RegisterGuardrailRequest(BaseModel): guardrail_name: str litellm_params: Dict[str, Any] # guardrail, mode, api_base required; api_key, headers, etc. optional - guardrail_info: Optional[Dict[str, Any]] = None + guardrail_info: Optional[Dict[str, object]] = None team_id: Optional[str] = None def get_litellm_params_dict(self) -> Dict[str, Any]: @@ -600,8 +644,8 @@ class GuardrailSubmissionItem(BaseModel): team_guardrail: bool = ( False # True when submitted via team (team_id set); use to distinguish team vs regular guardrails ) - litellm_params: Optional[Dict[str, Any]] = None - guardrail_info: Optional[Dict[str, Any]] = None + litellm_params: Optional[Dict[str, object]] = None + guardrail_info: Optional[Dict[str, object]] = None submitted_by_user_id: Optional[str] = None submitted_by_email: Optional[str] = None submitted_at: Optional[datetime] = None @@ -685,9 +729,7 @@ async def register_guardrail( ) try: - existing = await GuardrailsRepository(prisma_client).table.find_unique( - where={"guardrail_name": request.guardrail_name} - ) + existing = await _guardrails_table(prisma_client).find_unique(where={"guardrail_name": request.guardrail_name}) if existing is not None: raise HTTPException( status_code=400, @@ -708,7 +750,8 @@ async def register_guardrail( guardrail_info_str = safe_dumps(guardrail_info) try: - created = await GuardrailsRepository(prisma_client).table.create( + created = await _create_guardrail_row( + prisma_client, data={ "guardrail_name": request.guardrail_name, "litellm_params": litellm_params_str, @@ -718,7 +761,7 @@ async def register_guardrail( "submitted_at": now, "created_at": now, "updated_at": now, - } + }, ) return RegisterGuardrailResponse( guardrail_id=created.guardrail_id, @@ -731,7 +774,7 @@ async def register_guardrail( raise HTTPException(status_code=500, detail=str(e)) -def _parse_json_field(value: Any) -> Optional[Dict[str, Any]]: +def _parse_json_field(value: object) -> Optional[Dict[str, Any]]: if value is None: return None if isinstance(value, dict): @@ -768,7 +811,7 @@ async def _get_user_team_ids(user_api_key_dict: UserAPIKeyAuth) -> List[str]: return [t for t in user_obj.teams if t] -def _row_to_submission_item(row: Any) -> GuardrailSubmissionItem: +def _row_to_submission_item(row: "LiteLLM_GuardrailsTable") -> GuardrailSubmissionItem: from litellm.litellm_core_utils.litellm_logging import _get_masked_values guardrail_info = _parse_json_field(row.guardrail_info) or {} @@ -835,7 +878,7 @@ async def list_guardrail_submissions( ) try: - where_clause: Dict[str, Any] = {"team_id": {"not": None}} + where_clause: Dict[str, object] = {"team_id": {"not": None}} if visible_team_ids is not None: if not visible_team_ids: # Non-admin with no team memberships: nothing visible. @@ -846,10 +889,7 @@ async def list_guardrail_submissions( where_clause["team_id"] = {"in": visible_team_ids} # Single query: fetch team guardrails visible to the caller - all_team_rows = await GuardrailsRepository(prisma_client).table.find_many( - where=where_clause, - order={"created_at": "desc"}, - ) + all_team_rows = await _find_team_guardrail_rows(prisma_client, where_clause) # Derive summary counts from the full result set total = len(all_team_rows) @@ -909,7 +949,7 @@ async def get_guardrail_submission( is_admin = user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN try: - row = await GuardrailsRepository(prisma_client).table.find_unique(where={"guardrail_id": guardrail_id}) + row = await _guardrails_table(prisma_client).find_unique(where={"guardrail_id": guardrail_id}) if row is None: raise HTTPException(status_code=404, detail="Guardrail submission not found") if not is_admin: @@ -946,7 +986,7 @@ async def approve_guardrail_submission( raise HTTPException(status_code=500, detail="Prisma client not initialized") try: - row = await GuardrailsRepository(prisma_client).table.find_unique(where={"guardrail_id": guardrail_id}) + row = await _guardrails_table(prisma_client).find_unique(where={"guardrail_id": guardrail_id}) if row is None: raise HTTPException(status_code=404, detail="Guardrail submission not found") if row.status != "pending_review": @@ -956,7 +996,7 @@ async def approve_guardrail_submission( ) now = datetime.now(timezone.utc) - await GuardrailsRepository(prisma_client).table.update( + await _guardrails_table(prisma_client).update( where={"guardrail_id": guardrail_id}, data={"status": "active", "reviewed_at": now, "updated_at": now}, ) @@ -1026,7 +1066,7 @@ async def reject_guardrail_submission( raise HTTPException(status_code=500, detail="Prisma client not initialized") try: - row = await GuardrailsRepository(prisma_client).table.find_unique(where={"guardrail_id": guardrail_id}) + row = await _guardrails_table(prisma_client).find_unique(where={"guardrail_id": guardrail_id}) if row is None: raise HTTPException(status_code=404, detail="Guardrail submission not found") if row.status != "pending_review": @@ -1036,7 +1076,7 @@ async def reject_guardrail_submission( ) now = datetime.now(timezone.utc) - await GuardrailsRepository(prisma_client).table.update( + await _guardrails_table(prisma_client).update( where={"guardrail_id": guardrail_id}, data={"status": "rejected", "reviewed_at": now, "updated_at": now}, ) @@ -1886,13 +1926,13 @@ class TestCustomCodeGuardrailRequest(BaseModel): custom_code: str """The Python-like code containing the apply_guardrail function.""" - test_input: Dict[str, Any] + test_input: Dict[str, object] """The test input to pass to the guardrail. Should contain 'texts', optionally 'images', 'tools', etc.""" input_type: str = "request" """Whether this is a 'request' or 'response' input type.""" - request_data: Optional[Dict[str, Any]] = None + request_data: Optional[Dict[str, object]] = None """Optional mock request_data (model, user_id, team_id, metadata, etc.).""" @@ -1902,7 +1942,7 @@ class TestCustomCodeGuardrailResponse(BaseModel): success: bool """Whether the test executed successfully (no errors).""" - result: Optional[Dict[str, Any]] = None + result: Optional[Dict[str, object]] = None """The guardrail result: action (allow/block/modify), reason, modified_texts, etc.""" error: Optional[str] = None @@ -2006,7 +2046,7 @@ async def test_custom_code_guardrail( exec_globals = build_sandbox_globals() try: - compiled = compile_sandboxed(request.custom_code) + compiled: CodeType = compile_sandboxed(request.custom_code) exec(compiled, exec_globals) # noqa: S102 except SyntaxError as e: return TestCustomCodeGuardrailResponse( @@ -2030,7 +2070,7 @@ async def test_custom_code_guardrail( error_type="compilation", ) - apply_fn = exec_globals["apply_guardrail"] + apply_fn: object = exec_globals["apply_guardrail"] if not callable(apply_fn): return TestCustomCodeGuardrailResponse( success=False, @@ -2055,7 +2095,7 @@ async def test_custom_code_guardrail( # Step 4: Execute the function with timeout protection - def execute_guardrail(): + def execute_guardrail() -> object: return apply_fn(test_inputs, safe_request_data, request.input_type) try: diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 80d9ee21a44..1a0978c8eec 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -15,9 +15,9 @@ These are members of a Team on LiteLLM import asyncio import json import traceback -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from datetime import datetime, timezone -from typing import Any, Optional, cast +from typing import Any, cast import fastapi from fastapi import APIRouter, Depends, Header, HTTPException, Request, status @@ -28,6 +28,10 @@ from litellm._uuid import uuid from litellm.proxy._types import * from litellm.proxy.auth.auth_checks import get_team_object, get_user_object from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.user_api_key_cache import ( + object_permission_cache_key, + user_object_permission_id_cache_key, +) from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks from litellm.proxy.management_endpoints.common_daily_activity import ( DailySpendRecord, @@ -45,10 +49,6 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( generate_key_helper_fn, prepare_metadata_fields, ) -from litellm.proxy.common_utils.user_api_key_cache import ( - object_permission_cache_key, - user_object_permission_id_cache_key, -) from litellm.proxy.management_helpers.object_permission_utils import ( _set_object_permission, handle_update_object_permission_common, @@ -82,11 +82,74 @@ from litellm.types.proxy.management_endpoints.scim_v2 import ( ) if TYPE_CHECKING: + from prisma import models as prisma_models + from prisma import types as prisma_types + from prisma.actions import ( + LiteLLM_InvitationLinkActions, + LiteLLM_OrganizationMembershipActions, + LiteLLM_TeamMembershipActions, + LiteLLM_TeamTableActions, + LiteLLM_UserTableActions, + LiteLLM_VerificationTokenActions, + ) + + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.proxy_server import PrismaClient + from litellm.proxy.utils import ProxyLogging router = APIRouter() +def _user_table( + prisma_client: "PrismaClient | None", +) -> "LiteLLM_UserTableActions[prisma_models.LiteLLM_UserTable]": + user_table: LiteLLM_UserTableActions[prisma_models.LiteLLM_UserTable] = UserRepository(prisma_client).table + return user_table + + +def _team_table( + prisma_client: "PrismaClient | None", +) -> "LiteLLM_TeamTableActions[prisma_models.LiteLLM_TeamTable]": + team_table: LiteLLM_TeamTableActions[prisma_models.LiteLLM_TeamTable] = TeamRepository(prisma_client).table + return team_table + + +def _verification_token_table( + prisma_client: "PrismaClient | None", +) -> "LiteLLM_VerificationTokenActions[prisma_models.LiteLLM_VerificationToken]": + token_table: LiteLLM_VerificationTokenActions[prisma_models.LiteLLM_VerificationToken] = ( + VerificationTokenRepository(prisma_client).table + ) + return token_table + + +def _organization_membership_table( + prisma_client: "PrismaClient | None", +) -> "LiteLLM_OrganizationMembershipActions[prisma_models.LiteLLM_OrganizationMembership]": + membership_table: LiteLLM_OrganizationMembershipActions[prisma_models.LiteLLM_OrganizationMembership] = ( + OrganizationMembershipRepository(prisma_client).table + ) + return membership_table + + +def _invitation_link_table( + prisma_client: "PrismaClient | None", +) -> "LiteLLM_InvitationLinkActions[prisma_models.LiteLLM_InvitationLink]": + invitation_table: LiteLLM_InvitationLinkActions[prisma_models.LiteLLM_InvitationLink] = InvitationLinkRepository( + prisma_client + ).table + return invitation_table + + +def _team_membership_table( + prisma_client: "PrismaClient | None", +) -> "LiteLLM_TeamMembershipActions[prisma_models.LiteLLM_TeamMembership]": + team_membership_table: LiteLLM_TeamMembershipActions[prisma_models.LiteLLM_TeamMembership] = ( + TeamMembershipRepository(prisma_client).table + ) + return team_membership_table + + def _hash_password_in_dict(data: dict) -> None: """Hash password field in-place if present.""" if "password" in data and data["password"] is not None: @@ -138,7 +201,7 @@ def _update_internal_new_user_params(data_json: dict, data: NewUserRequest) -> d async def _check_duplicate_user_field( field_name: str, field_value: str | None, - prisma_client: Any, + prisma_client: "PrismaClient | None", *, case_insensitive: bool = False, label: str | None = None, @@ -177,7 +240,7 @@ async def _check_duplicate_user_field( ) -async def _check_duplicate_user_email(user_email: str | None, prisma_client: Any) -> None: +async def _check_duplicate_user_email(user_email: str | None, prisma_client: "PrismaClient | None") -> None: """ Helper function to check if a user email already exists in the database. """ @@ -190,7 +253,7 @@ async def _check_duplicate_user_email(user_email: str | None, prisma_client: Any ) -async def _check_duplicate_user_id(user_id: str | None, prisma_client: Any) -> None: +async def _check_duplicate_user_id(user_id: str | None, prisma_client: "PrismaClient | None") -> None: """ Helper function to check if a user id already exists in the database. """ @@ -724,8 +787,8 @@ _SCIM_DIRECTORY_METADATA_KEYS = frozenset( def _redact_scim_enterprise_metadata( - metadata: dict[str, Any] | None, -) -> dict[str, Any] | None: + metadata: dict[str, object] | None, +) -> dict[str, object] | None: """SCIM enterprise attributes, entitlements, and roles are persisted in user metadata so reporting can group on them, but they are directory-only fields that generic user-info endpoints must not surface; SCIM clients read them @@ -845,7 +908,7 @@ async def user_info( async def _check_user_info_v2_access( user_api_key_dict: UserAPIKeyAuth, target_user_id: str, -) -> Optional["LiteLLM_UserTable"]: +) -> "prisma_models.LiteLLM_UserTable | None": """ Check if the caller is allowed to access the target user's info. @@ -867,7 +930,7 @@ async def _check_user_info_v2_access( # Helper: fetch the target user row (reused across branches). object_permission is included so # callers can read the user's MCP/vector-store entitlements without a second round trip. async def _fetch_target_user(): - return await UserRepository(prisma_client).table.find_unique( + return await _user_table(prisma_client).find_unique( where={"user_id": target_user_id}, include={"object_permission": True} ) @@ -882,9 +945,7 @@ async def _check_user_info_v2_access( # Rule 3: Team admins can look up users in their teams if user_api_key_dict.user_id is not None: # Get caller's teams - caller_user = await UserRepository(prisma_client).table.find_unique( - where={"user_id": user_api_key_dict.user_id} - ) + caller_user = await _user_table(prisma_client).find_unique(where={"user_id": user_api_key_dict.user_id}) if caller_user is not None and caller_user.teams: # Fetch the target user ONCE, before the loop target_user = await _fetch_target_user() @@ -892,7 +953,7 @@ async def _check_user_info_v2_access( return None # Get all teams the caller belongs to - teams = await TeamRepository(prisma_client).table.find_many(where={"team_id": {"in": caller_user.teams}}) + teams = await _team_table(prisma_client).find_many(where={"team_id": {"in": caller_user.teams}}) for team in teams: team_obj = LiteLLM_TeamTable.model_validate(team.model_dump()) if _is_user_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team_obj): @@ -1160,7 +1221,7 @@ async def _schedule_user_update_audit_log( if prisma_client is None: return try: - updated_user_row = await UserRepository(prisma_client).table.find_first(where={"user_id": response["user_id"]}) + updated_user_row = await _user_table(prisma_client).find_first(where={"user_id": response["user_id"]}) if updated_user_row: user_row_typed = LiteLLM_UserTable.model_validate(updated_user_row.model_dump(exclude_none=True)) asyncio.create_task( @@ -1207,7 +1268,7 @@ def _check_user_update_authz( async def _invalidate_user_spend_counter_if_changed( - non_default_values: dict[str, Any], + non_default_values: Mapping[str, object], ) -> None: """Invalidate the cross-pod spend counter after a direct ``spend`` change. @@ -1295,13 +1356,9 @@ async def _update_single_user_helper( existing_user_row: BaseModel | None = None if user_request.user_id: - existing_user_row = await UserRepository(prisma_client).table.find_first( - where={"user_id": user_request.user_id} - ) + existing_user_row = await _user_table(prisma_client).find_first(where={"user_id": user_request.user_id}) elif user_request.user_email: - existing_user_row = await UserRepository(prisma_client).table.find_first( - where={"user_email": user_request.user_email} - ) + existing_user_row = await _user_table(prisma_client).find_first(where={"user_email": user_request.user_email}) _check_user_update_authz(user_request, user_api_key_dict, existing_user_row) @@ -1690,7 +1747,7 @@ async def bulk_user_update( detail="Only proxy admins can update all users at once.", ) # Optimized path for updating all users directly in database - all_users_in_db = await UserRepository(prisma_client).table.find_many(order={"created_at": "desc"}) + all_users_in_db = await _user_table(prisma_client).find_many(order={"created_at": "desc"}) if not all_users_in_db: raise HTTPException( @@ -1805,9 +1862,9 @@ async def bulk_user_update( async def get_user_key_counts( - prisma_client, + prisma_client: "PrismaClient | None", user_ids: list[str] | None = None, -): +) -> Mapping[str, int]: """ Helper function to get the count of keys for each user using Prisma's count method. @@ -1823,7 +1880,7 @@ async def get_user_key_counts( if not user_ids or len(user_ids) == 0: return {} - result = {} + result: dict[str, int] = {} # Get count for each user_id individually for user_id in user_ids: @@ -1876,9 +1933,9 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str async def _authorize_user_list_request( user_api_key_dict: UserAPIKeyAuth, organization_ids: str | None, - prisma_client: Any, - user_api_key_cache: Any, - proxy_logging_obj: Any, + prisma_client: "PrismaClient | None", + user_api_key_cache: "UserApiKeyCache", + proxy_logging_obj: "ProxyLogging | None", ) -> str | None: """ Authorize the /user/list request and return the (possibly scoped) organization_ids string. @@ -2016,7 +2073,7 @@ async def get_users( skip = (page - 1) * page_size # Build where conditions based on provided parameters - where_conditions: dict[str, Any] = {} + where_conditions: dict[str, object] = {} if role: where_conditions["user_role"] = role @@ -2064,7 +2121,7 @@ async def get_users( _validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None ) - users = await UserRepository(prisma_client).table.find_many( + users: Sequence[prisma_models.LiteLLM_UserTable] | None = await UserRepository(prisma_client).table.find_many( where=where_conditions, skip=skip, take=page_size, @@ -2072,7 +2129,7 @@ async def get_users( ) # Get total count of user rows - total_count = await UserRepository(prisma_client).table.count(where=where_conditions) + total_count: int = await UserRepository(prisma_client).table.count(where=where_conditions) # Get key count for each user if users is not None: @@ -2168,7 +2225,7 @@ async def delete_user( caller_admin_org_ids: set = set() if not caller_is_proxy_admin: caller_memberships = ( - await OrganizationMembershipRepository(prisma_client).table.find_many( + await _organization_membership_table(prisma_client).find_many( where={ "user_id": user_api_key_dict.user_id, "user_role": LitellmUserRoles.ORG_ADMIN.value, @@ -2188,7 +2245,7 @@ async def delete_user( # an N+1 DB call when delete_user is called with a large user_ids list. target_org_ids_by_user: dict[str, set] = {} if not caller_is_proxy_admin: - all_target_memberships = await OrganizationMembershipRepository(prisma_client).table.find_many( + all_target_memberships = await _organization_membership_table(prisma_client).find_many( where={"user_id": {"in": data.user_ids}} ) for m in all_target_memberships: @@ -2276,10 +2333,10 @@ async def delete_user( # End of Audit logging ## DELETE ASSOCIATED KEYS - await VerificationTokenRepository(prisma_client).table.delete_many(where={"user_id": {"in": data.user_ids}}) + await _verification_token_table(prisma_client).delete_many(where={"user_id": {"in": data.user_ids}}) ## DELETE ASSOCIATED INVITATION LINKS - await InvitationLinkRepository(prisma_client).table.delete_many( + await _invitation_link_table(prisma_client).delete_many( where={ "OR": [ {"user_id": {"in": data.user_ids}}, @@ -2290,13 +2347,13 @@ async def delete_user( ) ## DELETE ASSOCIATED ORGANIZATION MEMBERSHIPS - await OrganizationMembershipRepository(prisma_client).table.delete_many(where={"user_id": {"in": data.user_ids}}) + await _organization_membership_table(prisma_client).delete_many(where={"user_id": {"in": data.user_ids}}) ## DELETE ASSOCIATED TEAM MEMBERSHIPS - await TeamMembershipRepository(prisma_client).table.delete_many(where={"user_id": {"in": data.user_ids}}) + await _team_membership_table(prisma_client).delete_many(where={"user_id": {"in": data.user_ids}}) ## DELETE USERS - deleted_users = await UserRepository(prisma_client).table.delete_many(where={"user_id": {"in": data.user_ids}}) + deleted_users = await _user_table(prisma_client).delete_many(where={"user_id": {"in": data.user_ids}}) return deleted_users @@ -2348,9 +2405,9 @@ async def add_internal_user_to_organization( async def _resolve_org_filter_for_user_search( user_api_key_dict: UserAPIKeyAuth, team_id: str | None, - prisma_client: Any, - user_api_key_cache: Any, - proxy_logging_obj: Any, + prisma_client: "PrismaClient | None", + user_api_key_cache: "UserApiKeyCache", + proxy_logging_obj: "ProxyLogging | None", ) -> list[str] | None: """ Return a list of org IDs to filter by, or ``None`` for no filter. @@ -2414,9 +2471,9 @@ async def _resolve_org_filter_for_user_search( async def _resolve_team_org_filter( user_api_key_dict: UserAPIKeyAuth, team_id: str, - prisma_client: Any, - user_api_key_cache: Any, - proxy_logging_obj: Any, + prisma_client: "PrismaClient | None", + user_api_key_cache: "UserApiKeyCache", + proxy_logging_obj: "ProxyLogging | None", ) -> list[str]: """Look up the team and return its org as a filter list, or raise 403.""" from litellm.proxy.management_endpoints.common_utils import _is_user_team_admin @@ -2506,7 +2563,7 @@ async def ui_view_users( skip = (page - 1) * page_size # Build where conditions based on provided parameters - where_conditions: dict[str, Any] = {} + where_conditions: prisma_types.LiteLLM_UserTableWhereInput = {} if user_id: where_conditions["user_id"] = { @@ -2525,7 +2582,7 @@ async def ui_view_users( where_conditions["organization_memberships"] = {"some": {"organization_id": {"in": org_filter_ids}}} # Query users with pagination and filters - users: list[BaseModel] | None = await UserRepository(prisma_client).table.find_many( + users = await _user_table(prisma_client).find_many( where=where_conditions, skip=skip, take=page_size, @@ -2557,7 +2614,7 @@ async def _resolve_user_email_metadata( } if not user_ids: return {} - users = await UserRepository(prisma_client).table.find_many(where={"user_id": {"in": list(user_ids)}}) + users = await _user_table(prisma_client).find_many(where={"user_id": {"in": list(user_ids)}}) return {user.user_id: {"user_email": user.user_email, "user_alias": user.user_alias} for user in users} diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index a94a75fdfa3..b910eee7130 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -18,9 +18,9 @@ import os import re import secrets import traceback -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import datetime, timedelta, timezone -from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, cast +from typing import Any, Callable, Dict, List, Literal, Optional, Protocol, Tuple, TypeVar, cast import fastapi import yaml @@ -37,6 +37,7 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.models.credentials import CredentialItem from litellm.proxy._experimental.mcp_server.db import ( rotate_mcp_server_credentials_master_key, rotate_mcp_user_credentials_master_key, @@ -101,8 +102,9 @@ from litellm.proxy.utils import ( handle_exception_on_proxy, is_valid_api_key, ) +from litellm.repositories.base_repository import BaseRepository from litellm.repositories.budget_repository import BudgetRepository -from litellm.repositories.config_repository import ConfigRepository +from litellm.repositories.config_repository import ConfigParam, ConfigRepository from litellm.repositories.credentials_repository import CredentialsRepository from litellm.repositories.model_repository import ModelRepository from litellm.repositories.table_repositories import ( @@ -131,6 +133,68 @@ from litellm.types.utils import ( TeamUIKeyGenerationConfig, ) +_PrismaRowT = TypeVar("_PrismaRowT") +_RepositoryModelT = TypeVar("_RepositoryModelT", bound=BaseModel) + + +class _PrismaTableActions(Protocol[_PrismaRowT]): + """Typed view of the Prisma table actions a repository exposes through its untyped ``table``.""" + + async def find_unique( + self, + *, + where: Mapping[str, object], + include: Mapping[str, object] | None = None, + ) -> _PrismaRowT | None: ... + + async def find_first( + self, + *, + where: Mapping[str, object], + include: Mapping[str, object] | None = None, + ) -> _PrismaRowT | None: ... + + async def find_many( + self, + *, + where: Mapping[str, object] | None = None, + include: Mapping[str, object] | None = None, + order: Mapping[str, object] | None = None, + skip: int | None = None, + take: int | None = None, + ) -> list[_PrismaRowT]: ... + + async def count(self, *, where: Mapping[str, object] | None = None) -> int: ... + + async def create_many(self, *, data: Sequence[Mapping[str, object]]) -> int: ... + + async def update( + self, + *, + where: Mapping[str, object], + data: Mapping[str, object], + ) -> _PrismaRowT | None: ... + + +def _prisma_table( + repository: BaseRepository[_RepositoryModelT], +) -> _PrismaTableActions[_RepositoryModelT]: + return repository.table + + +def _deleted_verification_token_table( + prisma_client: PrismaClient, +) -> _PrismaTableActions[LiteLLM_DeletedVerificationToken]: + return DeletedVerificationTokenRepository(prisma_client).table + + +def _credentials_table(prisma_client: PrismaClient) -> _PrismaTableActions[CredentialItem]: + return CredentialsRepository(prisma_client).table + + +def _config_table(prisma_client: PrismaClient) -> _PrismaTableActions[ConfigParam]: + return ConfigRepository(prisma_client).table + async def _check_custom_key_allowed(custom_key_value: Optional[str]) -> None: """Raise 403 if custom API keys are disabled and a custom key was provided.""" @@ -490,7 +554,7 @@ _NON_ADMIN_SAFE_ALLOWED_ROUTES_PRESETS = frozenset({"llm_api_routes", "info_rout def _validate_caller_can_change_key_ownership( data: Optional[BaseModel], - existing_key_row: Any, + existing_key_row: LiteLLM_VerificationToken, user_api_key_dict: UserAPIKeyAuth, ) -> None: """ @@ -670,7 +734,7 @@ async def validate_team_id_used_in_service_account_request( ) # check if team_id exists in the database - team = await TeamRepository(prisma_client).table.find_unique( + team = await _prisma_table(TeamRepository(prisma_client)).find_unique( where={"team_id": team_id}, ) if team is None: @@ -1261,7 +1325,7 @@ async def _check_team_key_limits( # calculate allocated tpm/rpm limit # check if specified tpm/rpm limit is greater than allocated tpm/rpm limit - keys = await VerificationTokenRepository(prisma_client).table.find_many( + keys = await _prisma_table(VerificationTokenRepository(prisma_client)).find_many( where={"team_id": team_table.team_id}, ) # Exclude the key being updated to avoid double-counting its limits. @@ -1405,7 +1469,7 @@ async def _validate_caller_can_assign_key_org( detail="Cannot assign a key to an organization without a user_id on the caller's token", ) - user_row = await UserRepository(prisma_client).table.find_unique( + user_row = await _prisma_table(UserRepository(prisma_client)).find_unique( where={"user_id": user_api_key_dict.user_id}, include={"organization_memberships": True}, ) @@ -1443,7 +1507,7 @@ async def _check_org_key_limits( # get all organization keys # calculate allocated tpm/rpm limit # check if specified tpm/rpm limit is greater than allocated tpm/rpm limit - keys = await VerificationTokenRepository(prisma_client).table.find_many( + keys = await _prisma_table(VerificationTokenRepository(prisma_client)).find_many( where={"organization_id": org_table.organization_id}, ) # Exclude the key being updated to avoid double-counting its limits. @@ -2048,9 +2112,9 @@ async def _get_and_validate_existing_key( if token is not None: hashed_token = _hash_token_if_needed(token=token) - existing_key_row: LiteLLM_VerificationToken | None = await VerificationTokenRepository( - prisma_client - ).table.find_unique(where={"token": hashed_token}) + existing_key_row: LiteLLM_VerificationToken | None = await _prisma_table( + VerificationTokenRepository(prisma_client) + ).find_unique(where={"token": hashed_token}) if existing_key_row is None: raise ProxyException( @@ -2070,7 +2134,7 @@ async def _get_and_validate_existing_key( code=status.HTTP_400_BAD_REQUEST, ) - rows: list[LiteLLM_VerificationToken] = await VerificationTokenRepository(prisma_client).table.find_many( + rows: list[LiteLLM_VerificationToken] = await _prisma_table(VerificationTokenRepository(prisma_client)).find_many( where={"key_alias": key_alias}, take=2 ) @@ -2112,7 +2176,7 @@ async def _process_single_key_update( litellm_changed_by: Optional[str], prisma_client: Optional[PrismaClient], user_api_key_cache: UserApiKeyCache, - proxy_logging_obj: Any, + proxy_logging_obj: ProxyLogging, llm_router: Optional[Router], user_custom_key_update: Optional[Callable] = None, existing_key_row: Optional[LiteLLM_VerificationToken] = None, @@ -2265,9 +2329,9 @@ async def _process_single_key_update( async def _validate_mcp_servers_for_key_update( data: "UpdateKeyRequest", team_obj: Optional["LiteLLM_TeamTableCachedObj"], - existing_key_row: Any, - prisma_client: Any, - user_api_key_cache: Any, + existing_key_row: LiteLLM_VerificationToken, + prisma_client: PrismaClient | None, + user_api_key_cache: UserApiKeyCache, is_proxy_admin: bool, ) -> Optional[ObjectPermissionDict]: """Validate MCP servers in object_permission against the effective team.""" @@ -2302,12 +2366,12 @@ async def _validate_mcp_servers_for_key_update( async def _validate_update_key_data( data: UpdateKeyRequest, - existing_key_row: Any, + existing_key_row: LiteLLM_VerificationToken, user_api_key_dict: UserAPIKeyAuth, - llm_router: Any, + llm_router: Router | None, premium_user: bool, prisma_client: Any, - user_api_key_cache: Any, + user_api_key_cache: UserApiKeyCache, ) -> None: """Validate permissions and constraints for key update.""" # Reject NaN/±inf spend before it can reach the DB / spend counter. @@ -2939,7 +3003,7 @@ def _build_failed_team_key_update( else: error_message = str(exception) - key_info: Optional[Dict[str, Any]] = None + key_info: dict[str, object] | None = None if existing_key_row is not None: if hasattr(existing_key_row, "model_dump"): key_info = existing_key_row.model_dump() @@ -3416,7 +3480,7 @@ async def info_key_fn_v2( # Resolve key_aliases to tokens so we never pass token=None (unbounded query) tokens_to_query = list(data.keys) if data.keys else [] if data.key_aliases: - alias_rows = await VerificationTokenRepository(prisma_client).table.find_many( + alias_rows = await _prisma_table(VerificationTokenRepository(prisma_client)).find_many( where={"key_alias": {"in": data.key_aliases}}, include={"litellm_budget_table": True}, ) @@ -4088,7 +4152,7 @@ def _transform_verification_tokens_to_deleted_records( keys: List[LiteLLM_VerificationToken], user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: Optional[str] = None, -) -> List[Dict[str, Any]]: +) -> list[dict[str, object]]: """Transform verification tokens into deleted token records ready for persistence.""" if not keys: return [] @@ -4141,13 +4205,13 @@ def _transform_verification_tokens_to_deleted_records( async def _save_deleted_verification_token_records( - records: List[Dict[str, Any]], + records: Sequence[Mapping[str, object]], prisma_client: PrismaClient, ) -> None: """Save deleted verification token records to the database.""" if not records: return - await DeletedVerificationTokenRepository(prisma_client).table.create_many(data=records) + await _deleted_verification_token_table(prisma_client).create_many(data=records) async def _persist_deleted_verification_tokens( @@ -4175,7 +4239,7 @@ async def delete_key_aliases( user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: Optional[str] = None, ) -> Tuple[Optional[Dict], List[LiteLLM_VerificationToken]]: - _keys_being_deleted = await VerificationTokenRepository(prisma_client).table.find_many( + _keys_being_deleted = await _prisma_table(VerificationTokenRepository(prisma_client)).find_many( where={"key_alias": {"in": key_aliases}} ) @@ -4212,7 +4276,7 @@ async def _rotate_master_key( from litellm.proxy.proxy_server import proxy_config try: - models: Optional[List] = await ModelRepository(prisma_client).table.find_many() + models: Optional[List] = await _prisma_table(ModelRepository(prisma_client)).find_many() except Exception: models = None # 2. process model table @@ -4242,7 +4306,7 @@ async def _rotate_master_key( ) # 3. process config table try: - config = await ConfigRepository(prisma_client).table.find_many() + config = await _config_table(prisma_client).find_many() except Exception: config = None @@ -4263,7 +4327,7 @@ async def _rotate_master_key( ) if encrypted_env_vars: - await ConfigRepository(prisma_client).table.update( + await _config_table(prisma_client).update( where={"param_name": "environment_variables"}, data={"param_value": prisma.Json(encrypted_env_vars)}, # type: ignore[attr-defined] ) @@ -4307,7 +4371,7 @@ async def _rotate_master_key( # 5. process credentials table try: - credentials = await CredentialsRepository(prisma_client).table.find_many() + credentials = await _credentials_table(prisma_client).find_many() except Exception: credentials = None if credentials: @@ -4330,7 +4394,7 @@ async def _rotate_master_key( _cred_data["credential_info"] = prisma.Json( # type: ignore[attr-defined] _cred_data["credential_info"] ) - await CredentialsRepository(prisma_client).table.update( + await _credentials_table(prisma_client).update( where={"credential_name": cred.credential_name}, data={ **_cred_data, @@ -4772,7 +4836,7 @@ async def regenerate_key_fn( else: hashed_api_key = hash_token(key) - _key_in_db = await VerificationTokenRepository(prisma_client).table.find_unique( + _key_in_db = await _prisma_table(VerificationTokenRepository(prisma_client)).find_unique( where={"token": hashed_api_key}, ) if _key_in_db is None: @@ -4976,7 +5040,7 @@ async def reset_key_spend_fn( else: hashed_api_key = hash_token(key) - _key_in_db = await VerificationTokenRepository(prisma_client).table.find_unique( + _key_in_db = await _prisma_table(VerificationTokenRepository(prisma_client)).find_unique( where={"token": hashed_api_key}, include={"litellm_budget_table": True}, ) @@ -4996,7 +5060,7 @@ async def reset_key_spend_fn( user_api_key_cache=user_api_key_cache, ) - updated_key = await VerificationTokenRepository(prisma_client).table.update( + updated_key = await _prisma_table(VerificationTokenRepository(prisma_client)).update( where={"token": hashed_api_key}, data={"spend": reset_to}, ) @@ -5067,7 +5131,7 @@ async def validate_key_list_check( param="user_id", code=status.HTTP_403_FORBIDDEN, ) - complete_user_info_db_obj: Optional[BaseModel] = await UserRepository(prisma_client).table.find_unique( + complete_user_info_db_obj: Optional[BaseModel] = await _prisma_table(UserRepository(prisma_client)).find_unique( where={"user_id": user_api_key_dict.user_id}, include={"organization_memberships": True}, ) @@ -5421,8 +5485,8 @@ async def list_keys( async def _apply_non_admin_alias_scope( user_api_key_dict: UserAPIKeyAuth, - prisma_client: Any, - query_params: List[Any], + prisma_client: PrismaClient, + query_params: list[object], where_parts: List[str], ) -> None: """Append SQL scope conditions so non-admin users only see aliases for @@ -5435,7 +5499,9 @@ async def _apply_non_admin_alias_scope( # Look up the user's teams from the user table user_teams: List[str] = [] if user_api_key_dict.user_id: - user_row = await UserRepository(prisma_client).table.find_unique(where={"user_id": user_api_key_dict.user_id}) + user_row = await _prisma_table(UserRepository(prisma_client)).find_unique( + where={"user_id": user_api_key_dict.user_id} + ) if user_row is not None: user_teams = getattr(user_row, "teams", []) or [] @@ -5493,7 +5559,7 @@ async def key_aliases( # support column-level SELECT projection on find_many. # # $1 is always UI_SESSION_TOKEN_TEAM_ID (filters out UI session tokens). - query_params: List[Any] = [UI_SESSION_TOKEN_TEAM_ID] + query_params: list[object] = [UI_SESSION_TOKEN_TEAM_ID] where_parts = [ "key_alias IS NOT NULL", "key_alias != ''", @@ -5601,7 +5667,7 @@ def _validate_sort_params(sort_by: Optional[str], sort_order: str) -> Optional[D return order_by -def _build_expires_where_clause(expires_filter: str, now: datetime) -> dict[str, Any]: +def _build_expires_where_clause(expires_filter: str, now: datetime) -> dict[str, object]: if expires_filter == "expired": return {"AND": [{"expires": {"not": None}}, {"expires": {"lt": now}}]} return {"OR": [{"expires": None}, {"expires": {"gte": now}}]} @@ -5848,11 +5914,11 @@ async def _list_key_helper( # Get total count of keys if use_deleted_table: - total_count = await DeletedVerificationTokenRepository(prisma_client).table.count( + total_count = await _deleted_verification_token_table(prisma_client).count( where=where # type: ignore ) else: - total_count = await VerificationTokenRepository(prisma_client).table.count( + total_count = await _prisma_table(VerificationTokenRepository(prisma_client)).count( where=where # type: ignore ) @@ -5931,8 +5997,8 @@ def _get_condition_to_filter_out_ui_session_tokens() -> Dict[str, Any]: async def _check_key_admin_access( user_api_key_dict: UserAPIKeyAuth, - hashed_token: str, - prisma_client: Any, + hashed_token: str | None, + prisma_client: PrismaClient | None, user_api_key_cache: UserApiKeyCache, route: str, ) -> None: @@ -5951,7 +6017,9 @@ async def _check_key_admin_access( return # Look up the target key to find its team - target_key_row = await VerificationTokenRepository(prisma_client).table.find_unique(where={"token": hashed_token}) + target_key_row = await _prisma_table(VerificationTokenRepository(prisma_client)).find_unique( + where={"token": hashed_token} + ) if target_key_row is None: raise HTTPException( status_code=404, @@ -6047,7 +6115,9 @@ async def block_key( ) # Check if the key exists before trying to block it - existing_record = await VerificationTokenRepository(prisma_client).table.find_unique(where={"token": hashed_token}) + existing_record = await _prisma_table(VerificationTokenRepository(prisma_client)).find_unique( + where={"token": hashed_token} + ) if existing_record is None: raise ProxyException( message="Key not found.", @@ -6077,7 +6147,7 @@ async def block_key( ) ) - record = await VerificationTokenRepository(prisma_client).table.update( + record = await _prisma_table(VerificationTokenRepository(prisma_client)).update( where={"token": hashed_token}, data={"blocked": True}, # type: ignore ) @@ -6158,7 +6228,9 @@ async def unblock_key( ) # Check if the key exists before trying to unblock it - existing_record = await VerificationTokenRepository(prisma_client).table.find_unique(where={"token": hashed_token}) + existing_record = await _prisma_table(VerificationTokenRepository(prisma_client)).find_unique( + where={"token": hashed_token} + ) if existing_record is None: raise ProxyException( message="Key not found.", @@ -6188,7 +6260,7 @@ async def unblock_key( ) ) - record = await VerificationTokenRepository(prisma_client).table.update( + record = await _prisma_table(VerificationTokenRepository(prisma_client)).update( where={"token": hashed_token}, data={"blocked": False}, # type: ignore ) @@ -6443,7 +6515,7 @@ def _validate_key_alias_format(key_alias: Optional[str]) -> None: async def _enforce_unique_key_alias( key_alias: Optional[str], - prisma_client: Any, + prisma_client: PrismaClient | None, existing_key_token: Optional[str] = None, ) -> None: """ @@ -6459,12 +6531,12 @@ async def _enforce_unique_key_alias( ProxyException: If key alias already exists on a different key """ if key_alias is not None and prisma_client is not None: - where_clause: dict[str, Any] = {"key_alias": key_alias} + where_clause: dict[str, object] = {"key_alias": key_alias} if existing_key_token: # Exclude the current key from the uniqueness check where_clause["NOT"] = {"token": existing_key_token} - existing_key = await VerificationTokenRepository(prisma_client).table.find_first(where=where_clause) + existing_key = await _prisma_table(VerificationTokenRepository(prisma_client)).find_first(where=where_clause) if existing_key is not None: raise ProxyException( message=f"Key with alias '{key_alias}' already exists. Unique key aliases across all keys are required.", diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 41904e3883b..4885ec42578 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -16,7 +16,7 @@ import json from collections.abc import Mapping, Sequence from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union, cast -from fastapi import APIRouter, Depends, HTTPException, Header, Request, status +from fastapi import APIRouter, Depends, Header, HTTPException, Request, status from pydantic import BaseModel, ConfigDict, Field from litellm._logging import verbose_proxy_logger @@ -39,6 +39,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.management_endpoints.common_utils import _is_user_team_admin from litellm.proxy.management_endpoints.team_endpoints import ( _refresh_cached_team, @@ -49,18 +50,18 @@ from litellm.proxy.management_endpoints.team_endpoints import ( update_team as _legacy_update_team, ) from litellm.proxy.management_helpers.audit_logs import create_object_audit_log -from litellm.proxy.utils import PrismaClient +from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.repositories.model_repository import ModelRepository from litellm.repositories.table_repositories import ModelTableRepository from litellm.repositories.team_repository import TeamRepository from litellm.router import Router -from litellm.types.proxy.management_endpoints.model_management_endpoints import ( - UpdateUsefulLinksRequest, -) from litellm.router_utils.auto_router_model_naming import ( STRATEGY_ROUTER_PARAM_FIELDS, validate_strategy_router_model_write, ) +from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateUsefulLinksRequest, +) from litellm.types.router import ( SPECIAL_MODEL_INFO_PARAMS, Deployment, @@ -843,8 +844,8 @@ async def _get_team_public_model_names( async def _remove_unbacked_team_models( model_params: Deployment, prisma_client: PrismaClient, - user_api_key_cache: Any, - proxy_logging_obj: Any, + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging, llm_router: Router | None = None, ) -> None: """ @@ -904,7 +905,7 @@ async def _remove_unbacked_team_models( if existing_team_row is None: return - updated_team_row = await prisma_client.db.litellm_teamtable.update( + updated_team_row: LiteLLM_TeamTable = await prisma_client.db.litellm_teamtable.update( where={"team_id": team_id}, data={"models": [model for model in existing_team_row.models if model not in names_to_remove]}, include={"object_permission": True}, # type: ignore diff --git a/litellm/proxy/management_endpoints/scim/scim_v2.py b/litellm/proxy/management_endpoints/scim/scim_v2.py index e44299d018d..f1f71bf6458 100644 --- a/litellm/proxy/management_endpoints/scim/scim_v2.py +++ b/litellm/proxy/management_endpoints/scim/scim_v2.py @@ -7,7 +7,18 @@ This is an enterprise feature and requires a premium license. import re from collections.abc import Mapping, Sequence from itertools import chain -from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Set, Tuple +from typing import ( + TYPE_CHECKING, + Dict, + Iterable, + List, + NamedTuple, + Optional, + Protocol, + Set, + Tuple, + overload, +) from fastapi import ( APIRouter, @@ -69,13 +80,95 @@ from litellm.repositories.verification_token_repository import ( ) from litellm.types.proxy.management_endpoints.scim_v2 import * +if TYPE_CHECKING: + from prisma.models import LiteLLM_VerificationToken as PrismaVerificationToken + + +class _UserTableClient(Protocol): + async def find_first(self, where: Mapping[str, object]) -> LiteLLM_UserTable | None: ... + + async def find_unique(self, where: Mapping[str, object]) -> LiteLLM_UserTable | None: ... + + async def find_many( + self, + where: Mapping[str, object] | None = None, + skip: int | None = None, + take: int | None = None, + order: Mapping[str, str] | None = None, + ) -> Sequence[LiteLLM_UserTable]: ... + + async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> LiteLLM_UserTable: ... + + async def delete(self, where: Mapping[str, object]) -> LiteLLM_UserTable | None: ... + + async def count(self, where: Mapping[str, object] | None = None) -> int: ... + + +class _TeamTableClient(Protocol): + async def find_unique(self, where: Mapping[str, object]) -> LiteLLM_TeamTable | None: ... + + async def find_many( + self, + where: Mapping[str, object] | None = None, + skip: int | None = None, + take: int | None = None, + order: Mapping[str, str] | None = None, + ) -> Sequence[LiteLLM_TeamTable]: ... + + async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> LiteLLM_TeamTable: ... + + async def delete(self, where: Mapping[str, object]) -> LiteLLM_TeamTable | None: ... + + async def count(self, where: Mapping[str, object] | None = None) -> int: ... + + +class _VerificationTokenTableClient(Protocol): + async def find_many(self, where: Mapping[str, object] | None = None) -> "Sequence[PrismaVerificationToken]": ... + + async def update( + self, where: Mapping[str, object], data: Mapping[str, object] + ) -> "PrismaVerificationToken | None": ... + + +class _UserReferencingTableClient(Protocol): + async def delete_many(self, where: Mapping[str, object]) -> int: ... + + +@overload +def _table(repository: UserRepository) -> _UserTableClient: ... + + +@overload +def _table(repository: TeamRepository) -> _TeamTableClient: ... + + +@overload +def _table(repository: VerificationTokenRepository) -> _VerificationTokenTableClient: ... + + +@overload +def _table( + repository: InvitationLinkRepository | OrganizationMembershipRepository | TeamMembershipRepository, +) -> _UserReferencingTableClient: ... + + +def _table( + repository: UserRepository + | TeamRepository + | VerificationTokenRepository + | InvitationLinkRepository + | OrganizationMembershipRepository + | TeamMembershipRepository, +) -> object: + return repository.table + class UserProvisionerHelpers: """Helper methods for user provisioning operations.""" @staticmethod async def handle_existing_user_by_email( - prisma_client, + prisma_client: PrismaClient, new_user_request: NewUserRequest, admin_group: Optional[str] = None, ) -> Optional[SCIMUser]: @@ -97,7 +190,7 @@ class UserProvisionerHelpers: if not new_user_request.user_email: return None - existing_user = await UserRepository(prisma_client).table.find_first( + existing_user = await _table(UserRepository(prisma_client)).find_first( where={"user_email": new_user_request.user_email} ) @@ -107,7 +200,7 @@ class UserProvisionerHelpers: new_teams = list(dict.fromkeys(new_user_request.teams or [])) if new_user_request.user_id != existing_user.user_id: - await UserRepository(prisma_client).table.update( + await _table(UserRepository(prisma_client)).update( where={"user_id": existing_user.user_id}, data={"user_id": new_user_request.user_id}, ) @@ -119,7 +212,7 @@ class UserProvisionerHelpers: raise_on_error=True, ) - updated_user = await UserRepository(prisma_client).table.update( + updated_user = await _table(UserRepository(prisma_client)).update( where={"user_id": new_user_request.user_id}, data={ "user_email": new_user_request.user_email, @@ -177,11 +270,11 @@ async def _get_prisma_client_or_raise_exception(): return prisma_client -async def _check_user_exists(user_id: str): +async def _check_user_exists(user_id: str) -> LiteLLM_UserTable: """Check if user exists and return user, raise 404 if not found.""" prisma_client = await _get_prisma_client_or_raise_exception() - user = await UserRepository(prisma_client).table.find_unique(where={"user_id": user_id}) + user = await _table(UserRepository(prisma_client)).find_unique(where={"user_id": user_id}) if not user: raise HTTPException(status_code=404, detail={"error": f"User not found with ID: {user_id}"}) @@ -189,11 +282,11 @@ async def _check_user_exists(user_id: str): return user -async def _check_team_exists(team_id: str): +async def _check_team_exists(team_id: str) -> LiteLLM_TeamTable: """Check if team exists and return team, raise 404 if not found.""" prisma_client = await _get_prisma_client_or_raise_exception() - team = await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id}) + team = await _table(TeamRepository(prisma_client)).find_unique(where={"team_id": team_id}) if not team: raise HTTPException(status_code=404, detail={"error": f"Group not found with ID: {team_id}"}) @@ -236,9 +329,9 @@ def _build_scim_metadata( enterprise: Optional[SCIMEnterpriseUser] = None, entitlements: list[SCIMMultiValuedAttribute] | None = None, roles: list[SCIMMultiValuedAttribute] | None = None, -) -> Dict[str, Any]: +) -> dict[str, object]: """Build metadata dictionary with SCIM data.""" - metadata: Dict[str, Any] = { + metadata: dict[str, object] = { "scim_metadata": LiteLLM_UserScimMetadata( givenName=given_name, familyName=family_name, @@ -338,13 +431,15 @@ def _resolve_scim_user_role( return default_role -async def _scim_groups_from_team_ids(prisma_client: Any, team_ids: list[str]) -> list[SCIMUserGroup]: +async def _scim_groups_from_team_ids(prisma_client: PrismaClient, team_ids: list[str]) -> list[SCIMUserGroup]: """ Build SCIMUserGroup objects from team ids, populating display from each team's alias so admin-group matching by display name works the same way it does on PUT (where SCIM groups carry display names natively). """ - teams = [await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id}) for team_id in team_ids] + teams = [ + await _table(TeamRepository(prisma_client)).find_unique(where={"team_id": team_id}) for team_id in team_ids + ] return [ SCIMUserGroup( value=team_id, @@ -354,7 +449,7 @@ async def _scim_groups_from_team_ids(prisma_client: Any, team_ids: list[str]) -> ] -async def _recompute_scim_member_roles(prisma_client: Any, user_ids: Iterable[str]) -> None: +async def _recompute_scim_member_roles(prisma_client: PrismaClient, user_ids: Iterable[str]) -> None: """ Recompute and persist each user's global proxy role from their resulting team membership. No-op unless scim_admin_group is configured, so a SCIM group write @@ -367,7 +462,7 @@ async def _recompute_scim_member_roles(prisma_client: Any, user_ids: Iterable[st default_role = _default_scim_user_role() for user_id in user_ids: - user = await UserRepository(prisma_client).table.find_unique(where={"user_id": user_id}) + user = await _table(UserRepository(prisma_client)).find_unique(where={"user_id": user_id}) if user is None: continue resolved_role = _resolve_scim_user_role( @@ -375,7 +470,7 @@ async def _recompute_scim_member_roles(prisma_client: Any, user_ids: Iterable[st admin_group, default_role, ) - await UserRepository(prisma_client).table.update( + await _table(UserRepository(prisma_client)).update( where={"user_id": user_id}, data={"user_role": resolved_role}, ) @@ -471,7 +566,7 @@ async def _classify_group_member(member: SCIMMember, prisma_client: PrismaClient if member_type == "group": return _SkippedGroupMember(value=value, reason="nested_group") - user = await UserRepository(prisma_client).table.find_unique(where={"user_id": value}) + user = await _table(UserRepository(prisma_client)).find_unique(where={"user_id": value}) if user is not None: return _ResolvedUserMember(user_id=value) @@ -479,7 +574,7 @@ async def _classify_group_member(member: SCIMMember, prisma_client: PrismaClient return _SkippedGroupMember(value=value, reason="non_user_type") if member_type is None: - team = await TeamRepository(prisma_client).table.find_unique(where={"team_id": value}) + team = await _table(TeamRepository(prisma_client)).find_unique(where={"team_id": value}) if team is not None and _team_metadata_has_scim_provenance(team.metadata): return _SkippedGroupMember(value=value, reason="existing_team") @@ -619,7 +714,7 @@ async def _get_team_members_display(member_ids: List[str]) -> List[SCIMMember]: members: List[SCIMMember] = [] for member_id in member_ids: - user = await UserRepository(prisma_client).table.find_unique(where={"user_id": member_id}) + user = await _table(UserRepository(prisma_client)).find_unique(where={"user_id": member_id}) if user: display_name = user.user_email or user.user_id members.append(SCIMMember(value=user.user_id, display=display_name, type="User")) @@ -652,7 +747,7 @@ async def _handle_team_membership_changes( SCIM_BLOCKED_METADATA_KEY = "scim_blocked" -def _key_was_scim_blocked(metadata: Any) -> bool: +def _key_was_scim_blocked(metadata: object) -> bool: """True if a verification token carries the SCIM-block marker in metadata.""" return isinstance(metadata, dict) and metadata.get(SCIM_BLOCKED_METADATA_KEY) is True @@ -676,7 +771,7 @@ async def _set_user_keys_blocked(user_id: str, blocked: bool) -> int: # `blocked` is a nullable column with no default, so existing rows # typically hold NULL; treat NULL as "not blocked" since SQL equality # on NULL would otherwise silently skip them. - candidates = await VerificationTokenRepository(prisma_client).table.find_many( + candidates = await _table(VerificationTokenRepository(prisma_client)).find_many( where={ "user_id": user_id, "OR": [{"blocked": False}, {"blocked": None}], @@ -684,7 +779,7 @@ async def _set_user_keys_blocked(user_id: str, blocked: bool) -> int: ) affected_keys = candidates else: - candidates = await VerificationTokenRepository(prisma_client).table.find_many( + candidates = await _table(VerificationTokenRepository(prisma_client)).find_many( where={"user_id": user_id, "blocked": True}, ) affected_keys = [k for k in candidates if _key_was_scim_blocked(k.metadata)] @@ -693,12 +788,12 @@ async def _set_user_keys_blocked(user_id: str, blocked: bool) -> int: return 0 for key_row in affected_keys: - current_metadata: Dict[str, Any] = dict(key_row.metadata) if isinstance(key_row.metadata, dict) else {} + current_metadata: dict[str, object] = dict(key_row.metadata) if isinstance(key_row.metadata, dict) else {} if blocked: new_metadata = {**current_metadata, SCIM_BLOCKED_METADATA_KEY: True} else: new_metadata = {k: v for k, v in current_metadata.items() if k != SCIM_BLOCKED_METADATA_KEY} - await VerificationTokenRepository(prisma_client).table.update( + await _table(VerificationTokenRepository(prisma_client)).update( where={"token": key_row.token}, data={"blocked": blocked, "metadata": safe_dumps(new_metadata)}, ) @@ -719,14 +814,14 @@ async def _set_user_keys_blocked(user_id: str, blocked: bool) -> int: return len(affected_keys) -async def _delete_rows_referencing_user(prisma_client: Any, *, user_id: str) -> None: +async def _delete_rows_referencing_user(prisma_client: PrismaClient, *, user_id: str) -> None: """Drop rows whose foreign keys reference ``LiteLLM_UserTable.user_id``. Required before deleting the user row itself, otherwise Postgres rejects the user delete with an FK constraint violation (e.g. ``LiteLLM_InvitationLink_user_id_fkey``). """ - await InvitationLinkRepository(prisma_client).table.delete_many( + await _table(InvitationLinkRepository(prisma_client)).delete_many( where={ "OR": [ {"user_id": user_id}, @@ -735,11 +830,11 @@ async def _delete_rows_referencing_user(prisma_client: Any, *, user_id: str) -> ] } ) - await OrganizationMembershipRepository(prisma_client).table.delete_many(where={"user_id": user_id}) - await TeamMembershipRepository(prisma_client).table.delete_many(where={"user_id": user_id}) + await _table(OrganizationMembershipRepository(prisma_client)).delete_many(where={"user_id": user_id}) + await _table(TeamMembershipRepository(prisma_client)).delete_many(where={"user_id": user_id}) -def _scim_active_value(metadata: Optional[Dict[str, Any]]) -> Optional[bool]: +def _scim_active_value(metadata: Optional[Mapping[str, object]]) -> Optional[bool]: """Read the SCIM active flag from a user's metadata dict, if present.""" if not metadata: return None @@ -749,6 +844,12 @@ def _scim_active_value(metadata: Optional[Dict[str, Any]]) -> Optional[bool]: return bool(value) +def _user_scim_active(user: LiteLLM_UserTable) -> Optional[bool]: + """Read the SCIM active flag off a user row's metadata, if present.""" + metadata: dict[str, object] | None = user.metadata + return _scim_active_value(metadata) + + async def _create_user_if_not_exists(user_id: str, created_via: str = "scim_group") -> Optional[NewUserResponse]: """ Helper function to create a user if they don't exist. @@ -820,7 +921,7 @@ async def set_scim_content_type(response: Response): response.headers["Content-Type"] = "application/scim+json" -def _get_resource_types(base_url: str = "/scim/v2") -> list: +def _get_resource_types(base_url: str = "/scim/v2") -> Sequence[SCIMResourceType]: """Return the list of SCIM ResourceType definitions per RFC 7643 Section 6.""" return [ SCIMResourceType( @@ -848,7 +949,7 @@ def _get_resource_types(base_url: str = "/scim/v2") -> list: ] -def _get_schemas() -> list: +def _get_schemas() -> Sequence[SCIMSchema]: """Return the list of SCIM Schema definitions per RFC 7643 Section 7.""" return [ SCIMSchema( @@ -1241,7 +1342,7 @@ async def get_users( try: prisma_client = await _get_prisma_client_or_raise_exception() # Parse filter if provided (basic support) - where_conditions: Dict[str, Any] = {} + where_conditions: dict[str, object] = {} if filter: # Okta locates users by userName before deprovisioning. LiteLLM # exposes SCIM userName from user_email, while older SCIM-created @@ -1258,7 +1359,7 @@ async def get_users( where_conditions["user_email"] = filter_value # Get users from database - users: List[LiteLLM_UserTable] = await UserRepository(prisma_client).table.find_many( + users: Sequence[LiteLLM_UserTable] = await _table(UserRepository(prisma_client)).find_many( where=where_conditions, skip=(startIndex - 1), take=count, @@ -1266,7 +1367,7 @@ async def get_users( ) # Get total count for pagination - total_count = await UserRepository(prisma_client).table.count(where=where_conditions) + total_count = await _table(UserRepository(prisma_client)).count(where=where_conditions) # Convert to SCIM format scim_users: List[SCIMUser] = [] @@ -1330,7 +1431,7 @@ async def create_user( # Check if user already exists if user.userName: - existing_user = await UserRepository(prisma_client).table.find_unique(where={"user_id": user.userName}) + existing_user = await _table(UserRepository(prisma_client)).find_unique(where={"user_id": user.userName}) if existing_user: raise HTTPException( status_code=409, @@ -1406,7 +1507,7 @@ async def update_user( prisma_client = await _get_prisma_client_or_raise_exception() existing_user = await _check_user_exists(user_id) - prev_active = _scim_active_value(existing_user.metadata) + prev_active = _user_scim_active(existing_user) user_data = _extract_scim_user_data(user) @@ -1447,7 +1548,7 @@ async def update_user( user.groups or [], admin_group, _default_scim_user_role() ) - updated_user = await UserRepository(prisma_client).table.update( + updated_user = await _table(UserRepository(prisma_client)).update( where={"user_id": user_id}, data=update_data, ) @@ -1483,19 +1584,20 @@ async def delete_user( existing_user = await _check_user_exists(user_id) # Get teams user belongs to - teams = [] - if existing_user.teams: - for team_id in existing_user.teams: - team = await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id}) - if team: - teams.append(team) + found_teams = tuple( + [ + await _table(TeamRepository(prisma_client)).find_unique(where={"team_id": team_id}) + for team_id in existing_user.teams or [] + ] + ) + teams = tuple(team for team in found_teams if team) # Remove user from all teams for team in teams: - current_members = team.members or [] + current_members: Sequence[str] = team.members or [] if user_id in current_members: new_members = [m for m in current_members if m != user_id] - await TeamRepository(prisma_client).table.update( + await _table(TeamRepository(prisma_client)).update( where={"team_id": team.team_id}, data={"members": new_members} ) @@ -1511,7 +1613,7 @@ async def delete_user( await _delete_rows_referencing_user(prisma_client, user_id=user_id) # Delete user - await UserRepository(prisma_client).table.delete(where={"user_id": user_id}) + await _table(UserRepository(prisma_client)).delete(where={"user_id": user_id}) return Response(status_code=204) except Exception as e: @@ -1581,7 +1683,7 @@ def _extract_ids_from_path_filter(path: str | None, attribute: str) -> List[str] return [extracted] if extracted else [] -def _handle_displayname_update(op_type: str, value: Any, update_data: Dict[str, Any]) -> None: +def _handle_displayname_update(op_type: str, value: object, update_data: dict[str, object]) -> None: """Handle displayname updates.""" if op_type == "remove": update_data["user_alias"] = None @@ -1589,7 +1691,7 @@ def _handle_displayname_update(op_type: str, value: Any, update_data: Dict[str, update_data["user_alias"] = str(value) -def _handle_externalid_update(op_type: str, value: Any, update_data: Dict[str, Any]) -> None: +def _handle_externalid_update(op_type: str, value: object, update_data: dict[str, object]) -> None: """Handle externalid updates.""" if op_type == "remove": update_data["sso_user_id"] = None @@ -1597,7 +1699,7 @@ def _handle_externalid_update(op_type: str, value: Any, update_data: Dict[str, A update_data["sso_user_id"] = str(value) -def _handle_active_update(op_type: str, value: Any, metadata: Dict[str, Any]) -> None: +def _handle_active_update(op_type: str, value: object, metadata: dict[str, object]) -> None: """Handle active status updates.""" if op_type == "remove": metadata.pop("scim_active", None) @@ -1610,7 +1712,7 @@ def _handle_active_update(op_type: str, value: Any, metadata: Dict[str, Any]) -> metadata["scim_active"] = bool_val -def _handle_name_update(path: str, op_type: str, value: Any, scim_metadata: Dict[str, Any]) -> None: +def _handle_name_update(path: str, op_type: str, value: object, scim_metadata: dict[str, object]) -> None: """Handle name field updates (givenName, familyName).""" if path == "name.givenname": if op_type == "remove": @@ -1624,7 +1726,7 @@ def _handle_name_update(path: str, op_type: str, value: Any, scim_metadata: Dict scim_metadata["familyName"] = str(value) -def _handle_group_operations(op_type: str, value: Any, teams_set: Set[str], path: str | None) -> Set[str] | None: +def _handle_group_operations(op_type: str, value: object, teams_set: Set[str], path: str | None) -> Set[str] | None: """Handle group/team membership operations.""" group_values = _extract_group_values(value) if not group_values and value is None: @@ -1644,7 +1746,7 @@ def _multi_valued_attribute_base(path: str) -> str: return path.split("[", 1)[0].split(".", 1)[0] -def _handle_multi_valued_attribute_update(path: str, op_type: str, value: Any, metadata: dict[str, Any]) -> None: +def _handle_multi_valued_attribute_update(path: str, op_type: str, value: object, metadata: dict[str, object]) -> None: """Handle add/replace/remove for the entitlements and roles multi-valued attributes.""" base = _multi_valued_attribute_base(path) metadata_key = SCIM_MULTI_VALUED_ATTRIBUTE_METADATA_KEYS[base] @@ -1681,7 +1783,7 @@ def _handle_multi_valued_attribute_update(path: str, op_type: str, value: Any, m metadata[metadata_key] = dumped -def _handle_generic_metadata(path: str, op_type: str, value: Any, metadata: Dict[str, Any]) -> None: +def _handle_generic_metadata(path: str, op_type: str, value: object, metadata: dict[str, object]) -> None: """Handle generic metadata operations for unknown paths.""" if op_type == "remove": metadata.pop(path, None) @@ -1692,9 +1794,9 @@ def _handle_generic_metadata(path: str, op_type: str, value: Any, metadata: Dict def _apply_patch_ops( existing_user: LiteLLM_UserTable, patch_ops: SCIMPatchOp, -) -> Tuple[Dict[str, Any], Set[str]]: +) -> Tuple[dict[str, object], Set[str]]: """Apply patch operations and return update data and final team set.""" - update_data: Dict[str, Any] = {} + update_data: dict[str, object] = {} metadata = existing_user.metadata or {} scim_metadata = metadata.get("scim_metadata", {}) @@ -1843,14 +1945,15 @@ async def patch_user( prisma_client = await _get_prisma_client_or_raise_exception() existing_user = await _check_user_exists(user_id) - prev_active = _scim_active_value(existing_user.metadata) + prev_active = _user_scim_active(existing_user) update_data, final_team_set = _apply_patch_ops( existing_user=existing_user, patch_ops=patch_ops, ) - new_active = _scim_active_value(update_data.get("metadata")) + patched_metadata = update_data.get("metadata") + new_active = _scim_active_value(patched_metadata if isinstance(patched_metadata, Mapping) else None) # Handle team membership changes await _handle_team_membership_changes( @@ -1875,7 +1978,7 @@ async def patch_user( update_data["metadata"] = safe_dumps(update_data["metadata"]) - updated_user = await UserRepository(prisma_client).table.update( + updated_user = await _table(UserRepository(prisma_client)).update( where={"user_id": user_id}, data=update_data, ) @@ -1891,6 +1994,12 @@ async def patch_user( raise handle_exception_on_proxy(e) +class _TeamWhereConditions(TypedDict, total=False): + """The team columns SCIM GET /Groups can filter on, as Prisma where-conditions.""" + + team_alias: str + + # Group Endpoints @scim_router.get( "/Groups", @@ -1915,7 +2024,7 @@ async def get_groups( try: prisma_client = await _get_prisma_client_or_raise_exception() # Parse filter if provided (basic support) - where_conditions = {} + where_conditions: _TeamWhereConditions = {} if filter: # Very basic filter support - only handling displayName eq if "displayName eq" in filter: @@ -1923,7 +2032,7 @@ async def get_groups( where_conditions["team_alias"] = team_alias # Get teams from database - teams = await TeamRepository(prisma_client).table.find_many( + teams = await _table(TeamRepository(prisma_client)).find_many( where=where_conditions, skip=(startIndex - 1), take=count, @@ -1931,10 +2040,10 @@ async def get_groups( ) # Get total count for pagination - total_count = await TeamRepository(prisma_client).table.count(where=where_conditions) + total_count = await _table(TeamRepository(prisma_client)).count(where=where_conditions) # Convert to SCIM format - scim_groups = [] + scim_groups: List[SCIMGroup] = [] for team in teams: # Get team members with display names. members_with_roles is the # source of truth; the legacy `members` column is not populated by @@ -2018,7 +2127,7 @@ async def create_group( team_id = group.id or group.externalId or str(uuid.uuid4()) # Check if team already exists - existing_team = await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id}) + existing_team = await _table(TeamRepository(prisma_client)).find_unique(where={"team_id": team_id}) if existing_team: raise HTTPException( @@ -2091,7 +2200,7 @@ async def update_group( } # Update team in database - updated_team = await TeamRepository(prisma_client).table.update( + updated_team = await _table(TeamRepository(prisma_client)).update( where={"team_id": group_id}, data=update_data, ) @@ -2145,19 +2254,19 @@ async def delete_group( # For each member, remove this team from their teams list for member_id in member_ids: - user = await UserRepository(prisma_client).table.find_unique(where={"user_id": member_id}) + user = await _table(UserRepository(prisma_client)).find_unique(where={"user_id": member_id}) if user: current_teams = user.teams or [] if group_id in current_teams: new_teams = [t for t in current_teams if t != group_id] - await UserRepository(prisma_client).table.update( + await _table(UserRepository(prisma_client)).update( where={"user_id": member_id}, data={"teams": new_teams} ) await _recompute_scim_member_roles(prisma_client, member_ids) # Delete team - await TeamRepository(prisma_client).table.delete(where={"team_id": group_id}) + await _table(TeamRepository(prisma_client)).delete(where={"team_id": group_id}) return Response(status_code=204) @@ -2166,8 +2275,8 @@ async def delete_group( async def _process_group_patch_operations( - patch_ops: SCIMPatchOp, existing_team, prisma_client -) -> Tuple[Dict[str, Any], Set[str], Set[str] | None]: + patch_ops: SCIMPatchOp, existing_team: LiteLLM_TeamTable, prisma_client: PrismaClient +) -> Tuple[dict[str, object], Set[str], Set[str] | None]: """Process patch operations for a group and return update data, final members and, when the request contained a member ``replace`` op, the absolute target roster it declared (``None`` otherwise). @@ -2183,7 +2292,7 @@ async def _process_group_patch_operations( have admitted - the phantom users this endpoint used to create for nested groups - impossible to clean up. """ - update_data: Dict[str, Any] = {} + update_data: dict[str, object] = {} # Create a fresh copy of existing metadata to avoid Prisma issues metadata = {**(existing_team.metadata or {}), SCIM_MANAGED_TEAM_METADATA_KEY: True} @@ -2251,7 +2360,7 @@ async def _process_group_patch_operations( return update_data, final_members, replace_target -async def _apply_group_patch_updates(group_id: str, update_data: Dict[str, Any], prisma_client): +async def _apply_group_patch_updates(group_id: str, update_data: dict[str, object], prisma_client: PrismaClient): """Apply the group's metadata/displayName patch updates to the database. Membership itself is not written here; it is reconciled onto the source of @@ -2330,7 +2439,7 @@ async def patch_group( # Apply the metadata/displayName updates to the database updated_team = await _apply_group_patch_updates(group_id, update_data, prisma_client) - refreshed_team = await TeamRepository(prisma_client).table.find_unique(where={"team_id": group_id}) + refreshed_team = await _table(TeamRepository(prisma_client)).find_unique(where={"team_id": group_id}) refreshed_current = ( set( await _get_team_member_user_ids_from_team(LiteLLM_TeamTable.model_validate(refreshed_team.model_dump())) @@ -2356,7 +2465,7 @@ async def patch_group( ) # Refresh team one more time to get final state after membership changes - final_team = await TeamRepository(prisma_client).table.find_unique(where={"team_id": group_id}) + final_team = await _table(TeamRepository(prisma_client)).find_unique(where={"team_id": group_id}) if final_team: updated_team = final_team diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index fa01f43d049..95c89c24851 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -14,7 +14,19 @@ import json import math import traceback from datetime import datetime, timezone -from typing import Annotated, Any, Dict, List, Mapping, Optional, Tuple, Union, cast +from typing import ( + Annotated, + Dict, + List, + Mapping, + Optional, + Protocol, + Sequence, + Tuple, + TypeVar, + Union, + cast, +) import fastapi from fastapi import APIRouter, Depends, Header, HTTPException, Request, status @@ -30,11 +42,14 @@ from litellm.proxy._types import ( BlockTeamRequest, CommonProxyErrors, DeleteTeamRequest, + LiteLLM_AccessGroupTable, LiteLLM_AuditLogs, + LiteLLM_BudgetTableFull, LiteLLM_DeletedTeamTable, LiteLLM_ManagementEndpoint_MetadataFields, LiteLLM_ManagementEndpoint_MetadataFields_Premium, LiteLLM_ModelTable, + LiteLLM_OrganizationMembershipTable, LiteLLM_OrganizationTable, LiteLLM_OrganizationTableWithMembers, LiteLLM_TeamMembership, @@ -78,6 +93,7 @@ from litellm.proxy.auth.auth_checks import ( from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars from litellm.proxy.common_utils.json_merge_patch import apply_json_merge_patch +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.management_endpoints.common_utils import ( _check_passthrough_routes_caller_permission, _is_user_org_admin_for_team, @@ -106,7 +122,7 @@ from litellm.proxy.management_helpers.utils import ( add_new_member, management_endpoint_wrapper, ) -from litellm.proxy.utils import PrismaClient, handle_exception_on_proxy +from litellm.proxy.utils import PrismaClient, ProxyLogging, handle_exception_on_proxy from litellm.repositories.budget_repository import BudgetRepository from litellm.repositories.organization_repository import OrganizationRepository from litellm.repositories.table_repositories import ( @@ -141,6 +157,127 @@ from litellm.types.proxy.management_endpoints.team_endpoints import ( router = APIRouter() +_DbRecordT = TypeVar("_DbRecordT") + + +class _PrismaTableActions(Protocol[_DbRecordT]): + async def find_unique( + self, + where: Mapping[str, object], + include: Mapping[str, bool] | None = None, + ) -> _DbRecordT | None: ... + + async def find_first( + self, + where: Mapping[str, object] | None = None, + order: Mapping[str, str] | None = None, + ) -> _DbRecordT | None: ... + + async def find_many( + self, + where: Mapping[str, object] | None = None, + include: Mapping[str, bool] | None = None, + order: Mapping[str, str] | None = None, + skip: int | None = None, + take: int | None = None, + cursor: Mapping[str, object] | None = None, + ) -> list[_DbRecordT]: ... + + async def create( + self, + data: Mapping[str, object], + include: Mapping[str, bool] | None = None, + ) -> _DbRecordT: ... + + async def create_many( + self, + data: Sequence[Mapping[str, object]], + skip_duplicates: bool | None = None, + ) -> int: ... + + async def update( + self, + where: Mapping[str, object], + data: Mapping[str, object], + include: Mapping[str, bool] | None = None, + ) -> _DbRecordT: ... + + async def update_many( + self, + where: Mapping[str, object], + data: Mapping[str, object], + ) -> int: ... + + async def upsert( + self, + where: Mapping[str, object], + data: Mapping[str, Mapping[str, object]], + ) -> _DbRecordT: ... + + async def delete_many( + self, + where: Mapping[str, object] | None = None, + ) -> int: ... + + async def count( + self, + where: Mapping[str, object] | None = None, + ) -> int: ... + + +def _team_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_TeamTable]": + team_table: _PrismaTableActions[LiteLLM_TeamTable] = TeamRepository(prisma_client).table + return team_table + + +def _team_membership_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_TeamMembership]": + membership_table: _PrismaTableActions[LiteLLM_TeamMembership] = TeamMembershipRepository(prisma_client).table + return membership_table + + +def _user_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_UserTable]": + user_table: _PrismaTableActions[LiteLLM_UserTable] = UserRepository(prisma_client).table + return user_table + + +def _model_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_ModelTable]": + model_table: _PrismaTableActions[LiteLLM_ModelTable] = ModelTableRepository(prisma_client).table + return model_table + + +def _org_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_OrganizationTable]": + org_table: _PrismaTableActions[LiteLLM_OrganizationTable] = OrganizationRepository(prisma_client).table + return org_table + + +def _org_membership_db( + prisma_client: PrismaClient | None, +) -> "_PrismaTableActions[LiteLLM_OrganizationMembershipTable]": + org_membership_table: _PrismaTableActions[LiteLLM_OrganizationMembershipTable] = OrganizationMembershipRepository( + prisma_client + ).table + return org_membership_table + + +def _budget_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_BudgetTableFull]": + budget_table: _PrismaTableActions[LiteLLM_BudgetTableFull] = BudgetRepository(prisma_client).table + return budget_table + + +def _deleted_team_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_DeletedTeamTable]": + deleted_team_table: _PrismaTableActions[LiteLLM_DeletedTeamTable] = DeletedTeamRepository(prisma_client).table + return deleted_team_table + + +def _access_group_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_AccessGroupTable]": + access_group_table: _PrismaTableActions[LiteLLM_AccessGroupTable] = AccessGroupRepository(prisma_client).table + return access_group_table + + +def _tokens_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_VerificationToken]": + tokens_table: _PrismaTableActions[LiteLLM_VerificationToken] = VerificationTokenRepository(prisma_client).table + return tokens_table + def _sanitize_for_log(value: object) -> str: """Strip CR/LF from user-controlled values to prevent log injection.""" @@ -152,9 +289,9 @@ def _sanitize_for_log(value: object) -> str: async def _refresh_cached_team( - team_row: Any, - user_api_key_cache: Any, - proxy_logging_obj: Any, + team_row: LiteLLM_TeamTable, + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging, ) -> None: """ Refresh the in-memory cached team object after a DB write. @@ -396,7 +533,7 @@ class TeamMemberBudgetHandler: @staticmethod async def backfill_team_member_budget_entries( team_id: str, - members_with_roles: List[Union[Member, dict]], + members_with_roles: Sequence[Union[Member, dict[str, object]]], team_member_budget_id: str, prisma_client: PrismaClient, ) -> None: @@ -415,7 +552,7 @@ class TeamMemberBudgetHandler: return # Batch-fetch existing memberships for this team (avoids N+1 queries) - existing_memberships = await TeamMembershipRepository(prisma_client).table.find_many(where={"team_id": team_id}) + existing_memberships = await _team_membership_db(prisma_client).find_many(where={"team_id": team_id}) existing_user_ids = {m.user_id for m in existing_memberships} # Identify members with no existing membership row. @@ -448,7 +585,7 @@ class TeamMemberBudgetHandler: # Heal existing membership rows that predate the team_member_budget # configuration: populate budget_id where it is currently NULL. # Rows with an explicit budget_id (per-member override) are left alone. - updated = await TeamMembershipRepository(prisma_client).table.update_many( + updated = await _team_membership_db(prisma_client).update_many( where={"team_id": team_id, "budget_id": None}, data={"budget_id": team_member_budget_id}, ) @@ -461,7 +598,7 @@ class TeamMemberBudgetHandler: ) -def _get_default_team_param(field: str) -> Any: +def _get_default_team_param(field: str) -> object: """ Returns a default value for the given field from litellm.default_team_params config. Returns None if no default is configured. @@ -504,7 +641,7 @@ async def get_all_team_memberships( # else: # where_obj = {"user_id": str(user_id), "team_id": {"in": team_id}} - team_memberships = await TeamMembershipRepository(prisma_client).table.find_many( + team_memberships = await _team_membership_db(prisma_client).find_many( where=where_obj, include={"litellm_budget_table": True}, ) @@ -766,7 +903,7 @@ async def _check_org_team_limits( # calculate allocated tpm/rpm limit # check if specified tpm/rpm limit is greater than allocated tpm/rpm limit - teams = await TeamRepository(prisma_client).table.find_many( + teams = await _team_db(prisma_client).find_many( where={"organization_id": org_table.organization_id}, ) @@ -791,7 +928,7 @@ async def _check_user_team_limits( data: Union[NewTeamRequest, UpdateTeamRequest], user_api_key_dict: UserAPIKeyAuth, prisma_client: PrismaClient, - user_api_key_cache: Any, + user_api_key_cache: UserApiKeyCache, ) -> None: """ Enforce the caller's personal limits when CREATING a standalone team. @@ -1052,7 +1189,7 @@ async def new_team( ) # Check if license is over limit - total_teams = await TeamRepository(prisma_client).table.count() + total_teams = await _team_db(prisma_client).count() if total_teams and _license_check.is_team_count_over_limit(team_count=total_teams): raise HTTPException( status_code=403, @@ -1154,7 +1291,7 @@ async def new_team( created_by=user_api_key_dict.user_id or litellm_proxy_admin_name, updated_by=user_api_key_dict.user_id or litellm_proxy_admin_name, ) - model_dict = await ModelTableRepository(prisma_client).table.create( + model_dict = await _model_db(prisma_client).create( {**litellm_modeltable.json(exclude_none=True)} # type: ignore ) # type: ignore @@ -1358,11 +1495,11 @@ async def _create_team_update_audit_log( async def _update_model_table( data: UpdateTeamRequest, - model_id: Optional[str], + model_id: Optional[int], prisma_client: PrismaClient, user_api_key_dict: UserAPIKeyAuth, litellm_proxy_admin_name: str, -) -> Optional[str]: +) -> Optional[int]: """ Upsert model table and return the model id """ @@ -1375,11 +1512,11 @@ async def _update_model_table( updated_by=user_api_key_dict.user_id or litellm_proxy_admin_name, ) if model_id is None: - model_dict = await ModelTableRepository(prisma_client).table.create( + model_dict = await _model_db(prisma_client).create( data={**litellm_modeltable.json(exclude_none=True)} # type: ignore ) else: - model_dict = await ModelTableRepository(prisma_client).table.upsert( + model_dict = await _model_db(prisma_client).upsert( where={"id": model_id}, data={ "update": {**litellm_modeltable.json(exclude_none=True)}, # type: ignore @@ -1395,7 +1532,7 @@ async def _update_model_table( async def _auto_add_team_members_to_organization( team: LiteLLM_TeamTable, organization: LiteLLM_OrganizationTableWithMembers, - prisma_client: Any, + prisma_client: PrismaClient, ) -> None: """ When moving a team to an org, ensure all team members are also org members. @@ -1433,11 +1570,11 @@ async def _auto_add_team_members_to_organization( async def fetch_and_validate_organization( organization_id: str, - existing_team_row: Any, + existing_team_row: LiteLLM_TeamTable, llm_router: Optional[Router], - prisma_client: Any, + prisma_client: PrismaClient, user_api_key_dict: Optional[UserAPIKeyAuth] = None, -) -> Any: +) -> LiteLLM_OrganizationTable: """ Fetch and validate an organization for team update operations. @@ -1456,7 +1593,7 @@ async def fetch_and_validate_organization( if llm_router is None: raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value}) - organization_row = await OrganizationRepository(prisma_client).table.find_unique( + organization_row = await _org_db(prisma_client).find_unique( where={"organization_id": organization_id}, include={"litellm_budget_table": True, "members": True, "teams": True}, ) @@ -1758,7 +1895,7 @@ async def update_team( ): # Is the caller org_admin of the destination org? caller_memberships = ( - await OrganizationMembershipRepository(prisma_client).table.find_many( + await _org_membership_db(prisma_client).find_many( where={ "user_id": user_api_key_dict.user_id, "organization_id": data.organization_id, @@ -2005,7 +2142,7 @@ async def patch_team( patch_fields = data.model_dump(exclude_unset=True, exclude={"team_id"}) if "metadata" in patch_fields: - existing_team_row = await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id}) + existing_team_row = await _team_db(prisma_client).find_unique(where={"team_id": team_id}) if existing_team_row is None: raise HTTPException( status_code=404, @@ -2487,7 +2624,7 @@ async def _validate_and_populate_member_user_info( # Case 2: Only user_email provided - populate user_id from DB if member.user_email is not None and member.user_id is None: - user_by_email = await UserRepository(prisma_client).table.find_first( + user_by_email = await _user_db(prisma_client).find_first( where={"user_email": {"equals": member.user_email, "mode": "insensitive"}} ) @@ -2516,7 +2653,7 @@ async def _validate_and_populate_member_user_info( # Case 3: Only user_id provided - populate user_email from DB if user exists if member.user_id is not None and member.user_email is None: - user_by_id = await UserRepository(prisma_client).table.find_unique(where={"user_id": member.user_id}) + user_by_id = await _user_db(prisma_client).find_unique(where={"user_id": member.user_id}) if user_by_id is None: # User doesn't exist yet - allow it to pass with user_email as None @@ -2707,7 +2844,7 @@ async def team_member_delete( detail={"error": "Either user_id or user_email needs to be passed in"}, ) - _existing_team_row = await TeamRepository(prisma_client).table.find_unique(where={"team_id": data.team_id}) + _existing_team_row = await _team_db(prisma_client).find_unique(where={"team_id": data.team_id}) if _existing_team_row is None: raise HTTPException( @@ -2745,7 +2882,7 @@ async def team_member_delete( _db_new_team_members: List[dict] = [m.model_dump() for m in new_team_members] - _ = await TeamRepository(prisma_client).table.update( + _ = await _team_db(prisma_client).update( where={ "team_id": data.team_id, }, @@ -2835,7 +2972,7 @@ _MEMBER_BUDGET_PATCH_FIELDS = { } -def _build_member_budget_patch(data: TeamMemberUpdateRequest) -> Dict[str, Any]: +def _build_member_budget_patch(data: TeamMemberUpdateRequest) -> Dict[str, object]: """Map the budget fields the request actually set (merge-patch: a sent value updates, an explicit null clears, an absent field is left untouched) to their budget-table columns.""" @@ -2911,7 +3048,7 @@ async def team_member_update( _validate_budget_duration(data.budget_duration) - _existing_team_row = await TeamRepository(prisma_client).table.find_unique(where={"team_id": data.team_id}) + _existing_team_row = await _team_db(prisma_client).find_unique(where={"team_id": data.team_id}) if _existing_team_row is None: raise HTTPException( @@ -3007,7 +3144,7 @@ async def team_member_update( team_table.members_with_roles = team_members _db_team_members: List[dict] = [m.model_dump() for m in team_members] - await TeamRepository(prisma_client).table.update( + await _team_db(prisma_client).update( where={"team_id": data.team_id}, data={"members_with_roles": json.dumps(_db_team_members)}, # type: ignore ) @@ -3138,7 +3275,7 @@ async def bulk_team_member_add( }, ) # get all users from the database - all_users_in_db = await UserRepository(prisma_client).table.find_many(order={"created_at": "desc"}) + all_users_in_db = await _user_db(prisma_client).find_many(order={"created_at": "desc"}) data.members = [ Member( user_id=user.user_id, @@ -3254,9 +3391,7 @@ async def delete_team( team_rows: List[LiteLLM_TeamTable] = [] for team_id in data.team_ids: try: - team_row_base: Optional[BaseModel] = await TeamRepository(prisma_client).table.find_unique( - where={"team_id": team_id} - ) + team_row_base: Optional[BaseModel] = await _team_db(prisma_client).find_unique(where={"team_id": team_id}) if team_row_base is None: raise Exception except Exception: @@ -3379,7 +3514,7 @@ def _transform_teams_to_deleted_records( teams: List[LiteLLM_TeamTable], user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: Optional[str] = None, -) -> List[Dict[str, Any]]: +) -> List[Dict[str, object]]: """Transform teams into deleted team records ready for persistence.""" if not teams: return [] @@ -3424,13 +3559,13 @@ def _transform_teams_to_deleted_records( async def _save_deleted_team_records( - records: List[Dict[str, Any]], + records: List[Dict[str, object]], prisma_client: PrismaClient, ) -> None: """Save deleted team records to the database.""" if not records: return - await DeletedTeamRepository(prisma_client).table.create_many(data=records) + await _deleted_team_db(prisma_client).create_many(data=records) async def _persist_deleted_team_records( @@ -3506,9 +3641,7 @@ async def _add_team_member_budget_table( team_info_response_object: TeamInfoResponseObjectTeamTable, ) -> TeamInfoResponseObjectTeamTable: try: - team_budget = await BudgetRepository(prisma_client).table.find_unique( - where={"budget_id": team_member_budget_id} - ) + team_budget = await _budget_db(prisma_client).find_unique(where={"budget_id": team_member_budget_id}) team_info_response_object.team_member_budget_table = team_budget except Exception: verbose_proxy_logger.info( @@ -3518,7 +3651,7 @@ async def _add_team_member_budget_table( return team_info_response_object -async def _resolve_team_access_group_resources(_team_info: Any) -> None: +async def _resolve_team_access_group_resources(_team_info: TeamInfoResponseObjectTeamTable) -> None: """Populate access_group_models / mcp_server_ids / agent_ids on the team info response by resolving inherited resources from its access groups.""" if not _team_info.access_group_ids: @@ -3572,7 +3705,7 @@ async def team_info( ) try: - team_info: Optional[BaseModel] = await TeamRepository(prisma_client).table.find_unique( + team_info: Optional[BaseModel] = await _team_db(prisma_client).find_unique( where={"team_id": team_id}, include={"litellm_model_table": True, "object_permission": True}, ) @@ -3819,7 +3952,7 @@ async def block_team( if prisma_client is None: raise Exception("No DB Connected.") - existing_team = await TeamRepository(prisma_client).table.find_unique(where={"team_id": data.team_id}) + existing_team = await _team_db(prisma_client).find_unique(where={"team_id": data.team_id}) if existing_team is None: raise HTTPException( status_code=404, @@ -3832,7 +3965,7 @@ async def block_team( user_api_key_dict=user_api_key_dict, ) - record = await TeamRepository(prisma_client).table.update( + record = await _team_db(prisma_client).update( where={"team_id": data.team_id}, data={"blocked": True}, # type: ignore ) @@ -3868,7 +4001,7 @@ async def unblock_team( if prisma_client is None: raise Exception("No DB Connected.") - existing_team = await TeamRepository(prisma_client).table.find_unique(where={"team_id": data.team_id}) + existing_team = await _team_db(prisma_client).find_unique(where={"team_id": data.team_id}) if existing_team is None: raise HTTPException( status_code=404, @@ -3881,7 +4014,7 @@ async def unblock_team( user_api_key_dict=user_api_key_dict, ) - record = await TeamRepository(prisma_client).table.update( + record = await _team_db(prisma_client).update( where={"team_id": data.team_id}, data={"blocked": False}, # type: ignore ) @@ -3915,7 +4048,7 @@ async def list_available_teams( return [] # filter out teams that the user is already a member of - user_info = await UserRepository(prisma_client).table.find_unique(where={"user_id": user_api_key_dict.user_id}) + user_info = await _user_db(prisma_client).find_unique(where={"user_id": user_api_key_dict.user_id}) if user_info is None: raise HTTPException( status_code=404, @@ -3925,7 +4058,7 @@ async def list_available_teams( available_teams = [team for team in available_teams if team not in user_info_correct_type.teams] - available_teams_db = await TeamRepository(prisma_client).table.find_many(where={"team_id": {"in": available_teams}}) + available_teams_db = await _team_db(prisma_client).find_many(where={"team_id": {"in": available_teams}}) available_teams_correct_type = [LiteLLM_TeamTable.model_validate(team.model_dump()) for team in available_teams_db] @@ -3934,9 +4067,9 @@ async def list_available_teams( async def _get_org_admin_org_ids( user_id: str, - prisma_client: Any, - user_api_key_cache: Any, - proxy_logging_obj: Any, + prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging, ) -> Optional[List[str]]: """ Return the list of organization IDs where the user is an org admin. @@ -3976,16 +4109,16 @@ async def _build_team_list_where_conditions( search: Optional[str] = None, search_team_id_match: TeamIdSearchMatch = "exact", org_admin_org_ids: Optional[List[str]] = None, - user_api_key_cache: Optional[Any] = None, - proxy_logging_obj: Optional[Any] = None, -) -> Optional[Dict[str, Any]]: + user_api_key_cache: Optional[UserApiKeyCache] = None, + proxy_logging_obj: Optional[ProxyLogging] = None, +) -> Optional[Dict[str, object]]: """ Build where conditions for team list query. Returns None when the query is guaranteed to yield no results (e.g. user has no team memberships), allowing the caller to skip the DB round-trip. """ - where_conditions: Dict[str, Any] = {} + where_conditions: Dict[str, object] = {} if team_id: where_conditions["team_id"] = team_id @@ -4067,7 +4200,7 @@ async def _batch_resolve_access_group_resources( return {} unique_ids = list(set(all_access_group_ids)) - rows = await AccessGroupRepository(_prisma_client).table.find_many( + rows = await _access_group_db(_prisma_client).find_many( where={"access_group_id": {"in": unique_ids}}, ) @@ -4115,8 +4248,8 @@ def _convert_teams_to_response_models( async def _get_keys_count_by_team( - prisma_client: Any, - teams: list, + prisma_client: PrismaClient, + teams: Sequence[LiteLLM_TeamTable], ) -> Dict[str, int]: """Aggregate virtual-key counts per team for the given page of teams. @@ -4140,9 +4273,9 @@ async def _enforce_list_team_v2_access( user_api_key_dict: UserAPIKeyAuth, user_id: Optional[str], organization_id: Optional[str], - prisma_client: Any, - user_api_key_cache: Any, - proxy_logging_obj: Any, + prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging, ) -> Tuple[Optional[str], Optional[List[str]]]: """Enforce access control for list_team_v2. @@ -4341,23 +4474,23 @@ async def list_team_v2( # Get teams with pagination if use_deleted_table: - teams = await DeletedTeamRepository(prisma_client).table.find_many( + teams = await _deleted_team_db(prisma_client).find_many( where=where_conditions, skip=skip, take=page_size, order=order_by if order_by else {"created_at": "desc"}, # Default sort ) # Get total count for pagination - total_count = await DeletedTeamRepository(prisma_client).table.count(where=where_conditions) + total_count = await _deleted_team_db(prisma_client).count(where=where_conditions) else: - teams = await TeamRepository(prisma_client).table.find_many( + teams = await _team_db(prisma_client).find_many( where=where_conditions, skip=skip, take=page_size, order=order_by if order_by else {"created_at": "desc"}, # Default sort ) # Get total count for pagination - total_count = await TeamRepository(prisma_client).table.count(where=where_conditions) + total_count = await _team_db(prisma_client).count(where=where_conditions) # Calculate total pages total_pages = -(-total_count // page_size) # Ceiling division @@ -4400,9 +4533,9 @@ async def list_team_v2( async def _authorize_and_filter_teams( user_api_key_dict: UserAPIKeyAuth, user_id: Optional[str], - prisma_client: Any, - user_api_key_cache: Any, - proxy_logging_obj: Any, + prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging, ) -> list: """ Authorize the /team/list request and return filtered teams. @@ -4574,7 +4707,7 @@ async def get_paginated_teams( # Calculate skip for pagination skip = (page - 1) * page_size # Get total count - total_count = await TeamRepository(prisma_client).table.count() + total_count = await _team_db(prisma_client).count() # Get paginated teams teams = await TeamRepository(prisma_client).table.find_many( @@ -4710,7 +4843,7 @@ async def team_model_add( raise HTTPException(status_code=500, detail={"error": "No db connected"}) # Get existing team - team_row = await TeamRepository(prisma_client).table.find_unique(where={"team_id": data.team_id}) + team_row = await _team_db(prisma_client).find_unique(where={"team_id": data.team_id}) if team_row is None: raise HTTPException( @@ -4756,7 +4889,7 @@ async def team_model_add( # the writer and lets Prisma bump updated_at. # `include` mirrors the relations the auth path consumes off the cached # team object so that `_refresh_cached_team` doesn't null them out. - updated_team = await TeamRepository(prisma_client).table.update( + updated_team = await _team_db(prisma_client).update( where={"team_id": data.team_id}, data={"updated_at": datetime.now(timezone.utc)}, include={"object_permission": True}, # type: ignore @@ -4810,7 +4943,7 @@ async def team_model_delete( raise HTTPException(status_code=500, detail={"error": "No db connected"}) # Get existing team - team_row = await TeamRepository(prisma_client).table.find_unique(where={"team_id": data.team_id}) + team_row = await _team_db(prisma_client).find_unique(where={"team_id": data.team_id}) if team_row is None: raise HTTPException( @@ -4973,7 +5106,7 @@ async def update_team_member_permissions( }, ) # Update the team member permissions - updated_team = await TeamRepository(prisma_client).table.update( + updated_team = await _team_db(prisma_client).update( where={"team_id": data.team_id}, data={"team_member_permissions": data.team_member_permissions}, ) @@ -5043,7 +5176,7 @@ async def bulk_update_team_member_permissions( } -async def _compute_and_batch_updates(prisma_client, teams, permissions_to_add: set) -> int: +async def _compute_and_batch_updates(prisma_client, teams: Sequence[LiteLLM_TeamTable], permissions_to_add: set) -> int: """Compute merged permissions and batch-write updates. Returns count of teams updated.""" updates = [] for team in teams: @@ -5065,9 +5198,11 @@ async def _compute_and_batch_updates(prisma_client, teams, permissions_to_add: s return len(updates) -async def _append_permissions_to_specific_teams(prisma_client, team_ids: List[str], permissions_to_add: set) -> int: +async def _append_permissions_to_specific_teams( + prisma_client: PrismaClient, team_ids: List[str], permissions_to_add: set +) -> int: """Fetch specific teams by ID and append permissions.""" - teams = await TeamRepository(prisma_client).table.find_many( + teams = await _team_db(prisma_client).find_many( where={"team_id": {"in": team_ids}}, ) @@ -5082,7 +5217,7 @@ async def _append_permissions_to_specific_teams(prisma_client, team_ids: List[st return await _compute_and_batch_updates(prisma_client, teams, permissions_to_add) -async def _append_permissions_to_all_teams(prisma_client, permissions_to_add: set) -> int: +async def _append_permissions_to_all_teams(prisma_client: PrismaClient, permissions_to_add: set) -> int: """Paginated read + batched write across all teams.""" teams_updated = 0 cursor = None @@ -5228,9 +5363,7 @@ async def get_team_daily_activity( # If user does not have full team view, filter by their API keys if not has_full_team_view: # Get all API keys for this user - user_keys = await VerificationTokenRepository(prisma_client).table.find_many( - where={"user_id": user_api_key_dict.user_id} - ) + user_keys = await _tokens_db(prisma_client).find_many(where={"user_id": user_api_key_dict.user_id}) user_api_keys = [key.token for key in user_keys if key.token] # If user has no API keys, return empty result if not user_api_keys: diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index f7e429ddb31..bc37058689e 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -11,7 +11,10 @@ from typing import ( Literal, Mapping, NamedTuple, + Protocol, Sequence, + TypedDict, + TypeVar, Union, ) @@ -48,6 +51,204 @@ router = APIRouter() SPEND_LOGS_PAGINATION_COUNT_CAP = 10000 +_RowT = TypeVar("_RowT") + + +class _SupportsModelDump(Protocol): + def model_dump(self) -> Mapping[str, object]: ... + + +class _SpendLogOwnershipRow(Protocol): + user: str | None + team_id: str | None + + +class _ActivityRow(TypedDict): + date: str + api_requests: int + total_tokens: int + + +class _ActivityModelRow(TypedDict): + model_group: str + date: str + api_requests: int + total_tokens: int + + +class _DeploymentExceptionsRow(TypedDict): + api_base: str + date: str + num_rate_limit_exceptions: int + + +class _ExceptionsRow(TypedDict): + date: str + num_rate_limit_exceptions: int + + +class _ModelIdSpendRow(TypedDict): + model_id: str + spend: float + + +class _TagNameRow(TypedDict): + individual_request_tag: str + + +class _TeamSpendRow(TypedDict): + team_alias: str | None + total_spend: float + + +class _TagSpendRow(TypedDict): + individual_request_tag: str + total_spend: float + + +class _SpendLogsCountRow(TypedDict): + total_count: int + + +class _PgClassRow(TypedDict): + relname: str + relkind: str + + +class _TotalSpendRow(TypedDict): + total_spend: float + + +class _TeamDailySpendRow(TypedDict): + team_alias: str | None + spend_date: str | None + total_spend: float + + +class _EndUserRow(TypedDict): + end_user: str | None + + +class _DailyTagSpendRow(TypedDict): + individual_request_tag: str + log_count: int + total_spend: float + + +class _SessionCountAggregate(TypedDict): + session_id: int + + +class _SessionCountRow(TypedDict): + session_id: str + _count: _SessionCountAggregate + + +class _SessionSpendRow(TypedDict): + session_id: str + session_total_spend: float + mcp_tool_call_count: int + mcp_tool_call_spend: float + + +async def _query_raw(prisma_client: PrismaClient, sql_query: str, *args: object) -> Sequence[_RowT]: + """Run a raw read query and return its rows as the row type the caller declares.""" + return await prisma_client.db.query_raw(sql_query, *args) + + +async def _query_raw_or_none(prisma_client: PrismaClient, sql_query: str, *args: object) -> Sequence[_RowT] | None: + """``_query_raw`` for the call sites that guard the result against ``None``.""" + return await _query_raw(prisma_client, sql_query, *args) + + +class _SpendLogsTable(Protocol): + """The subset of the Prisma spend-logs table API this module uses.""" + + async def find_many( + self, *, where: Mapping[str, object], order: Mapping[str, str] + ) -> Sequence[_SupportsModelDump]: ... + + async def find_unique( + self, *, where: Mapping[str, object], include: None = None + ) -> _SpendLogOwnershipRow | None: ... + + async def count(self, *, where: Mapping[str, object]) -> int: ... + + async def group_by( + self, *, by: Sequence[str], where: Mapping[str, object], count: Mapping[str, bool] + ) -> Sequence[_SessionCountRow]: ... + + +class _TeamTable(Protocol): + """The subset of the Prisma team table API this module uses.""" + + async def find_unique(self, *, where: Mapping[str, object]) -> _SupportsModelDump | None: ... + + async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_SupportsModelDump]: ... + + async def update_many(self, *, data: Mapping[str, float], where: Mapping[str, object]) -> int: ... + + +class _VerificationTokenTable(Protocol): + """The subset of the Prisma verification token table API this module uses.""" + + async def update_many(self, *, data: Mapping[str, float], where: Mapping[str, object]) -> int: ... + + +def _spend_logs_table(prisma_client: PrismaClient) -> _SpendLogsTable: + return SpendLogsRepository(prisma_client).table + + +def _team_table(prisma_client: PrismaClient) -> _TeamTable: + return TeamRepository(prisma_client).table + + +def _verification_token_table(prisma_client: PrismaClient) -> _VerificationTokenTable: + return VerificationTokenRepository(prisma_client).table + + +async def _find_spend_logs( + prisma_client: PrismaClient, + where: Mapping[str, object], + order: Mapping[str, str], +) -> Sequence[_SupportsModelDump]: + """Read spend log rows as Prisma model instances.""" + return await _spend_logs_table(prisma_client).find_many(where=where, order=order) + + +async def _find_spend_log_row(prisma_client: PrismaClient, request_id: str) -> _SpendLogOwnershipRow | None: + """Read the single spend log row identified by ``request_id``.""" + return await _spend_logs_table(prisma_client).find_unique( + where={"request_id": request_id}, + include=None, + ) + + +async def _count_spend_logs(prisma_client: PrismaClient, where: Mapping[str, object]) -> int: + """Count the spend log rows matching ``where``.""" + return await _spend_logs_table(prisma_client).count(where=where) + + +async def _count_logs_per_session( + prisma_client: PrismaClient, session_ids: Sequence[str | None] +) -> Sequence[_SessionCountRow]: + """Count spend log rows per session for the given session ids.""" + return await _spend_logs_table(prisma_client).group_by( + by=["session_id"], + where={"session_id": {"in": session_ids}}, + count={"session_id": True}, + ) + + +async def _find_team_row(prisma_client: PrismaClient, team_id: str) -> _SupportsModelDump | None: + """Read a single team row as a Prisma model instance.""" + return await _team_table(prisma_client).find_unique(where={"team_id": team_id}) + + +async def _find_team_rows(prisma_client: PrismaClient, team_ids: Sequence[str]) -> Sequence[_SupportsModelDump]: + """Read team rows as Prisma model instances.""" + return await _team_table(prisma_client).find_many(where={"team_id": {"in": team_ids}}) + @router.get( "/spend/keys", @@ -281,7 +482,9 @@ async def get_global_activity_internal_user( AND "user" = $3 GROUP BY date_trunc('day', "startTime") """ - db_response = await prisma_client.db.query_raw(sql_query, start_date, end_date, user_id) + db_response: Sequence[_ActivityRow] | None = await _query_raw_or_none( + prisma_client, sql_query, start_date, end_date, user_id + ) return db_response @@ -345,6 +548,7 @@ async def get_global_activity( "Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys" ) + db_response: Sequence[_ActivityRow] | None if ( user_api_key_dict.user_role == LitellmUserRoles.INTERNAL_USER or user_api_key_dict.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY @@ -361,7 +565,7 @@ async def get_global_activity( AND "startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') GROUP BY date_trunc('day', "startTime") """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj) + db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj) if db_response is None: return [] @@ -420,7 +624,9 @@ async def get_global_activity_model_internal_user( AND "user" = $3 GROUP BY model_group, date_trunc('day', "startTime") """ - db_response = await prisma_client.db.query_raw(sql_query, start_date, end_date, user_id) + db_response: Sequence[_ActivityModelRow] | None = await _query_raw_or_none( + prisma_client, sql_query, start_date, end_date, user_id + ) return db_response @@ -507,6 +713,7 @@ async def get_global_activity_model( "Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys" ) + db_response: Sequence[_ActivityModelRow] | None if ( user_api_key_dict.user_role == LitellmUserRoles.INTERNAL_USER or user_api_key_dict.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY @@ -524,7 +731,7 @@ async def get_global_activity_model( AND "startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') GROUP BY model_group, date_trunc('day', "startTime") """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj) + db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj) if db_response is None: return [] @@ -672,7 +879,9 @@ async def get_global_activity_exceptions_per_deployment( ORDER BY date; """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj, model_group) + db_response: Sequence[_DeploymentExceptionsRow] | None = await _query_raw_or_none( + prisma_client, sql_query, start_date_obj, end_date_obj, model_group + ) if db_response is None: return [] @@ -795,7 +1004,9 @@ async def get_global_activity_exceptions( ORDER BY date; """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj, model_group) + db_response: Sequence[_ExceptionsRow] | None = await _query_raw_or_none( + prisma_client, sql_query, start_date_obj, end_date_obj, model_group + ) if db_response is None: return [] @@ -883,6 +1094,7 @@ async def get_global_spend_provider( "Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys" ) + db_response: Sequence[_ModelIdSpendRow] | None if ( user_api_key_dict.user_role == LitellmUserRoles.INTERNAL_USER or user_api_key_dict.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY @@ -902,7 +1114,7 @@ async def get_global_spend_provider( AND "user" = $3 GROUP BY model_id """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj, user_id) + db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj, user_id) else: sql_query = """ SELECT @@ -914,7 +1126,7 @@ async def get_global_spend_provider( AND length(model_id) > 0 GROUP BY model_id """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj) + db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj) if db_response is None: return [] @@ -1042,6 +1254,7 @@ async def get_global_spend_report( if premium_user is not True: verbose_proxy_logger.debug("accessing /spend/report but not a premium user") raise ValueError("/spend/report endpoint " + CommonProxyErrors.not_premium_user.value) + db_response: Sequence[Mapping[str, object]] | None if api_key is not None: verbose_proxy_logger.debug("Getting /spend for api_key: [set=%s]", api_key is not None) if api_key.startswith("sk-"): @@ -1082,7 +1295,7 @@ async def get_global_spend_report( ORDER BY total_cost DESC; """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj, api_key) + db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj, api_key) if db_response is None: return [] @@ -1125,7 +1338,9 @@ async def get_global_spend_report( ORDER BY total_cost DESC; """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj, internal_user_id) + db_response = await _query_raw_or_none( + prisma_client, sql_query, start_date_obj, end_date_obj, internal_user_id + ) if db_response is None: return [] @@ -1190,7 +1405,7 @@ async def get_global_spend_report( group_by_day; """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj) + db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj) if db_response is None: return [] @@ -1231,7 +1446,7 @@ async def get_global_spend_report( ORDER BY total_cost DESC; """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj) + db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj) if db_response is None: return [] @@ -1268,7 +1483,7 @@ async def global_get_all_tag_names(): FROM "LiteLLM_SpendLogs"; """ - db_response = await prisma_client.db.query_raw(sql_query) + db_response: Sequence[_TagNameRow] | None = await _query_raw_or_none(prisma_client, sql_query) if db_response is None: return [] @@ -1415,7 +1630,9 @@ async def _get_spend_report_for_time_range( ORDER BY total_spend DESC; """ - response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj) + response: Sequence[_TeamSpendRow] | None = await _query_raw_or_none( + prisma_client, sql_query, start_date_obj, end_date_obj + ) # get spend per tag for today sql_query = """ @@ -1429,7 +1646,9 @@ async def _get_spend_report_for_time_range( ORDER BY total_spend DESC; """ - spend_per_tag = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj) + spend_per_tag: Sequence[_TagSpendRow] | None = await _query_raw_or_none( + prisma_client, sql_query, start_date_obj, end_date_obj + ) return response, spend_per_tag except Exception as e: @@ -1894,7 +2113,7 @@ async def ui_view_spend_logs( # (messages, response, proxy_server_request can be hundreds of KB per row). # These are only needed in the detail endpoint /spend/logs/ui/{request_id}. sql_conditions: List[str] = [] - sql_params: List[Any] = [] + sql_params: list[object] = [] p = 1 # parameter index counter # Date range. Wrap the param side with `AT TIME ZONE 'UTC'` so comparison @@ -2002,7 +2221,9 @@ async def ui_view_spend_logs( LIMIT ${p} ) AS bounded_matches """ - count_rows = await prisma_client.db.query_raw(count_query, *sql_params, SPEND_LOGS_PAGINATION_COUNT_CAP + 1) + count_rows: Sequence[_SpendLogsCountRow] | None = await _query_raw_or_none( + prisma_client, count_query, *sql_params, SPEND_LOGS_PAGINATION_COUNT_CAP + 1 + ) raw_total = int(count_rows[0]["total_count"]) if count_rows else 0 total_is_capped = raw_total > SPEND_LOGS_PAGINATION_COUNT_CAP total_records = SPEND_LOGS_PAGINATION_COUNT_CAP if total_is_capped else raw_total @@ -2067,7 +2288,7 @@ def _spend_log_field_has_content(value: Union[str, list, dict] | None) -> bool: return True -def _hydrate_spend_log_metadata(rows: Sequence[Any]) -> None: +def _hydrate_spend_log_metadata(rows: Sequence[Mapping[str, object]]) -> None: """Re-hydrate the JSONB ``metadata`` column returned by ``query_raw`` as a string. The Prisma serialiser bypasses the model-layer JSON hydration we get on the ORM @@ -2227,7 +2448,9 @@ async def ui_view_request_response_for_request_id( WHERE request_id = $1 LIMIT 1 """ - db_result = await prisma_client.db.query_raw(sql_query, request_id) + db_result: Sequence[Mapping[str, object]] | None = await _query_raw_or_none( + prisma_client, sql_query, request_id + ) if db_result and len(db_result) > 0: resolved = await _resolve_request_response_payload(db_result[0], cold_storage_handler=ColdStorageHandler()) return resolved._asdict() @@ -2359,11 +2582,10 @@ async def view_spend_logs( # Check if user wants unsummarized data if not summarize: # Return filtered individual log entries (similar to UI endpoint) - data = await SpendLogsRepository(prisma_client).table.find_many( - where=filter_query, # type: ignore - order={ - "startTime": "desc", - }, + data = await _find_spend_logs( + prisma_client, + where=filter_query, + order={"startTime": "desc"}, ) return data @@ -2421,7 +2643,7 @@ async def view_spend_logs( return response else: - scoped_filter: Dict[str, Any] = {} + scoped_filter: dict[str, str] = {} if api_key is not None and isinstance(api_key, str): if api_key.startswith("sk-"): hashed_token = prisma_client.hash_token(token=api_key) @@ -2437,8 +2659,9 @@ async def view_spend_logs( spend_logs = await prisma_client.get_data(table_name="spend", query_type="find_all") return spend_logs - data = await SpendLogsRepository(prisma_client).table.find_many( - where=scoped_filter, # type: ignore + data = await _find_spend_logs( + prisma_client, + where=scoped_filter, order={"startTime": "desc"}, ) return data @@ -2489,8 +2712,8 @@ async def global_spend_reset(): code=status.HTTP_401_UNAUTHORIZED, ) - await VerificationTokenRepository(prisma_client).table.update_many(data={"spend": 0.0}, where={}) - await TeamRepository(prisma_client).table.update_many(data={"spend": 0.0}, where={}) + await _verification_token_table(prisma_client).update_many(data={"spend": 0.0}, where={}) + await _team_table(prisma_client).update_many(data={"spend": 0.0}, where={}) return { "message": "Spend for all API Keys and Teams reset successfully", @@ -2533,7 +2756,7 @@ async def global_spend_refresh(): WHERE relname = 'MonthlyGlobalSpend'; """ try: - resp = await prisma_client.db.query_raw(sql_query) + resp: Sequence[_PgClassRow] = await _query_raw(prisma_client, sql_query) return resp[0]["relkind"] == "m" except Exception: @@ -2562,7 +2785,7 @@ async def global_spend_refresh(): }, ) await new_client.db.connect() - await new_client.db.query_raw(sql_query) + await _query_raw(new_client, sql_query) verbose_proxy_logger.info("MonthlyGlobalSpend view refreshed") return { "message": "MonthlyGlobalSpend view refreshed", @@ -2601,13 +2824,13 @@ async def global_spend_for_internal_user( ORDER BY "date"; """ - response = await prisma_client.db.query_raw(sql_query, api_key, user_id) + response: Sequence[Mapping[str, object]] = await _query_raw(prisma_client, sql_query, api_key, user_id) return response sql_query = """SELECT * FROM "MonthlyGlobalSpendPerUserPerKey" WHERE "user" = $1 ORDER BY "date";""" - response = await prisma_client.db.query_raw(sql_query, user_id) + response = await _query_raw(prisma_client, sql_query, user_id) return response except Exception as e: @@ -2652,6 +2875,7 @@ async def global_spend_logs( code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) + response: Sequence[Mapping[str, object]] if ( user_api_key_dict.user_role == LitellmUserRoles.INTERNAL_USER or user_api_key_dict.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY @@ -2669,7 +2893,7 @@ async def global_spend_logs( if api_key is None: sql_query = """SELECT * FROM "MonthlyGlobalSpend" ORDER BY "date";""" - response = await prisma_client.db.query_raw(query=sql_query) + response = await _query_raw(prisma_client, sql_query) return response else: @@ -2679,7 +2903,7 @@ async def global_spend_logs( ORDER BY "date"; """ - response = await prisma_client.db.query_raw(sql_query, api_key) + response = await _query_raw(prisma_client, sql_query, api_key) return response @@ -2726,7 +2950,7 @@ async def global_spend(): if prisma_client is None: raise HTTPException(status_code=500, detail={"error": "No db connected"}) sql_query = """SELECT SUM(spend) as total_spend FROM "MonthlyGlobalSpend";""" - response = await prisma_client.db.query_raw(query=sql_query) + response: Sequence[_TotalSpendRow] | None = await _query_raw_or_none(prisma_client, sql_query) if response is not None: if isinstance(response, list) and len(response) > 0: total_spend = response[0].get("total_spend", 0.0) @@ -2791,7 +3015,7 @@ async def global_spend_key_internal_user(user_api_key_dict: UserAPIKeyAuth, limi """ - response = await prisma_client.db.query_raw(sql_query, user_id, limit) + response: Sequence[Mapping[str, object]] = await _query_raw(prisma_client, sql_query, user_id, limit) return response @@ -2816,6 +3040,7 @@ async def global_spend_keys( """ from litellm.proxy.proxy_server import prisma_client + response: Sequence[Mapping[str, object]] if ( user_api_key_dict.user_role == LitellmUserRoles.INTERNAL_USER or user_api_key_dict.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY @@ -2828,14 +3053,14 @@ async def global_spend_keys( sql_query = """SELECT * FROM "Last30dKeysBySpend";""" if limit is None: - response = await prisma_client.db.query_raw(sql_query) + response = await _query_raw(prisma_client, sql_query) return response try: limit = int(limit) if limit < 1: raise ValueError("Limit must be greater than 0") sql_query = """SELECT * FROM "Last30dKeysBySpend" LIMIT $1 ;""" - response = await prisma_client.db.query_raw(sql_query, limit) + response = await _query_raw(prisma_client, sql_query, limit) except ValueError as e: raise HTTPException(status_code=422, detail={"error": f"Invalid limit: {limit}, error: {e}"}) from e @@ -2875,7 +3100,7 @@ async def global_spend_per_team(): ORDER BY spend_date; """ - response = await prisma_client.db.query_raw(query=sql_query) + response: Sequence[_TeamDailySpendRow] = await _query_raw(prisma_client, sql_query) # transform the response for the Admin UI spend_by_date = {} @@ -2952,7 +3177,7 @@ async def global_view_all_end_users(): SELECT DISTINCT end_user FROM "LiteLLM_SpendLogs" """ - db_response = await prisma_client.db.query_raw(query=sql_query) + db_response: Sequence[_EndUserRow] | None = await _query_raw_or_none(prisma_client, sql_query) if db_response is None: return [] @@ -3009,7 +3234,9 @@ GROUP BY end_user ORDER BY total_spend DESC LIMIT 100 """ - response = await prisma_client.db.query_raw(sql_query, startTime, endTime, selected_api_key) + response: Sequence[Mapping[str, object]] = await _query_raw( + prisma_client, sql_query, startTime, endTime, selected_api_key + ) return response @@ -3040,7 +3267,7 @@ async def global_spend_models_internal_user(user_api_key_dict: UserAPIKeyAuth, l LIMIT $2; """ - response = await prisma_client.db.query_raw(sql_query, user_id, limit) + response: Sequence[Mapping[str, object]] = await _query_raw(prisma_client, sql_query, user_id, limit) return response @@ -3077,7 +3304,7 @@ async def global_spend_models( sql_query = """SELECT * FROM "Last30dModelsBySpend" LIMIT $1 ;""" - response = await prisma_client.db.query_raw(sql_query, int(limit)) + response: Sequence[Mapping[str, object]] = await _query_raw(prisma_client, sql_query, int(limit)) return response @@ -3169,14 +3396,17 @@ async def provider_budgets() -> ProviderBudgetResponse: async def get_spend_by_tags(prisma_client: PrismaClient, start_date=None, end_date=None): - response = await prisma_client.db.query_raw(""" + response: Sequence[Mapping[str, object]] = await _query_raw( + prisma_client, + """ SELECT jsonb_array_elements_text(request_tags) AS individual_request_tag, COUNT(*) AS log_count, SUM(spend) AS total_spend FROM "LiteLLM_SpendLogs" GROUP BY individual_request_tag; - """) + """, + ) return response @@ -3203,7 +3433,7 @@ async def ui_get_spend_by_tags( if prisma_client is None: raise HTTPException(status_code=500, detail={"error": "No db connected"}) - response = None + response: Sequence[_DailyTagSpendRow] | None = None if tags_list is None or (isinstance(tags_list, list) and "all-tags" in tags_list): # Get spend for all tags sql_query = """ @@ -3216,7 +3446,8 @@ async def ui_get_spend_by_tags( WHERE spend_date >= $1::date AND spend_date <= $2::date ORDER BY total_spend DESC; """ - response = await prisma_client.db.query_raw( + response = await _query_raw( + prisma_client, sql_query, start_date, end_date, @@ -3234,7 +3465,8 @@ async def ui_get_spend_by_tags( GROUP BY individual_request_tag ORDER BY total_spend DESC; """ - response = await prisma_client.db.query_raw( + response = await _query_raw( + prisma_client, sql_query, start_date, end_date, @@ -3353,7 +3585,7 @@ async def ui_view_session_spend_logs( skip = (page - 1) * page_size # Get total count for pagination metadata - total_records = await SpendLogsRepository(prisma_client).table.count(where=where_conditions) + total_records = await _count_spend_logs(prisma_client, where_conditions) # Query with raw SQL to exclude heavy columns (messages, response, proxy_server_request) sql_query = f""" @@ -3370,7 +3602,9 @@ async def ui_view_session_spend_logs( ORDER BY "startTime" DESC LIMIT $2 OFFSET $3 """ - result = await prisma_client.db.query_raw(sql_query, session_id, page_size, skip, *scope_params) + result: Sequence[Mapping[str, object]] = await _query_raw( + prisma_client, sql_query, session_id, page_size, skip, *scope_params + ) _hydrate_spend_log_metadata(result) total_pages = (total_records + page_size - 1) // page_size @@ -3434,7 +3668,7 @@ async def _build_ui_spend_logs_response( """ count_map: dict[str, int] = {} if enrich_session_counts: - session_ids = list( + session_ids: Sequence[str | None] = list( { (row.get("session_id") if isinstance(row, dict) else getattr(row, "session_id", None)) for row in data @@ -3446,11 +3680,7 @@ async def _build_ui_spend_logs_response( # is bounded by page_size (typically 25-50 distinct session IDs). # If performance degrades at scale, consider short-lived caching or # folding the count into the main query via a window function. - counts = await SpendLogsRepository(prisma_client).table.group_by( - by=["session_id"], - where={"session_id": {"in": session_ids}}, - count={"session_id": True}, - ) + counts = await _count_logs_per_session(prisma_client, session_ids) count_map = {r["session_id"]: r["_count"]["session_id"] for r in counts if r.get("session_id")} session_spend_map: dict[str, dict[str, Union[int, float]]] = {} @@ -3461,14 +3691,15 @@ async def _build_ui_spend_logs_response( # Collect api_keys already present in the authorized page rows so the # aggregate is scoped to the same ownership as the main query — prevents # cross-tenant disclosure via a colliding session_id. - authorized_api_keys = list( + authorized_api_keys: Sequence[str | None] = list( { (row.get("api_key") if isinstance(row, dict) else getattr(row, "api_key", None)) for row in data if (row.get("api_key") if isinstance(row, dict) else getattr(row, "api_key", None)) } ) - rows = await prisma_client.db.query_raw( + rows: Sequence[_SessionSpendRow] = await _query_raw( + prisma_client, """ SELECT session_id, COALESCE(SUM(spend), 0)::double precision AS session_total_spend, @@ -3531,7 +3762,7 @@ async def _build_ui_spend_logs_response( } -def _build_status_filter_condition(status_filter: str | None) -> Dict[str, Any]: +def _build_status_filter_condition(status_filter: str | None) -> Mapping[str, object]: """ Helper function to build the status filter condition for database queries. @@ -3539,7 +3770,7 @@ def _build_status_filter_condition(status_filter: str | None) -> Dict[str, Any]: status_filter (Optional[str]): The status to filter by. Can be "success" or "failure". Returns: - Dict[str, Any]: A dictionary containing the status filter condition. + Mapping[str, object]: A mapping containing the status filter condition. """ if status_filter is None: return {} @@ -3568,7 +3799,7 @@ def _is_admin_view_safe(user_api_key_dict: UserAPIKeyAuth) -> bool: async def _can_team_member_view_log( - prisma_client, + prisma_client: PrismaClient, user_api_key_dict: UserAPIKeyAuth, team_id: str | None, ) -> bool: @@ -3584,7 +3815,7 @@ async def _can_team_member_view_log( if team_id is None: return False - team_row = await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id}) + team_row = await _find_team_row(prisma_client, team_id) if team_row is None: return False team_obj = LiteLLM_TeamTable.model_validate(team_row.model_dump()) @@ -3614,7 +3845,7 @@ def _can_user_view_spend_log(user_api_key_dict: UserAPIKeyAuth) -> bool: async def _assert_user_can_view_request_id( - prisma_client, + prisma_client: PrismaClient, user_api_key_dict: UserAPIKeyAuth, request_id: str, ) -> None: @@ -3624,10 +3855,7 @@ async def _assert_user_can_view_request_id( permitted teams (admin or ``/spend/logs`` permission). Raises HTTP 403 if not. """ - row = await SpendLogsRepository(prisma_client).table.find_unique( - where={"request_id": request_id}, - include=None, - ) + row = await _find_spend_log_row(prisma_client, request_id) if row is None: return @@ -3650,7 +3878,7 @@ async def _assert_user_can_view_request_id( async def _get_permitted_team_ids_for_spend_logs( - prisma_client, + prisma_client: PrismaClient, user_api_key_dict: UserAPIKeyAuth, ) -> List[str]: """ @@ -3675,7 +3903,7 @@ async def _get_permitted_team_ids_for_spend_logs( if user_obj is None or not user_obj.teams: return [] - team_rows = await TeamRepository(prisma_client).table.find_many(where={"team_id": {"in": user_obj.teams}}) + team_rows = await _find_team_rows(prisma_client, user_obj.teams) permitted: List[str] = [] for team_row in team_rows: diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 9d30e40cd54..02c4712828d 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -1,5 +1,6 @@ import re import traceback +from collections.abc import Mapping, Sequence from datetime import datetime from typing import ( TYPE_CHECKING, @@ -22,7 +23,11 @@ from litellm.proxy._experimental.mcp_server.utils import ( ) from litellm.responses.main import aresponses from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator -from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.llms.openai import ( + ResponseInputParam, + ResponsesAPIResponse, + ResponsesAPIStreamingResponse, +) from litellm.types.utils import ( CallTypes, Choices, @@ -32,8 +37,10 @@ from litellm.types.utils import ( from litellm.utils import Rules, function_setup if TYPE_CHECKING: + from mcp.types import CallToolResult from mcp.types import Tool as MCPTool + from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging else: MCPTool = Any @@ -94,7 +101,7 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def _parse_mcp_tools( - tools: Optional[Iterable[ToolParam]], + tools: Iterable[Mapping[str, object]] | None, ) -> Tuple[List[ToolParam], List[Any]]: """ Parse tools and separate MCP tools with litellm_proxy from other tools. @@ -134,8 +141,8 @@ class LiteLLM_Proxy_MCP_Handler: async def _apply_toolset_permissions( resolved_toolset_ids: List[str], resolved_mcp_servers: List[str], - user_api_key_auth: Any, - ) -> Any: + user_api_key_auth: "UserAPIKeyAuth", + ) -> "UserAPIKeyAuth": """Apply resolved toolset permissions to user_api_key_auth and return updated auth.""" from litellm.proxy._types import LiteLLM_ObjectPermissionTable @@ -174,8 +181,8 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod async def _get_mcp_tools_from_manager( - user_api_key_auth: Any, - mcp_tools_with_litellm_proxy: Optional[Iterable[ToolParam]], + user_api_key_auth: "UserAPIKeyAuth | None", + mcp_tools_with_litellm_proxy: Iterable[Mapping[str, object]] | None, litellm_trace_id: Optional[str] = None, mcp_auth_header: Optional[str] = None, mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, @@ -330,7 +337,7 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def _filter_mcp_tools_by_allowed_tools( - mcp_tools: List[MCPTool], mcp_tools_with_litellm_proxy: List[ToolParam] + mcp_tools: List[MCPTool], mcp_tools_with_litellm_proxy: Sequence[Mapping[str, object]] ) -> List[MCPTool]: """Filter MCP tools based on allowed_tools parameter from the original tool configs.""" # Collect all allowed tool names from all MCP tool configs @@ -368,8 +375,8 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod async def _process_mcp_tools_to_openai_format( - user_api_key_auth: Any, - mcp_tools_with_litellm_proxy: List[ToolParam], + user_api_key_auth: "UserAPIKeyAuth | None", + mcp_tools_with_litellm_proxy: Sequence[Mapping[str, object]], litellm_trace_id: Optional[str] = None, request_tags: Optional[list[str]] = None, ) -> tuple[List[Any], dict[str, str]]: @@ -402,12 +409,12 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod async def _process_mcp_tools_without_openai_transform( user_api_key_auth: Any, - mcp_tools_with_litellm_proxy: List[ToolParam], + mcp_tools_with_litellm_proxy: Sequence[Mapping[str, object]], litellm_trace_id: Optional[str] = None, mcp_auth_header: Optional[str] = None, mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, request_tags: Optional[list[str]] = None, - ) -> tuple[List[Any], dict[str, str]]: + ) -> tuple[List[MCPTool], dict[str, str]]: """ Process MCP tools through filtering and deduplication pipeline without OpenAI transformation. This is useful for cases where we need the original MCP tool objects (e.g., for events). @@ -453,7 +460,7 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def _transform_mcp_tools_to_openai( - mcp_tools: List[Any], + mcp_tools: Sequence[MCPTool], target_format: Literal["responses", "chat"] = "responses", ) -> List[Any]: """Transform MCP tools to OpenAI-compatible format.""" @@ -464,7 +471,6 @@ class LiteLLM_Proxy_MCP_Handler: openai_tools: List[Any] = [] for mcp_tool in mcp_tools: - openai_tool: Any if target_format == "chat": openai_tool = transform_mcp_tool_to_openai_tool(mcp_tool) else: @@ -475,7 +481,7 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def _should_auto_execute_tools( - mcp_tools_with_litellm_proxy: Union[List[Dict[str, Any]], List[ToolParam]], + mcp_tools_with_litellm_proxy: Sequence[Mapping[str, object]], ) -> bool: """Check if we should auto-execute tool calls. @@ -514,9 +520,9 @@ class LiteLLM_Proxy_MCP_Handler: return tool_calls @staticmethod - def _extract_tool_calls_from_chat_response(response: ModelResponse) -> List[Any]: + def _extract_tool_calls_from_chat_response(response: ModelResponse) -> list[object]: """Extract tool calls from a chat completion response.""" - tool_calls: List[Any] = [] + tool_calls: list[object] = [] try: for choice in response.choices: @@ -583,7 +589,7 @@ class LiteLLM_Proxy_MCP_Handler: return tool_arguments or {} @staticmethod - def _parse_mcp_result(result: Any) -> str: + def _parse_mcp_result(result: "CallToolResult") -> str: """Parse MCP tool call result and extract meaningful content.""" if not result or not hasattr(result, "content") or not result.content: return "Tool executed successfully" @@ -626,7 +632,7 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod async def _execute_tool_calls( tool_server_map: dict[str, str], - tool_calls: List[Any], + tool_calls: Sequence[object], user_api_key_auth: Any, mcp_auth_header: Optional[str] = None, mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, @@ -908,7 +914,7 @@ class LiteLLM_Proxy_MCP_Handler: def _create_follow_up_messages_for_chat( original_messages: List[Any], response: ModelResponse, - tool_results: List[Dict[str, Any]], + tool_results: Sequence[Mapping[str, object]], ) -> List[Any]: """Create follow-up chat messages that include tool execution results.""" from copy import deepcopy @@ -952,8 +958,8 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def _create_follow_up_input( response: ResponsesAPIResponse, - tool_results: List[Dict[str, Any]], - original_input: Any = None, + tool_results: Sequence[Mapping[str, object]], + original_input: str | ResponseInputParam | None = None, ) -> List[Any]: """Create follow-up input with tool results in proper format.""" follow_up_input: List[Any] = [] @@ -1049,7 +1055,7 @@ class LiteLLM_Proxy_MCP_Handler: *, proxy_logging_obj: Optional["ProxyLogging"], user_api_key_auth: Any, - request_data: Dict[str, Any], + request_data: dict[str, object], error: Exception, ) -> None: """Log MCP tool failures via proxy logging hooks.""" @@ -1071,11 +1077,11 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def _create_mcp_streaming_response( - input: Union[str, Any], + input: str | ResponseInputParam, model: str, - all_tools: Optional[List[Any]], - mcp_tools_with_litellm_proxy: List[Any], - mcp_discovery_events: List[Any], + all_tools: Sequence[object] | None, + mcp_tools_with_litellm_proxy: list[Mapping[str, object]], + mcp_discovery_events: list[ResponsesAPIStreamingResponse], call_params: Dict[str, Any], previous_response_id: Optional[str], tool_server_map: dict[str, str], @@ -1116,9 +1122,9 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def _build_request_params( - input: Union[str, Any], + input: str | ResponseInputParam, model: str, - all_tools: Optional[List[Any]], + all_tools: Sequence[object] | None, call_params: Dict[str, Any], previous_response_id: Optional[str], **kwargs, @@ -1149,7 +1155,9 @@ class LiteLLM_Proxy_MCP_Handler: return request_params @staticmethod - def _create_tool_execution_events(tool_calls: List[Any], tool_results: List[Dict[str, Any]]) -> List[Any]: + def _create_tool_execution_events( + tool_calls: Sequence[object], tool_results: List[Dict[str, Any]] + ) -> list[ResponsesAPIStreamingResponse]: """ Create MCP tool execution events for streaming. @@ -1163,7 +1171,7 @@ class LiteLLM_Proxy_MCP_Handler: from litellm._uuid import uuid from litellm.responses.mcp.mcp_streaming_iterator import create_mcp_call_events - tool_execution_events: List[Any] = [] + tool_execution_events: list[ResponsesAPIStreamingResponse] = [] # Create events for each tool execution for tool_result in tool_results: @@ -1233,8 +1241,8 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def _add_mcp_output_elements_to_response( response: ResponsesAPIResponse, - mcp_tools_fetched: List[Any], - tool_results: List[Dict[str, Any]], + mcp_tools_fetched: Sequence[object], + tool_results: Sequence[Mapping[str, object]], ) -> ResponsesAPIResponse: """Add custom output elements to the final response for MCP tool execution.""" # Import the required classes for creating output items diff --git a/litellm/videos/main.py b/litellm/videos/main.py index 6d81fec36b3..cc3a15f9cd2 100644 --- a/litellm/videos/main.py +++ b/litellm/videos/main.py @@ -2,7 +2,7 @@ import asyncio import contextvars import json from functools import partial -from typing import Any, Coroutine, Dict, List, Literal, Optional, Union, overload +from typing import Coroutine, Dict, List, Literal, Optional, Union, overload import litellm from litellm.constants import DEFAULT_VIDEO_ENDPOINT_MODEL @@ -40,9 +40,9 @@ async def avideo_generation( custom_llm_provider=None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> VideoObject: """ @@ -126,13 +126,13 @@ def video_generation( user: Optional[str] = None, timeout: int = 600, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, *, avideo_generation: Literal[True], - **kwargs: Any, -) -> Coroutine[Any, Any, VideoObject]: + **kwargs: object, +) -> Coroutine[object, object, VideoObject]: ... @@ -146,12 +146,12 @@ def video_generation( user: Optional[str] = None, timeout: int = 600, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, *, avideo_generation: Literal[False] = False, - **kwargs: Any, + **kwargs: object, ) -> VideoObject: ... @@ -170,13 +170,13 @@ def video_generation( custom_llm_provider=None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> Union[ VideoObject, - Coroutine[Any, Any, VideoObject], + Coroutine[object, object, VideoObject], ]: """ Maps the https://api.openai.com/v1/videos endpoint. @@ -277,13 +277,13 @@ def video_content( variant: Optional[str] = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> Union[ bytes, - Coroutine[Any, Any, bytes], + Coroutine[object, object, bytes], ]: """ Download video content from OpenAI's video API. @@ -390,9 +390,9 @@ async def avideo_content( variant: Optional[str] = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> bytes: """ @@ -461,9 +461,9 @@ async def avideo_remix( custom_llm_provider=None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> VideoObject: """ @@ -528,13 +528,13 @@ def video_remix( prompt: str, timeout: int = 600, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, *, avideo_remix: Literal[True], - **kwargs: Any, -) -> Coroutine[Any, Any, VideoObject]: + **kwargs: object, +) -> Coroutine[object, object, VideoObject]: ... @@ -544,12 +544,12 @@ def video_remix( prompt: str, timeout: int = 600, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, *, avideo_remix: Literal[False] = False, - **kwargs: Any, + **kwargs: object, ) -> VideoObject: ... @@ -564,13 +564,13 @@ def video_remix( custom_llm_provider=None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> Union[ VideoObject, - Coroutine[Any, Any, VideoObject], + Coroutine[object, object, VideoObject], ]: """ Maps the https://api.openai.com/v1/videos/{video_id}/remix endpoint. @@ -668,9 +668,9 @@ async def avideo_list( custom_llm_provider=None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> List[VideoObject]: """ @@ -744,13 +744,13 @@ def video_list( order: Optional[str] = None, timeout: int = 600, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, *, avideo_list: Literal[True], - **kwargs: Any, -) -> Coroutine[Any, Any, List[VideoObject]]: + **kwargs: object, +) -> Coroutine[object, object, List[VideoObject]]: ... @@ -761,12 +761,12 @@ def video_list( order: Optional[str] = None, timeout: int = 600, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, *, avideo_list: Literal[False] = False, - **kwargs: Any, + **kwargs: object, ) -> List[VideoObject]: ... @@ -782,13 +782,13 @@ def video_list( custom_llm_provider=None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> Union[ List[VideoObject], - Coroutine[Any, Any, List[VideoObject]], + Coroutine[object, object, List[VideoObject]], ]: """ Maps the https://api.openai.com/v1/videos endpoint. @@ -882,9 +882,9 @@ async def avideo_status( custom_llm_provider=None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> VideoObject: """ @@ -947,13 +947,13 @@ def video_status( video_id: str, timeout: int = 600, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, *, avideo_status: Literal[True], - **kwargs: Any, -) -> Coroutine[Any, Any, VideoObject]: + **kwargs: object, +) -> Coroutine[object, object, VideoObject]: ... # Overload for when avideo_status=False (returns VideoObject) @@ -962,12 +962,12 @@ def video_status( video_id: str, timeout: int = 600, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, *, avideo_status: Literal[False] = False, - **kwargs: Any, + **kwargs: object, ) -> VideoObject: ... @@ -981,13 +981,13 @@ def video_status( custom_llm_provider=None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> Union[ VideoObject, - Coroutine[Any, Any, VideoObject], + Coroutine[object, object, VideoObject], ]: """ Retrieve video status from OpenAI's video API. @@ -1097,12 +1097,12 @@ def video_status( @client async def avideo_create_character( name: str, - video: Any, + video: FileTypes, timeout=600, custom_llm_provider=None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> CharacterObject: """ @@ -1152,14 +1152,14 @@ async def avideo_create_character( @client def video_create_character( name: str, - video: Any, + video: FileTypes, timeout=600, custom_llm_provider=None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, -) -> Union[CharacterObject, Coroutine[Any, Any, CharacterObject]]: +) -> Union[CharacterObject, Coroutine[object, object, CharacterObject]]: """ Create a character from an uploaded video file. Maps to POST /v1/videos/characters @@ -1230,9 +1230,9 @@ async def avideo_get_character( character_id: str, timeout=600, custom_llm_provider=None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> CharacterObject: """ @@ -1280,11 +1280,11 @@ def video_get_character( character_id: str, timeout=600, custom_llm_provider=None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, -) -> Union[CharacterObject, Coroutine[Any, Any, CharacterObject]]: +) -> Union[CharacterObject, Coroutine[object, object, CharacterObject]]: """ Retrieve a character by ID. Maps to GET /v1/videos/characters/{character_id} @@ -1355,9 +1355,9 @@ async def avideo_edit( prompt: str, timeout=600, custom_llm_provider=None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> VideoObject: """ @@ -1407,11 +1407,11 @@ def video_edit( prompt: str, timeout=600, custom_llm_provider=None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, -) -> Union[VideoObject, Coroutine[Any, Any, VideoObject]]: +) -> Union[VideoObject, Coroutine[object, object, VideoObject]]: """ Create a video edit job. Maps to POST /v1/videos/edits @@ -1486,9 +1486,9 @@ async def avideo_extension( seconds: str, timeout=600, custom_llm_provider=None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> VideoObject: """ @@ -1540,11 +1540,11 @@ def video_extension( seconds: str, timeout=600, custom_llm_provider=None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, -) -> Union[VideoObject, Coroutine[Any, Any, VideoObject]]: +) -> Union[VideoObject, Coroutine[object, object, VideoObject]]: """ Create a video extension. Maps to POST /v1/videos/extensions diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 6fb3ed748b6..b8650eea7aa 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -1,6 +1,6 @@ { "ANN001": { - "limit": 3118 + "limit": 3104 }, "ANN002": { "limit": 69 @@ -9,10 +9,10 @@ "limit": 831 }, "ANN201": { - "limit": 2138 + "limit": 2137 }, "ANN202": { - "limit": 944 + "limit": 941 }, "ANN204": { "limit": 724 @@ -24,7 +24,7 @@ "limit": 130 }, "ANN401": { - "limit": 2009 + "limit": 1851 }, "ASYNC230": { "limit": 14 @@ -123,7 +123,7 @@ "limit": 52 }, "I001": { - "limit": 270 + "limit": 261 }, "LOG015": { "limit": 8 @@ -222,7 +222,7 @@ "limit": 38 }, "RET504": { - "limit": 716 + "limit": 702 }, "RUF010": { "limit": 874 @@ -306,7 +306,7 @@ "limit": 9 }, "TID251": { - "limit": 2652 + "limit": 2649 }, "TRY002": { "limit": 547 @@ -324,7 +324,7 @@ "limit": 879 }, "UP006": { - "limit": 12135 + "limit": 12050 }, "UP007": { "limit": 2526 @@ -360,9 +360,9 @@ "limit": 4 }, "UP037": { - "limit": 105 + "limit": 104 }, "UP045": { - "limit": 17805 + "limit": 17793 } } diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 05499e83c42..ff037a2872e 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 23250 + "limit": 23191 }, "LIT002": { - "limit": 27277 + "limit": 27276 }, "LIT003": { "limit": 292 @@ -15,7 +15,7 @@ "limit": 0 }, "LIT006": { - "limit": 1108 + "limit": 1106 }, "LIT007": { "limit": 0 @@ -24,6 +24,6 @@ "limit": 1004 }, "LIT009": { - "limit": 2473 + "limit": 2467 } } From 48b3d1889395bb63c0a3a458167ed65073538b5d Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 1 Aug 2026 09:28:02 -0700 Subject: [PATCH 023/123] feat(ui): note in the add-member modal that search covers existing users only Both fields select from a server-side search over existing accounts, so a typed-in address or id never becomes a value. Say so up front rather than letting the form look like it accepts a new user and fail on submit. Applies to the organization member modal too, which shares this component. --- .../common_components/user_search_modal.test.tsx | 10 ++++++++++ .../components/common_components/user_search_modal.tsx | 10 +++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx b/ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx index 72b0e10e5d6..634c78d28d0 100644 --- a/ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx +++ b/ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx @@ -65,4 +65,14 @@ describe("UserSearchModal", () => { expect(userFilterUICall).not.toHaveBeenCalled(); }); + + it("tells the user that only existing accounts can be selected", () => { + renderModal(); + + const notice = screen.getByRole("alert"); + expect(notice).toHaveTextContent(/users that already exist/i); + expect(notice).toHaveTextContent(/ask a proxy admin to create their account first/i); + // info, not warning: a warning here would read as an error state on an empty form + expect(notice.className).toMatch(/ant-alert-info/); + }); }); diff --git a/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx b/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx index fafafd8e5d5..9c1d64a1f86 100644 --- a/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx +++ b/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx @@ -1,5 +1,5 @@ import { useState } from "react"; -import { Modal, Form, Button, Select, Tooltip } from "antd"; +import { Modal, Form, Button, Select, Tooltip, Alert } from "antd"; import { UserAddOutlined } from "@ant-design/icons"; import { useDebouncedCallback } from "@tanstack/react-pacer/debouncer"; import { userFilterUICall } from "@/components/networking"; @@ -140,6 +140,14 @@ const UserSearchModal: React.FC = ({ role: defaultRole, }} > + + onChange?.(e.target.value)} + > + + {organizations?.map((org) => ( + + ))} + + + + ), +})); + vi.mock("./ModelSelect/ModelSelect", () => { const ModelSelect = ({ value, onChange }: { value: string[]; onChange: (value: string[]) => void }) => ( Date: Mon, 3 Aug 2026 13:03:46 -0700 Subject: [PATCH 088/123] fix(ui): block Playground page for viewer roles on direct URL access (#35676) --- .../playground/components/chat_ui/ChatUI.tsx | 12 +--- .../app/(dashboard)/playground/page.test.tsx | 62 +++++++++++++++++++ .../src/app/(dashboard)/playground/page.tsx | 12 ++++ ui/litellm-dashboard/src/utils/roles.ts | 2 + 4 files changed, 77 insertions(+), 11 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/page.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx index 684814bfe5b..e7261db6260 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx @@ -22,7 +22,7 @@ import { UserOutlined, } from "@ant-design/icons"; import { Card, Text, TextInput, Title, Button as TremorButton } from "@tremor/react"; -import { Button, Input, Modal, Popover, Select, Spin, Tooltip, Typography, Upload } from "antd"; +import { Button, Input, Modal, Popover, Select, Spin, Tooltip, Upload } from "antd"; import React, { useEffect, useRef, useState } from "react"; import ReactMarkdown from "react-markdown"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; @@ -1016,16 +1016,6 @@ const ChatUI: React.FC = ({ NotificationsManager.success("Chat history cleared."); }; - if (userRole && userRole === "Admin Viewer") { - const { Title, Paragraph } = Typography; - return ( -
- Access Denied - Ask your proxy admin for access to test models -
- ); - } - const onModelChange = (value: string) => { setSelectedModel(value); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/page.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/page.test.tsx new file mode 100644 index 00000000000..54e99d9db29 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/page.test.tsx @@ -0,0 +1,62 @@ +import { render, screen } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import PlaygroundPage from "./page"; + +const authState = { userRole: "Admin" }; + +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => ({ + token: "token-1", + accessToken: "sk-test", + userId: "user-1", + userRole: authState.userRole, + disabledPersonalKeyCreation: false, + }), +})); + +vi.mock("@/utils/proxyUtils", () => ({ + fetchProxySettings: vi.fn().mockResolvedValue(null), +})); + +vi.mock("@/app/(dashboard)/playground/components/chat_ui/ChatUI", () => ({ + default: () =>
, +})); + +vi.mock("@/app/(dashboard)/playground/components/compareUI/CompareUI", () => ({ + default: () =>
, +})); + +vi.mock("@/app/(dashboard)/playground/components/complianceUI/ComplianceUI", () => ({ + default: () =>
, +})); + +vi.mock("@/app/(dashboard)/playground/components/chat_ui/AgentBuilderView", () => ({ + default: () =>
, +})); + +describe("PlaygroundPage role guard", () => { + beforeEach(() => { + authState.userRole = "Admin"; + }); + + it.each(["Internal Viewer", "Admin Viewer"])("blocks the entire playground for %s", (role) => { + authState.userRole = role; + render(); + + expect(screen.getByText("Access Denied")).toBeInTheDocument(); + expect(screen.queryByRole("tab")).not.toBeInTheDocument(); + expect(screen.queryByTestId("chat-ui")).not.toBeInTheDocument(); + expect(screen.queryByTestId("compare-ui")).not.toBeInTheDocument(); + expect(screen.queryByTestId("compliance-ui")).not.toBeInTheDocument(); + expect(screen.queryByTestId("agent-builder")).not.toBeInTheDocument(); + }); + + it.each(["Admin", "Internal User", "Org Admin"])("renders the playground for %s", (role) => { + authState.userRole = role; + render(); + + expect(screen.queryByText("Access Denied")).not.toBeInTheDocument(); + expect(screen.getByRole("tab", { name: "Chat" })).toBeInTheDocument(); + expect(screen.getByTestId("chat-ui")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/page.tsx index bd3c0e31456..8986084b1a7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/page.tsx @@ -9,6 +9,7 @@ import { TabGroup, TabList, Tab, TabPanels, TabPanel } from "@tremor/react"; import { DeprecationBanner } from "@/components/DeprecationBanner"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { fetchProxySettings } from "@/utils/proxyUtils"; +import { isViewOnlyRole } from "@/utils/roles"; interface ProxySettings { PROXY_BASE_URL?: string; @@ -35,6 +36,17 @@ export default function PlaygroundPage() { initializeProxySettings(); }, [accessToken]); + if (isViewOnlyRole(userRole)) { + return ( +
+

Access Denied

+

+ Your role does not have access to the Playground. Ask your proxy admin for access to test models. +

+
+ ); + } + return (
diff --git a/ui/litellm-dashboard/src/utils/roles.ts b/ui/litellm-dashboard/src/utils/roles.ts index 38f8496c2ae..90c77a61b2d 100644 --- a/ui/litellm-dashboard/src/utils/roles.ts +++ b/ui/litellm-dashboard/src/utils/roles.ts @@ -13,6 +13,8 @@ export const rolesWithWriteAccess = ["Internal User", "Admin", "proxy_admin"]; // Per the Admin Viewer principle: read parity with Proxy Admin, no writes, // no cost-incurring actions (Playground stays gated by `rolesWithWriteAccess`). export const rolesAllowedToViewWriteScopedPages = [...rolesWithWriteAccess, "Admin Viewer", "proxy_admin_viewer"]; +export const viewOnlyRoles = ["Admin Viewer", "Internal Viewer"]; +export const isViewOnlyRole = (role: string): boolean => viewOnlyRoles.includes(role); // Helper function to check if a role is in all_admin_roles export const isAdminRole = (role: string): boolean => { From 3bc4989ce4f24a1e6a7de92138759a4860f8e333 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 3 Aug 2026 13:15:32 -0700 Subject: [PATCH 089/123] chore(ui): update brace-expansion and postcss to current patch releases The dashboard pins both packages exactly in `overrides`, so the lockfile stays on whatever those pins say. Move brace-expansion from 5.0.8 to 5.0.9 and postcss from 8.5.22 to 8.5.23, both upstream patch releases, and regenerate the lockfile. `npm ci`, `next build`, and the 5888-test vitest suite all pass on the updated lockfile. --- ui/litellm-dashboard/package-lock.json | 14 +++++++------- ui/litellm-dashboard/package.json | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index c9953cce2ab..a1bd63151b4 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -67,7 +67,7 @@ "jsdom": "27.4.0", "knip": "5.83.1", "openapi-typescript": "7.13.0", - "postcss": "8.5.22", + "postcss": "8.5.23", "prettier": "3.2.5", "tailwindcss": "4.3.2", "tw-animate-css": "1.4.0", @@ -5529,9 +5529,9 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz", - "integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "dev": true, "license": "MIT", "dependencies": { @@ -11064,9 +11064,9 @@ } }, "node_modules/postcss": { - "version": "8.5.22", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.22.tgz", - "integrity": "sha512-KBDEIpLrvpv16pp3K0Fw+UCoZfopFjjgeB+0tA/aaThfEE74kKDLrgg603YvOWJyg3+WYtyq3xYsQWsIyZlPqQ==", + "version": "8.5.23", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.23.tgz", + "integrity": "sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==", "funding": [ { "type": "opencollective", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 32d93729dbe..4760b622f9b 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -79,7 +79,7 @@ "jsdom": "27.4.0", "knip": "5.83.1", "openapi-typescript": "7.13.0", - "postcss": "8.5.22", + "postcss": "8.5.23", "prettier": "3.2.5", "tailwindcss": "4.3.2", "tw-animate-css": "1.4.0", @@ -90,13 +90,13 @@ "overrides": { "prismjs": "1.30.0", "js-yaml": "4.3.0", - "brace-expansion": "5.0.8", + "brace-expansion": "5.0.9", "glob": "13.0.0", "minimatch": "10.2.4", "ws": "8.21.0", "braces": "3.0.3", "axios": "1.13.6", - "postcss": "8.5.22", + "postcss": "8.5.23", "esbuild": "0.28.1", "date-fns": "^4.4.0", "sharp": "^0.35.0" From cad319a862e744f5598b99b3e0c32bf33c185623 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 3 Aug 2026 13:24:44 -0700 Subject: [PATCH 090/123] chore(deps): update gitpython to 3.1.57 gitpython arrives transitively through mlflow-skinny, which accepts >=3.1.9,<4, so this is a lock-only move with no pyproject change. Relocked with `uv lock --upgrade-package gitpython`; gitpython is the only package whose version changed. `uv sync --all-groups --all-extras` and tests/test_litellm/integrations/test_mlflow.py pass on the result. --- uv.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uv.lock b/uv.lock index 0bfe9208872..15e65c9dffd 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-07-29T22:09:54.255381Z" +exclude-newer = "2026-07-31T20:23:04.658774Z" exclude-newer-span = "P3D" [manifest] @@ -2378,14 +2378,14 @@ wheels = [ [[package]] name = "gitpython" -version = "3.1.55" +version = "3.1.57" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "gitdb" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b2/ab/ba0d29f2fa2277ed6256b2ac09003494045355f3a10bf32f351761287870/gitpython-3.1.55.tar.gz", hash = "sha256:781e3b1624dad81b24e9524bf0297b69786a0706db2cbceec1e2b05c38e5152f", size = 225071, upload-time = "2026-07-23T02:52:43.246Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/0d/132ed135c871b6bf91adf16a0e43797cd535b81d4973b5d09291c54fc5ee/gitpython-3.1.57.tar.gz", hash = "sha256:c493ec57c0ef6b19743798b6a5af859c71814b524e7e6f97baa2f8e658961488", size = 225898, upload-time = "2026-07-26T07:33:26.351Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/20/6a/d3b8208d2f8aac66abe8ccc1c23fa2c89464ec42cc71a601e95d05902428/gitpython-3.1.55-py3-none-any.whl", hash = "sha256:7c9ec1e69c158c081632ab35c41471e302c96db2ae42165036a5d2403378812e", size = 216590, upload-time = "2026-07-23T02:52:41.932Z" }, + { url = "https://files.pythonhosted.org/packages/41/6e/2139de986d9c7c3ac86f1f8be43858ce90bdfe2f7175e6c80c650ba15242/gitpython-3.1.57-py3-none-any.whl", hash = "sha256:4ccf7d73c10f5c9e76043fbb2675ac5a1b3ff5b41e648f56bcbed5f63792ecaf", size = 217151, upload-time = "2026-07-26T07:33:24.838Z" }, ] [[package]] From 66bc70365f69ce77288689d681557d5cf539a450 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Mon, 3 Aug 2026 13:28:38 -0700 Subject: [PATCH 091/123] fix(caching): close evicted LLM clients so their connections are reclaimed (#35492) An evicted client was left for the garbage collector, but every OpenAI/Azure SDK client is a reference cycle, so nothing freed the client or its pooled TCP connections until a generational sweep ran. Driving 2000 azure calls through the official image with no forced collection, live clients and open sockets climbed from 202 to 1361 while the cache stayed at its 200-entry bound, and RSS grew 279 MB to 456 MB against a TLS upstream. Closing on eviction is what caused the earlier 'Cannot send a request, as the client has been closed' regression, so an evicted client litellm created is now closed only once a grace window has passed, by which point any request that was already holding it has finished. A client the caller supplied is never closed, since litellm does not own its lifecycle. Resolves LIT-4883 --- litellm/caching/evicted_client_closer.py | 276 ++++++++++++ litellm/caching/llm_caching_handler.py | 45 +- litellm/constants.py | 10 + litellm/llms/azure/common_utils.py | 2 + litellm/llms/custom_httpx/http_handler.py | 2 + litellm/llms/openai/common_utils.py | 23 +- litellm/llms/openai/openai.py | 10 +- .../caching/test_evicted_client_closer.py | 409 ++++++++++++++++++ .../caching/test_llm_caching_handler.py | 66 +++ .../llms/azure/test_azure_common_utils.py | 71 +++ .../llms/openai/test_openai_common_utils.py | 72 +++ 11 files changed, 975 insertions(+), 11 deletions(-) create mode 100644 litellm/caching/evicted_client_closer.py create mode 100644 tests/test_litellm/caching/test_evicted_client_closer.py diff --git a/litellm/caching/evicted_client_closer.py b/litellm/caching/evicted_client_closer.py new file mode 100644 index 00000000000..bca9656b252 --- /dev/null +++ b/litellm/caching/evicted_client_closer.py @@ -0,0 +1,276 @@ +""" +Deferred close of HTTP/SDK clients that the LLM client cache has evicted. + +Eviction only drops the cache's reference to a client. Every OpenAI/Azure SDK +client is a reference cycle (each resource namespace holds the client back), so +an evicted client and its pooled TCP connections survive until a generational +collection runs, which under load is thousands of requests later. + +Closing at eviction time is not an option: a request that was handed the client +just before it was evicted is still using it, and closing it underneath that +request raises ``RuntimeError: Cannot send a request, as the client has been +closed.`` + +So an evicted client is closed once two conditions hold. A grace window must +have passed since its eviction, which covers a request that holds the client +but is momentarily not on the wire, and the client must report no connection in +flight. The second condition is what keeps the first honest: a request may run +for ``litellm.request_timeout`` seconds, 6000 by default, and a streaming +response is bounded only by how long the upstream keeps sending, so no deadline +on its own can promise that a request has finished. + +Only clients litellm itself created are closed; a client the caller supplied is +left alone because litellm does not own its lifecycle. + +A client that closes synchronously is closed from wherever the cache is next +used. One whose close is a coroutine needs the event loop it was evicted on, so +it waits for a call from that loop rather than having work scheduled onto a loop +it does not belong to. Queued clients are therefore bucketed by what it takes to +close them, and each bucket is ordered by deadline, so a reap walks the entries +that are due rather than the whole queue. + +The queue holds its clients weakly, so waiting out a grace window never keeps +alive anything the collector would have reclaimed first. +""" + +import asyncio +import inspect +import threading +import time +import weakref +from collections import deque +from collections.abc import Awaitable, Callable, Iterator +from dataclasses import dataclass, replace + +from litellm.constants import ( + EVICTED_LLM_CLIENT_CLOSE_GRACE_SECONDS, + EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING, +) + +_CLOSABLE_ANYWHERE = "closable-anywhere" +_CLOSABLE_ON_ANY_LOOP = "closable-on-any-loop" + +_BucketKey = str | int + + +@dataclass(frozen=True, slots=True) +class _PendingClose: + """A queued close. + + The client is held weakly, so queueing one never keeps alive anything the + collector would otherwise have reclaimed first. + + ``needs_loop`` is set for a client whose close is a coroutine; those can only + be closed from the event loop they were evicted on, recorded in ``loop_id``. + A client that closes synchronously carries neither constraint. + """ + + client_ref: "weakref.ref[object]" + loop_id: int | None + needs_loop: bool + close_after: float + + +def _bucket_key(pending: _PendingClose) -> _BucketKey: + """Which reaps can close this entry: any at all, any running a loop, or one loop's.""" + if not pending.needs_loop: + return _CLOSABLE_ANYWHERE + if pending.loop_id is None: + return _CLOSABLE_ON_ANY_LOOP + return pending.loop_id + + +def _running_loop_id() -> int | None: + try: + return id(asyncio.get_running_loop()) + except RuntimeError: + return None + + +def _close_function(client: object) -> Callable[[], object] | None: + close_fn: Callable[[], object] | None = getattr(client, "aclose", None) or getattr(client, "close", None) + return close_fn + + +def _transport_of(client: object) -> object: + """The httpx transport behind an SDK wrapper, a litellm handler, or a bare client.""" + for holder in (getattr(client, "_client", None), getattr(client, "client", None), client): + transport: object = getattr(holder, "_transport", None) + if transport is not None: + return transport + return None + + +def _connection_is_idle(connection: object) -> bool: + """A pooled connection is idle unless it is servicing a request.""" + is_idle: object = getattr(connection, "is_idle", None) + return bool(is_idle()) if callable(is_idle) else True + + +def _pool_has_busy_connection(transport: object) -> bool | None: + """Whether the httpcore pool behind the transport is servicing a request. + + ``None`` when there is no such pool, so the caller can ask the other backend. + """ + pooled: object = getattr(getattr(transport, "_pool", None), "connections", None) + if not isinstance(pooled, (list, tuple)): + return None + return any( + not _connection_is_idle(connection) # pyright: ignore[reportUnknownArgumentType] # untyped pool list + for connection in pooled # pyright: ignore[reportUnknownVariableType] # untyped pool list + ) + + +def _has_connection_in_flight(client: object) -> bool: + """Whether the client is servicing a request right now. + + Both connection backends litellm uses already account for the connections + they have handed out, so this reads the client's own lease accounting rather + than inferring it from elapsed time: httpcore reports a non-idle connection + for the whole of a response including a stream, and aiohttp holds the + connection in ``_acquired`` over the same span. + + A client that cannot answer is reported as idle, which leaves the grace + window as the only guard, exactly as it was before this check existed. + """ + try: + transport = _transport_of(client) + pooled_busy = _pool_has_busy_connection(transport) + if pooled_busy is not None: + return pooled_busy + session: object = getattr(transport, "client", None) + return bool(getattr(getattr(session, "connector", None), "_acquired", None)) + except Exception: # noqa: BLE001 - a client that cannot report its state is treated as idle + return False + + +async def _close_quietly(closing: Awaitable[object]) -> None: + try: + await closing + except Exception: # noqa: BLE001 - a discarded client's close must never surface to callers + pass + + +class EvictedClientCloser: + """Closes evicted, litellm-owned clients once they are idle and out of grace.""" + + def __init__( + self, + grace_seconds: float = EVICTED_LLM_CLIENT_CLOSE_GRACE_SECONDS, + max_pending: int = EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING, + clock: Callable[[], float] = time.monotonic, + ) -> None: + self._grace_seconds = grace_seconds + self._max_pending = max_pending + self._clock = clock + self._owned: weakref.WeakSet[object] = weakref.WeakSet() + self._buckets: dict[_BucketKey, deque[_PendingClose]] = {} # mutable-ok: deadline-ordered queues + self._pending_count = 0 + self._queue_lock = threading.Lock() # the cache is reachable from every worker thread's loop + self._close_tasks: set[asyncio.Task[None]] = set() # mutable-ok: strong refs to running closes + + def mark_owned(self, client: object) -> None: + """Record that litellm created this client, so it may be closed on eviction.""" + try: + self._owned.add(client) + except TypeError: + pass # values that cannot be weak-referenced are never litellm clients + + def _is_owned(self, client: object) -> bool: + try: + return client in self._owned + except TypeError: + return False # unhashable values are never litellm clients + + def schedule(self, client: object) -> None: + """Queue an evicted client for closing once it is idle and out of grace. + + Past ``max_pending`` the client is left to the collector instead, so a + workload that churns the cache cannot grow this queue without bound. + Every queued entry comes due within one grace window, so the capacity it + occupies is returned within that window rather than held. + """ + if client is None or not self._is_owned(client): + return + close_fn = _close_function(client) + if close_fn is None: + return + if self._pending_count >= self._max_pending: + return + self._enqueue( + _PendingClose( + client_ref=weakref.ref(client), + loop_id=_running_loop_id(), + needs_loop=inspect.iscoroutinefunction(close_fn), + close_after=self._clock() + self._grace_seconds, + ) + ) + + def reap(self) -> None: + """Close every queued client that is due, idle, and closable from here. + + Called from the cache's read path, so the empty-queue exit comes first and + the work done past it is proportional to what is due, not to the queue. + """ + if not self._pending_count: + return + now = self._clock() + for pending in self._take_due(_running_loop_id(), now): + client = pending.client_ref() + if client is None: + continue + if _has_connection_in_flight(client): + self._enqueue(replace(pending, close_after=now + self._grace_seconds)) + continue + self._close(client) + + @property + def pending_count(self) -> int: + return self._pending_count + + def _enqueue(self, pending: _PendingClose) -> None: + """Append to the entry's bucket, dropping any dead entries it queues behind. + + Deadlines only ever move forward, so appending keeps each bucket ordered + by deadline, and entries whose client the collector already took sit at + the front rather than having to be searched for. + """ + with self._queue_lock: + bucket = self._buckets.setdefault(_bucket_key(pending), deque()) # mutable-ok: FIFO by design + while bucket and bucket[0].client_ref() is None: + bucket.popleft() + self._pending_count -= 1 + bucket.append(pending) + self._pending_count += 1 + + def _take_due(self, loop_id: int | None, now: float) -> tuple[_PendingClose, ...]: + buckets = (_CLOSABLE_ANYWHERE,) if loop_id is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_id) + with self._queue_lock: + return tuple(pending for key in buckets for pending in self._drain_locked(key, now)) + + def _drain_locked(self, key: _BucketKey, now: float) -> Iterator[_PendingClose]: + bucket = self._buckets.get(key) + if bucket is None: + return + while bucket and bucket[0].close_after <= now: + self._pending_count -= 1 + yield bucket.popleft() + if not bucket: + del self._buckets[key] + + def _close(self, client: object) -> None: + close_fn = _close_function(client) + if close_fn is None: + return + try: + closing = close_fn() + except Exception: # noqa: BLE001 - a discarded client's close must never surface to callers + return + if not inspect.isawaitable(closing): + return + task = asyncio.get_running_loop().create_task(_close_quietly(closing)) + self._close_tasks.add(task) + task.add_done_callback(self._close_tasks.discard) + + +default_evicted_client_closer = EvictedClientCloser() diff --git a/litellm/caching/llm_caching_handler.py b/litellm/caching/llm_caching_handler.py index c2274713bb9..7eae8ee3749 100644 --- a/litellm/caching/llm_caching_handler.py +++ b/litellm/caching/llm_caching_handler.py @@ -4,21 +4,44 @@ Add the event loop to the cache key, to prevent event loop closed errors. import asyncio +from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer from .in_memory_cache import InMemoryCache class LLMClientCache(InMemoryCache): """Cache for LLM HTTP clients (OpenAI, Azure, httpx, etc.). - IMPORTANT: This cache intentionally does NOT close clients on eviction. - Evicted clients may still be in use by in-flight requests. Closing them - eagerly causes ``RuntimeError: Cannot send a request, as the client has - been closed.`` errors in production after the TTL (1 hour) expires. + An evicted client is never closed on the spot: a request handed the client + just before eviction is still using it, and closing it there raises + ``RuntimeError: Cannot send a request, as the client has been closed.`` - Clients that are no longer referenced will be garbage-collected normally. - For explicit shutdown cleanup, use ``close_litellm_async_clients()``. + Nor can eviction be left to rely on garbage collection. The SDK clients are + reference cycles, so an evicted client and its open TCP connections survive + until a generational collection runs. Instead a client litellm created is + handed to ``EvictedClientCloser``, which closes it once a grace window has + passed. Clients the caller supplied are left untouched. """ + def __init__( + self, + max_size_in_memory: int | None = 200, + default_ttl: int | None = 600, + max_size_per_item: int | None = 1024, + evicted_client_closer: EvictedClientCloser | None = None, + ): + super().__init__( + max_size_in_memory=max_size_in_memory, + default_ttl=default_ttl, + max_size_per_item=max_size_per_item, + ) + self.evicted_client_closer = evicted_client_closer or default_evicted_client_closer + + def _remove_key(self, key: str) -> None: + evicted: object = self.cache_dict.get(key) + super()._remove_key(key) + self.evicted_client_closer.schedule(evicted) + self.evicted_client_closer.reap() + def update_cache_key_with_event_loop(self, key): """ Add the event loop to the cache key, to prevent event loop closed errors. @@ -31,16 +54,22 @@ class LLMClientCache(InMemoryCache): except RuntimeError: # handle no current running event loop return key - def set_cache(self, key, value, **kwargs): + def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs): + """``litellm_owned_client`` marks a client litellm built, so it may be closed once evicted.""" + if litellm_owned_client: + self.evicted_client_closer.mark_owned(value) key = self.update_cache_key_with_event_loop(key) return super().set_cache(key, value, **kwargs) - async def async_set_cache(self, key, value, **kwargs): + async def async_set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs): + if litellm_owned_client: + self.evicted_client_closer.mark_owned(value) key = self.update_cache_key_with_event_loop(key) return await super().async_set_cache(key, value, **kwargs) def get_cache(self, key, **kwargs): key = self.update_cache_key_with_event_loop(key) + self.evicted_client_closer.reap() return super().get_cache(key, **kwargs) diff --git a/litellm/constants.py b/litellm/constants.py index d46f62af000..06421e6ed6a 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -197,6 +197,16 @@ RUNWAYML_POLLING_TIMEOUT = int(os.getenv("RUNWAYML_POLLING_TIMEOUT", 600)) # 10 ########## Networking constants ############################################################## _DEFAULT_TTL_FOR_HTTPX_CLIENTS = 3600 # 1 hour, re-use the same httpx client for 1 hour +# The earliest an evicted, litellm-created client may be closed. A request handed the +# client just before eviction is still using it, so nothing is closed inside this window; +# past it, the client is closed once it reports no connection in flight. +EVICTED_LLM_CLIENT_CLOSE_GRACE_SECONDS = 900 + +# How many evicted clients may be queued for closing at once. Past this, an evicted client +# is left to the collector rather than letting a cache-churning workload grow the queue +# without bound. Each queued entry is ~100 bytes and comes due within one grace window. +EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING = 10_000 + # Aiohttp connection pooling - prevents memory leaks from unbounded connection growth # Set to 0 for unlimited (not recommended for production) AIOHTTP_CONNECTOR_LIMIT = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 1000)) diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index 8e0bd363a8a..8db422e00ff 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -508,6 +508,8 @@ class BaseAzureLLM(BaseOpenAILLM): openai_client=openai_client, client_initialization_params=client_initialization_params, client_type="azure", + litellm_owned_client=client is None + and self.owns_wrapped_http_client(azure_client_params.get("http_client")), ) return openai_client diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 046840e6fd0..3e34b483002 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -1411,6 +1411,7 @@ def get_async_httpx_client( key=_cache_key_name, value=_new_client, ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS, + litellm_owned_client=True, ) return _new_client @@ -1456,5 +1457,6 @@ def _get_httpx_client(params: dict | None = None) -> HTTPHandler: key=_cache_key_name, value=_new_client, ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS, + litellm_owned_client=True, ) return _new_client diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index e72680f387d..082764df208 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -134,13 +134,33 @@ class BaseOpenAILLM: _cached_client = litellm.in_memory_llm_clients_cache.get_cache(_cache_key) return _cached_client + @staticmethod + def owns_wrapped_http_client(http_client: Optional[Union[httpx.Client, httpx.AsyncClient]]) -> bool: + """Whether litellm may close an SDK client built around ``http_client``. + + ``_get_async_http_client`` / ``_get_sync_http_client`` hand back + ``litellm.aclient_session`` / ``litellm.client_session`` when the caller + configured one. The SDK's ``close()`` closes whatever http client it was + given, so an SDK client wrapping one of those shared sessions must never be + closed on eviction; the caller goes on using the session. ``None`` means the + SDK built its own http client, which litellm does own. + """ + if http_client is None: + return True + return http_client is not litellm.aclient_session and http_client is not litellm.client_session + @staticmethod def set_cached_openai_client( openai_client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI, client_type: Literal["openai", "azure"], client_initialization_params: dict, + litellm_owned_client: bool = False, ): - """Stores the OpenAI client in the in-memory cache for _DEFAULT_TTL_FOR_HTTPX_CLIENTS SECONDS""" + """Stores the OpenAI client in the in-memory cache for _DEFAULT_TTL_FOR_HTTPX_CLIENTS SECONDS + + ``litellm_owned_client`` says litellm built this client, so the cache may close it once it + is evicted. A client the caller supplied stays open, since litellm does not own it. + """ _cache_key = BaseOpenAILLM.get_openai_client_cache_key( client_initialization_params=client_initialization_params, client_type=client_type, @@ -149,6 +169,7 @@ class BaseOpenAILLM: key=_cache_key, value=openai_client, ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS, + litellm_owned_client=litellm_owned_client, ) @staticmethod diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 845ad22589f..7096cdbf8fd 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -366,11 +366,16 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): if cached_client: if isinstance(cached_client, OpenAI) or isinstance(cached_client, AsyncOpenAI): return cached_client + http_client: Optional[Union[httpx.Client, httpx.AsyncClient]] = ( + OpenAIChatCompletion._get_async_http_client(shared_session=shared_session) + if is_async + else OpenAIChatCompletion._get_sync_http_client() + ) if is_async: _new_client: OpenAI | AsyncOpenAI = AsyncOpenAI( api_key=api_key, base_url=api_base, - http_client=OpenAIChatCompletion._get_async_http_client(shared_session=shared_session), + http_client=http_client, timeout=timeout, max_retries=max_retries, organization=organization, @@ -379,7 +384,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): _new_client = OpenAI( api_key=api_key, base_url=api_base, - http_client=OpenAIChatCompletion._get_sync_http_client(), + http_client=http_client, timeout=timeout, max_retries=max_retries, organization=organization, @@ -390,6 +395,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): openai_client=_new_client, client_initialization_params=client_initialization_params, client_type="openai", + litellm_owned_client=self.owns_wrapped_http_client(http_client), ) return _new_client diff --git a/tests/test_litellm/caching/test_evicted_client_closer.py b/tests/test_litellm/caching/test_evicted_client_closer.py new file mode 100644 index 00000000000..a08fd58079d --- /dev/null +++ b/tests/test_litellm/caching/test_evicted_client_closer.py @@ -0,0 +1,409 @@ +""" +Tests for EvictedClientCloser. + +An evicted client must stay open long enough for a request that already holds it +to finish, and must then actually be closed, otherwise its connection pool is +retained until a generational collection runs. A client the caller supplied is +never closed, because litellm does not own its lifecycle. +""" + +import asyncio +import gc +import weakref + +import httpx +import pytest + +from litellm.caching.evicted_client_closer import EvictedClientCloser +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + +class FakeClock: + """Hand-advanced monotonic clock, so grace windows need no real waiting.""" + + def __init__(self) -> None: + self.now = 1000.0 + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + self.now += seconds + + +class AsyncClient: + def __init__(self) -> None: + self.closed = False + + async def close(self) -> None: + self.closed = True + + +class SyncClient: + def __init__(self) -> None: + self.closed = False + + def close(self) -> None: + self.closed = True + + +class CountingDeadline(float): + """A clock reading that tallies every deadline comparison made against it. + + Deadline comparisons are the work a reap does, so counting them says whether + that work tracks the entries that are due or the size of the whole queue. + """ + + comparisons = 0 + + def __add__(self, other: float) -> "CountingDeadline": + return CountingDeadline(float(self) + other) + + def __le__(self, other: float) -> bool: + CountingDeadline.comparisons += 1 + return float(self) <= float(other) + + def __gt__(self, other: float) -> bool: + CountingDeadline.comparisons += 1 + return float(self) > float(other) + + +def make_closer(clock: FakeClock, grace_seconds: float = 60.0) -> EvictedClientCloser: + return EvictedClientCloser(grace_seconds=grace_seconds, clock=clock) + + +async def _trickling_upstream(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + """Serves a chunked body slowly, so a request stays on the wire long enough to observe.""" + await reader.read(4096) + writer.write(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n") + await writer.drain() + for _ in range(6): + writer.write(b"5\r\nhello\r\n") + await writer.drain() + await asyncio.sleep(0.1) + writer.write(b"0\r\n\r\n") + await writer.drain() + + +@pytest.mark.asyncio +async def test_owned_client_is_closed_once_the_grace_window_elapses(): + clock = FakeClock() + closer = make_closer(clock) + client = AsyncClient() + + closer.mark_owned(client) + closer.schedule(client) + clock.advance(61.0) + closer.reap() + await asyncio.sleep(0.05) + + assert client.closed is True + assert closer.pending_count == 0 + + +@pytest.mark.asyncio +async def test_owned_client_stays_open_inside_the_grace_window(): + """A request handed the client just before eviction is still using it.""" + clock = FakeClock() + closer = make_closer(clock) + client = AsyncClient() + + closer.mark_owned(client) + closer.schedule(client) + clock.advance(59.0) + closer.reap() + await asyncio.sleep(0.05) + + assert client.closed is False + assert closer.pending_count == 1 + + +@pytest.mark.asyncio +async def test_caller_supplied_client_is_never_closed(): + clock = FakeClock() + closer = make_closer(clock) + client = AsyncClient() + + closer.schedule(client) + clock.advance(3600.0) + closer.reap() + await asyncio.sleep(0.05) + + assert client.closed is False + assert closer.pending_count == 0 + + +@pytest.mark.asyncio +async def test_sync_client_is_closed_once_the_grace_window_elapses(): + clock = FakeClock() + closer = make_closer(clock) + client = SyncClient() + + closer.mark_owned(client) + closer.schedule(client) + clock.advance(61.0) + closer.reap() + + assert client.closed is True + + +@pytest.mark.asyncio +async def test_a_failing_close_does_not_propagate_or_block_the_others(): + class ExplodingClient: + async def close(self) -> None: + raise RuntimeError("connection already gone") + + clock = FakeClock() + closer = make_closer(clock) + exploding, healthy = ExplodingClient(), AsyncClient() + + for client in (exploding, healthy): + closer.mark_owned(client) + closer.schedule(client) + clock.advance(61.0) + closer.reap() + await asyncio.sleep(0.05) + + assert healthy.closed is True + + +@pytest.mark.asyncio +async def test_an_unhashable_cached_value_does_not_break_eviction(): + """The cache holds arbitrary values; an ownership test must never raise on one.""" + + class Unhashable: + __hash__ = None # pyright: ignore[reportAssignmentType] # unhashable by construction + + clock = FakeClock() + closer = make_closer(clock) + + closer.mark_owned(Unhashable()) + closer.schedule(Unhashable()) + + assert closer.pending_count == 0 + + +@pytest.mark.asyncio +async def test_values_with_nothing_to_close_are_never_queued(): + """The cache holds plain values too; those have nothing to reclaim.""" + + class NotAClient: + pass + + clock = FakeClock() + closer = make_closer(clock) + value = NotAClient() + + closer.mark_owned(value) + closer.schedule(value) + + assert closer.pending_count == 0 + + +@pytest.mark.asyncio +async def test_a_queued_client_is_not_kept_alive_by_the_queue(): + """Waiting out a grace window must not retain what the collector would free first.""" + clock = FakeClock() + closer = make_closer(clock) + client = AsyncClient() + gone = weakref.ref(client) + + closer.mark_owned(client) + closer.schedule(client) + del client + gc.collect() + + assert gone() is None, "the pending queue is holding the client alive" + + clock.advance(61.0) + closer.reap() + assert closer.pending_count == 0 + + +def test_sync_client_evicted_outside_an_event_loop_is_still_closed(): + """The sync httpx handler is cached and evicted from call sites with no loop.""" + clock = FakeClock() + closer = make_closer(clock) + client = SyncClient() + + closer.mark_owned(client) + closer.schedule(client) + assert closer.pending_count == 1 + + clock.advance(61.0) + closer.reap() + + assert client.closed is True + assert closer.pending_count == 0 + + +@pytest.mark.asyncio +async def test_an_async_client_waits_for_a_loop_rather_than_being_dropped(): + clock = FakeClock() + closer = make_closer(clock) + client = AsyncClient() + closer.mark_owned(client) + + def schedule_outside_a_loop() -> None: + closer.schedule(client) + clock.advance(61.0) + closer.reap() + + await asyncio.to_thread(schedule_outside_a_loop) + assert client.closed is False, "no loop was running, so it could not have been closed" + assert closer.pending_count == 1 + + closer.reap() + await asyncio.sleep(0.05) + + assert client.closed is True + + +@pytest.mark.asyncio +async def test_a_client_evicted_on_another_event_loop_is_left_alone(): + """Closing a client bound to a different loop would schedule work on that loop.""" + clock = FakeClock() + closer = make_closer(clock) + client = AsyncClient() + closer.mark_owned(client) + + def schedule_on_its_own_loop() -> None: + asyncio.run(_schedule()) + + async def _schedule() -> None: + closer.schedule(client) + + await asyncio.to_thread(schedule_on_its_own_loop) + assert closer.pending_count == 1 + + clock.advance(61.0) + closer.reap() + await asyncio.sleep(0.05) + + assert client.closed is False + assert closer.pending_count == 1 + + +@pytest.mark.asyncio +async def test_a_client_serving_a_request_is_not_closed_when_its_grace_window_ends(): + """The grace window on its own cannot promise that a request has finished. + + ``litellm.request_timeout`` defaults to 6000 seconds and a streaming response + is bounded only by how long the upstream keeps sending, so a client past its + deadline is closed only once its own pool reports nothing in flight. + """ + server = await asyncio.start_server(_trickling_upstream, "127.0.0.1", 0) + port = server.sockets[0].getsockname()[1] + clock = FakeClock() + closer = make_closer(clock) + client = httpx.AsyncClient() + + closer.mark_owned(client) + closer.schedule(client) + + async def read_the_stream() -> int: + received = 0 + async with client.stream("GET", f"http://127.0.0.1:{port}/") as response: + async for chunk in response.aiter_bytes(): + received += len(chunk) + return received + + streaming = asyncio.create_task(read_the_stream()) + await asyncio.sleep(0.25) # the request is on the wire + clock.advance(3600.0) # and its grace window is long gone + closer.reap() + await asyncio.sleep(0.05) + + assert client.is_closed is False, "closed a client that was serving a request" + assert await streaming > 0, "the in-flight request did not survive the reap" + + clock.advance(3600.0) + closer.reap() + await asyncio.sleep(0.05) + + assert client.is_closed is True, "an idle client past its grace window must be closed" + assert closer.pending_count == 0 + server.close() + + +@pytest.mark.asyncio +async def test_the_aiohttp_backed_handler_is_not_closed_mid_request(): + """The default async path is aiohttp-backed, whose pool accounts for its own leases.""" + server = await asyncio.start_server(_trickling_upstream, "127.0.0.1", 0) + port = server.sockets[0].getsockname()[1] + clock = FakeClock() + closer = make_closer(clock) + handler = AsyncHTTPHandler() + + closer.mark_owned(handler) + closer.schedule(handler) + + request = asyncio.create_task(handler.get(f"http://127.0.0.1:{port}/")) + await asyncio.sleep(0.25) + clock.advance(3600.0) + closer.reap() + await asyncio.sleep(0.05) + + assert handler.client.is_closed is False, "closed a handler that was serving a request" + assert (await request).status_code == 200 + + clock.advance(3600.0) + closer.reap() + await asyncio.sleep(0.05) + + assert handler.client.is_closed is True + server.close() + + +def test_the_pending_queue_cannot_grow_past_its_bound(): + """A caller that churns the client cache must not be able to grow this queue.""" + clock = FakeClock() + closer = EvictedClientCloser(grace_seconds=60.0, max_pending=8, clock=clock) + clients = tuple(SyncClient() for _ in range(50)) + + for client in clients: + closer.mark_owned(client) + closer.schedule(client) + + assert closer.pending_count == 8, "the queue grew past max_pending" + + clock.advance(61.0) + closer.reap() + + assert closer.pending_count == 0 + assert sum(client.closed for client in clients) == 8, "everything queued should have been closed" + + +def test_a_reap_looks_at_what_is_due_rather_than_at_the_whole_queue(): + """Sustained churn evicts a client per request, and every read of the cache reaps. + + So the cost of a reap has to track the entries that are due, not the length of + the queue; a reap that filters the whole queue makes the pair quadratic. Each + bucket is ordered by deadline, so an up-to-date reap compares one entry per + bucket and stops. Counting the comparisons measures that directly, where a + wall-clock budget would only measure the machine. + """ + evictions = 1_000 + clock = FakeClock() + closer = EvictedClientCloser( + grace_seconds=60.0, + max_pending=evictions, + clock=lambda: CountingDeadline(clock.now), + ) + clients = tuple(SyncClient() for _ in range(evictions)) + for client in clients: + closer.mark_owned(client) + + CountingDeadline.comparisons = 0 + for client in clients: + closer.schedule(client) + closer.reap() # nothing is due yet, which is the hot path + clock.advance(61.0) + closer.reap() + + assert closer.pending_count == 0 + assert all(client.closed for client in clients) + assert CountingDeadline.comparisons < 10 * evictions, ( + f"{CountingDeadline.comparisons} deadline comparisons for {evictions} evictions; " + "a reap is walking the whole queue" + ) diff --git a/tests/test_litellm/caching/test_llm_caching_handler.py b/tests/test_litellm/caching/test_llm_caching_handler.py index 8e6a94945b0..5f0e82dbb80 100644 --- a/tests/test_litellm/caching/test_llm_caching_handler.py +++ b/tests/test_litellm/caching/test_llm_caching_handler.py @@ -19,6 +19,7 @@ sys.path.insert( 0, os.path.abspath("../../..") ) # Adds the parent directory to the system path +from litellm.caching.evicted_client_closer import EvictedClientCloser from litellm.caching.llm_caching_handler import LLMClientCache @@ -156,6 +157,71 @@ def test_remove_key_no_event_loop(): assert "test-key" not in cache.cache_dict +class _FakeClock: + """Hand-advanced monotonic clock, so grace windows need no real waiting.""" + + def __init__(self) -> None: + self.now = 1000.0 + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + self.now += seconds + + +@pytest.mark.asyncio +async def test_evicted_litellm_owned_client_is_closed_once_the_grace_window_elapses(): + """ + Eviction only drops the cache's reference. The SDK clients are reference + cycles, so without an explicit close the client keeps its connection pool + open until a generational collection runs. + """ + clock = _FakeClock() + cache = LLMClientCache( + max_size_in_memory=2, + evicted_client_closer=EvictedClientCloser(grace_seconds=60.0, clock=clock), + ) + + client = MockAsyncClient() + cache.set_cache("client-key", client, litellm_owned_client=True, ttl=600) + + cache.ttl_dict = {key: 0 for key in cache.ttl_dict} + cache.expiration_heap = [(0, key) for _, key in cache.expiration_heap] + cache.evict_cache() + await asyncio.sleep(0.1) + assert client.closed is False, "an in-flight request may still hold the client" + + clock.advance(61.0) + cache.get_cache("any-key") + await asyncio.sleep(0.1) + + assert client.closed is True + + +@pytest.mark.asyncio +async def test_evicted_caller_supplied_client_is_never_closed(): + """litellm does not own a client the caller passed in, so it must stay open.""" + clock = _FakeClock() + cache = LLMClientCache( + max_size_in_memory=2, + evicted_client_closer=EvictedClientCloser(grace_seconds=60.0, clock=clock), + ) + + client = MockAsyncClient() + cache.set_cache("client-key", client, ttl=600) + + cache.ttl_dict = {key: 0 for key in cache.ttl_dict} + cache.expiration_heap = [(0, key) for _, key in cache.expiration_heap] + cache.evict_cache() + + clock.advance(3600.0) + cache.get_cache("any-key") + await asyncio.sleep(0.1) + + assert client.closed is False + + def test_remove_key_removes_plain_values(): """ _remove_key correctly removes non-client values (strings, dicts, etc.). diff --git a/tests/test_litellm/llms/azure/test_azure_common_utils.py b/tests/test_litellm/llms/azure/test_azure_common_utils.py index a3280b90fe3..c5d4bd044cc 100644 --- a/tests/test_litellm/llms/azure/test_azure_common_utils.py +++ b/tests/test_litellm/llms/azure/test_azure_common_utils.py @@ -2034,3 +2034,74 @@ def test_azure_traditional_api_uses_azure_openai_client(): assert isinstance( async_client, AsyncAzureOpenAI ), f"Expected AsyncAzureOpenAI client for api_version={api_version}" + + +def test_evicting_an_azure_client_built_on_the_callers_session_leaves_it_open(monkeypatch): + """`initialize_azure_sdk_client` puts `litellm.aclient_session` on the SDK client. + + That session belongs to the caller. `AsyncAzureOpenAI.close()` closes whatever + http client it was handed, so treating the wrapper as litellm's to close would + close the caller's shared session out from under them. + """ + import httpx + + from litellm.caching.evicted_client_closer import EvictedClientCloser + from litellm.caching.llm_caching_handler import LLMClientCache + + shared_session = httpx.AsyncClient() + closer = EvictedClientCloser(grace_seconds=0.0) + monkeypatch.setattr(litellm, "aclient_session", shared_session) + monkeypatch.setattr( + litellm, + "in_memory_llm_clients_cache", + LLMClientCache(evicted_client_closer=closer), + ) + + wrapper = BaseAzureLLM().get_azure_openai_client( + api_key="not-a-real-key", + api_base="https://litellm.openai.azure.com", + api_version="2024-02-01", + litellm_params={}, + _is_async=True, + ) + + assert wrapper is not None + assert wrapper._client is shared_session, "the wrapper should be built on the caller's session" + + closer.schedule(wrapper) + closer.reap() + + assert closer.pending_count == 0, "a wrapper around the caller's session must never be queued" + assert shared_session.is_closed is False, "closed the session the caller configured" + + +def test_an_azure_client_litellm_built_its_own_http_client_for_is_still_closed(monkeypatch): + """The ownership check must not turn the reclaim off for the ordinary case.""" + from litellm.caching.evicted_client_closer import EvictedClientCloser + from litellm.caching.llm_caching_handler import LLMClientCache + + closer = EvictedClientCloser(grace_seconds=0.0) + monkeypatch.setattr(litellm, "aclient_session", None) + monkeypatch.setattr(litellm, "client_session", None) + monkeypatch.setattr( + litellm, + "in_memory_llm_clients_cache", + LLMClientCache(evicted_client_closer=closer), + ) + + wrapper = BaseAzureLLM().get_azure_openai_client( + api_key="not-a-real-key", + api_base="https://litellm.openai.azure.com", + api_version="2024-02-01", + litellm_params={}, + _is_async=False, + ) + + assert wrapper is not None + closer.schedule(wrapper) + + assert closer.pending_count == 1, "litellm built this client's http client, so it owns it" + + closer.reap() + + assert wrapper.is_closed() is True diff --git a/tests/test_litellm/llms/openai/test_openai_common_utils.py b/tests/test_litellm/llms/openai/test_openai_common_utils.py index ce25f7e9af6..a099b5c659f 100644 --- a/tests/test_litellm/llms/openai/test_openai_common_utils.py +++ b/tests/test_litellm/llms/openai/test_openai_common_utils.py @@ -175,3 +175,75 @@ def test_get_openai_client_cache_key(client_type): ) assert isinstance(key, str) assert "api_key=sk-test" in key + + +def test_evicting_a_client_built_on_the_callers_session_leaves_that_session_open(monkeypatch): + """`litellm.aclient_session` belongs to the caller, who goes on using it. + + `_get_async_http_client` hands that session straight back, so the SDK client + litellm builds around it is only a wrapper. The SDK's `close()` closes + whatever http client it was given, so treating the wrapper as litellm's to + close would close the caller's shared session out from under them. + """ + import httpx + + from litellm.caching.evicted_client_closer import EvictedClientCloser + from litellm.caching.llm_caching_handler import LLMClientCache + from litellm.llms.openai.openai import OpenAIChatCompletion + + shared_session = httpx.AsyncClient() + closer = EvictedClientCloser(grace_seconds=0.0) + monkeypatch.setattr(litellm, "aclient_session", shared_session) + monkeypatch.setattr( + litellm, + "in_memory_llm_clients_cache", + LLMClientCache(evicted_client_closer=closer), + ) + + wrapper = OpenAIChatCompletion()._get_openai_client( + is_async=True, + api_key="sk-not-a-real-key", + api_base="https://api.openai.com/v1", + max_retries=2, + ) + + assert wrapper is not None + assert wrapper._client is shared_session, "the wrapper should be built on the caller's session" + + closer.schedule(wrapper) + closer.reap() + + assert closer.pending_count == 0, "a wrapper around the caller's session must never be queued" + assert shared_session.is_closed is False, "closed the session the caller configured" + + +def test_a_client_litellm_built_its_own_http_client_for_is_still_closed(monkeypatch): + """The ownership check must not turn the reclaim off for the ordinary case.""" + from litellm.caching.evicted_client_closer import EvictedClientCloser + from litellm.caching.llm_caching_handler import LLMClientCache + from litellm.llms.openai.openai import OpenAIChatCompletion + + closer = EvictedClientCloser(grace_seconds=0.0) + monkeypatch.setattr(litellm, "aclient_session", None) + monkeypatch.setattr(litellm, "client_session", None) + monkeypatch.setattr( + litellm, + "in_memory_llm_clients_cache", + LLMClientCache(evicted_client_closer=closer), + ) + + wrapper = OpenAIChatCompletion()._get_openai_client( + is_async=False, + api_key="sk-not-a-real-key", + api_base="https://api.openai.com/v1", + max_retries=2, + ) + + assert wrapper is not None + closer.schedule(wrapper) + + assert closer.pending_count == 1, "litellm built this client's http client, so it owns it" + + closer.reap() + + assert wrapper.is_closed() is True From 9d5984b35836c9be130e69d5bcd37eeeb1796148 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 3 Aug 2026 13:39:07 -0700 Subject: [PATCH 092/123] refactor(ui): rename the create MCP server component to PascalCase (#35686) Pure rename, no behavior change. create_mcp_server.tsx and its test move to CreateMCPServer, the two importers and one stale e2e comment follow, and the local/filename-pascal-case suppression drops now that the file passes the rule on its own. The rename is scoped to this one component rather than the whole directory because three PRs are currently open against its snake_case siblings; the rest can follow once those land. --- tests/e2e/ui/tests/mcp/mcpServers.spec.ts | 2 +- ui/litellm-dashboard/eslint-suppressions.json | 31 +++++++++---------- ...rver.test.tsx => CreateMCPServer.test.tsx} | 2 +- ...ate_mcp_server.tsx => CreateMCPServer.tsx} | 0 .../mcp-servers/_components/mcp_discovery.tsx | 2 +- .../mcp-servers/_components/mcp_servers.tsx | 2 +- 6 files changed, 18 insertions(+), 21 deletions(-) rename ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/{create_mcp_server.test.tsx => CreateMCPServer.test.tsx} (99%) rename ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/{create_mcp_server.tsx => CreateMCPServer.tsx} (100%) diff --git a/tests/e2e/ui/tests/mcp/mcpServers.spec.ts b/tests/e2e/ui/tests/mcp/mcpServers.spec.ts index 37aabf9c057..43f21e77fbd 100644 --- a/tests/e2e/ui/tests/mcp/mcpServers.spec.ts +++ b/tests/e2e/ui/tests/mcp/mcpServers.spec.ts @@ -36,7 +36,7 @@ test.describe("MCP Servers", () => { await formModal.locator('input[id="url"]').fill("https://e2e-fake-mcp.test.local/mcp"); // Authentication: None - // The auth_type Form.Item has no label prop (create_mcp_server.tsx:795), so + // The auth_type Form.Item has no label prop (CreateMCPServer.tsx), so // it can't be anchored by label text. Scope via the enclosing Collapse // panel ("Authentication") instead — that anchor is stable even if the // placeholder copy changes. diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 8c6d940cfea..6c67f593aec 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -779,6 +779,20 @@ "count": 1 } }, + "src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx": { + "max-lines": { + "count": 1 + }, + "no-nested-ternary": { + "count": 1 + }, + "no-restricted-imports": { + "count": 2 + }, + "react-hooks/set-state-in-effect": { + "count": 4 + } + }, "src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx": { "no-restricted-imports": { "count": 1 @@ -900,23 +914,6 @@ "count": 1 } }, - "src/app/(dashboard)/mcp-servers/_components/create_mcp_server.tsx": { - "local/filename-pascal-case": { - "count": 1 - }, - "max-lines": { - "count": 1 - }, - "no-nested-ternary": { - "count": 1 - }, - "no-restricted-imports": { - "count": 2 - }, - "react-hooks/set-state-in-effect": { - "count": 4 - } - }, "src/app/(dashboard)/mcp-servers/_components/index.tsx": { "local/filename-pascal-case": { "count": 1 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/create_mcp_server.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.test.tsx similarity index 99% rename from ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/create_mcp_server.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.test.tsx index e6b70e170d2..45da71ed301 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/create_mcp_server.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.test.tsx @@ -3,7 +3,7 @@ import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; import * as networking from "@/components/networking"; import { setToken } from "@/utils/mcpTokenStore"; -import CreateMCPServer from "./create_mcp_server"; +import CreateMCPServer from "./CreateMCPServer"; import { selectAntOption } from "./testUtils"; vi.mock("@/components/networking", () => ({ diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/create_mcp_server.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/create_mcp_server.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_discovery.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_discovery.tsx index 5094f1a6761..b0b8b2eed93 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_discovery.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_discovery.tsx @@ -7,7 +7,7 @@ import { Skeleton } from "@/components/ui/skeleton"; import { cn } from "@/lib/cva.config"; import { fetchDiscoverableMCPServers } from "@/components/networking"; import { DiscoverableMCPServer, DiscoverMCPServersResponse } from "@/components/mcp_tools/types"; -import { mcpLogoImg } from "./create_mcp_server"; +import { mcpLogoImg } from "./CreateMCPServer"; import { resolveLogoSrc } from "@/lib/assetPaths"; interface MCPDiscoveryProps { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx index ebb1d710bf2..193246aab3c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx @@ -24,7 +24,7 @@ import NotificationsManager from "@/components/molecules/notifications_manager"; import { deleteMCPServer } from "@/components/networking"; import { MCPSubmissionsTab } from "./MCPSubmissionsTab"; import { MCPToolsetsTab } from "./MCPToolsetsTab"; -import CreateMCPServer from "./create_mcp_server"; +import CreateMCPServer from "./CreateMCPServer"; import MCPConnect from "./mcp_connect"; import MCPServerCard from "./MCPServerCard"; import { MCPServerView } from "./mcp_server_view"; From b03803b9189ba38d3d9805d5e342793ea721dc05 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 3 Aug 2026 13:21:52 -0700 Subject: [PATCH 093/123] refactor(ui): extract the MCP create form's logic and field groups Pulls four modules out of the 1398-line create component, which drops to 896 lines. No behavior changes: CreateMCPServer.test.tsx is untouched and all 77 of its tests pass against the refactored component, which is the review contract for this PR. createServerPayload.ts is a pure form-values-to-payload function whose failures are a tagged union instead of inline notification calls, so the transformation is reachable without a DOM. createOAuthUiState.ts owns the snapshot that survives the OAuth authorize redirect, keeping every presence guard the inline version had. AwsSigV4Fields and OpenApiByokFields are the two largest JSX blocks, moved verbatim so they can be diffed as moves. The create/edit setToken divergence, the mcpLogoImg export, and the untyped form-values bag are left alone on purpose; each is a behavior or cross-file change that does not belong in a move. --- ui/litellm-dashboard/eslint-suppressions.json | 10 + .../_components/AwsSigV4Fields.tsx | 155 +++++ .../_components/CreateMCPServer.tsx | 583 +++--------------- .../_components/OpenApiByokFields.tsx | 91 +++ .../_components/createOAuthUiState.ts | 96 +++ .../_components/createServerPayload.ts | 233 +++++++ 6 files changed, 666 insertions(+), 502 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createServerPayload.ts diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 6c67f593aec..107f66b8f1a 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -779,6 +779,11 @@ "count": 1 } }, + "src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx": { + "no-restricted-imports": { + "count": 1 + } + }, "src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx": { "max-lines": { "count": 1 @@ -870,6 +875,11 @@ "count": 1 } }, + "src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx": { + "no-restricted-imports": { + "count": 1 + } + }, "src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.test.tsx": { "no-restricted-imports": { "count": 1 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx new file mode 100644 index 00000000000..d4ae537bffa --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx @@ -0,0 +1,155 @@ +import React from "react"; +import { Form, Input, Tooltip } from "antd"; +import { InfoCircleOutlined } from "@ant-design/icons"; + +const AwsSigV4Fields: React.FC = () => ( + <> +

+ For MCP servers hosted on AWS Bedrock AgentCore.{" "} + + View docs → + +

+ + AWS Region + + + + + } + name={["credentials", "aws_region_name"]} + rules={[{ required: true, message: "AWS region is required for SigV4 auth" }]} + > + + + + AWS Service Name + + + + + } + name={["credentials", "aws_service_name"]} + > + + + + AWS Access Key ID + + + + + } + name={["credentials", "aws_access_key_id"]} + dependencies={[["credentials", "aws_secret_access_key"]]} + rules={[ + ({ getFieldValue }) => ({ + validator(_, value) { + const secretKey = getFieldValue(["credentials", "aws_secret_access_key"]); + if (secretKey && !value) { + return Promise.reject(new Error("Access Key ID is required when Secret Access Key is provided")); + } + return Promise.resolve(); + }, + }), + ]} + > + + + + AWS Secret Access Key + + + + + } + name={["credentials", "aws_secret_access_key"]} + dependencies={[["credentials", "aws_access_key_id"]]} + rules={[ + ({ getFieldValue }) => ({ + validator(_, value) { + const accessKeyId = getFieldValue(["credentials", "aws_access_key_id"]); + if (accessKeyId && !value) { + return Promise.reject(new Error("Secret Access Key is required when Access Key ID is provided")); + } + return Promise.resolve(); + }, + }), + ]} + > + + + + AWS Session Token + + + + + } + name={["credentials", "aws_session_token"]} + > + + + + AWS Role ARN + + + + + } + name={["credentials", "aws_role_name"]} + > + + + + AWS Session Name + + + + + } + name={["credentials", "aws_session_name"]} + > + + + +); + +export default AwsSigV4Fields; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx index 1d0262acdca..0785dd142ff 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx @@ -1,5 +1,5 @@ import React, { useState } from "react"; -import { Modal, Tooltip, Form, Select, Input, InputNumber, Switch, Collapse } from "antd"; +import { Modal, Tooltip, Form, Select, Input, InputNumber, Collapse } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; import { Button, TextInput } from "@tremor/react"; import { createMCPServer, registerMCPServer, storeMCPOAuthUserCredential } from "@/components/networking"; @@ -13,15 +13,22 @@ import { TRANSPORT, getMcpOAuthMode, MCP_OAUTH2_FLOW_M2M, - MCP_OAUTH2_FLOW_INTERACTIVE, isClientForwardedTokenMode, getOAuthAuthorizationIdentity, CLEARED_ON_INVALIDATION, isHeldOAuthTokenStale, preservedAdminCredentials, preservedDeclaredAppCredentials, - withoutMintedTokenCredentials, } from "@/components/mcp_tools/types"; +import { + AUTH_TYPES_REQUIRING_AUTH_VALUE, + BuildCreatePayloadResult, + buildCreateServerPayload, + reduceStaticHeaders, +} from "./createServerPayload"; +import { readCreateUiSnapshot, writeCreateUiSnapshot } from "./createOAuthUiState"; +import AwsSigV4Fields from "./AwsSigV4Fields"; +import OpenApiByokFields from "./OpenApiByokFields"; import OAuthFormFields from "./OAuthFormFields"; import TruePassthroughWarning from "./TruePassthroughWarning"; import PassthroughAuthorizeSection from "./PassthroughAuthorizeSection"; @@ -36,11 +43,10 @@ import OpenAPIFormSection, { OpenAPIKeyTool } from "./OpenAPIFormSection"; import MCPLogoSelector from "./MCPLogoSelector"; import EnvVarsSection from "./EnvVarsSection"; import { isAdminRole } from "@/utils/roles"; -import { validateMCPServerUrl, validateMCPServerName, normalizeEnvVars, TOOL_DISPLAY_NAME_PATTERN } from "./utils"; +import { validateMCPServerUrl, validateMCPServerName } from "./utils"; import NotificationsManager from "@/components/molecules/notifications_manager"; import { useMcpOAuthFlow } from "@/hooks/useMcpOAuthFlow"; import { useTestMCPConnection } from "@/hooks/useTestMCPConnection"; -import { getSecureItem, setSecureItem } from "@/utils/secureStorage"; import mcpLogo from "../../../../../public/assets/logos/mcp_logo.png"; export const mcpLogoImg = mcpLogo.src; @@ -57,25 +63,15 @@ interface CreateMCPServerProps { onBackToDiscovery?: () => void; } -const AUTH_TYPES_REQUIRING_AUTH_VALUE = [AUTH_TYPE.API_KEY, AUTH_TYPE.BEARER_TOKEN, AUTH_TYPE.TOKEN, AUTH_TYPE.BASIC]; -const AUTH_TYPES_REQUIRING_CREDENTIALS = [ - ...AUTH_TYPES_REQUIRING_AUTH_VALUE, - AUTH_TYPE.OAUTH2, - AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE, - AUTH_TYPE.OAUTH2_ID_JAG, - AUTH_TYPE.AWS_SIGV4, - AUTH_TYPE.TRUE_PASSTHROUGH, - AUTH_TYPE.OAUTH_DELEGATE, -]; -const CREATE_OAUTH_UI_STATE_KEY = "litellm-mcp-oauth-create-state"; - -const reduceStaticHeaders = (list: unknown): Record => { - if (!Array.isArray(list)) return {}; - return list.reduce((acc: Record, entry: Record) => { - const header = entry?.header?.trim(); - if (header) acc[header] = (entry?.value ?? "").trim(); - return acc; - }, {}); +const payloadErrorMessage = (result: Exclude): string => { + switch (result.kind) { + case "invalid_tool_display_name": + return `Tool display name "${result.displayName}" is invalid. Only letters, digits, underscores, and hyphens are allowed (no spaces).`; + case "invalid_stdio_json": + return "Invalid JSON in stdio configuration"; + case "invalid_token_validation_json": + return "Invalid JSON in Token Validation Rules"; + } }; const CreateMCPServer: React.FC = ({ @@ -147,29 +143,18 @@ const CreateMCPServer: React.FC = ({ const isM2MFlow = isOAuthAuthType && formValues.oauth_flow_type === OAUTH_FLOW.M2M; const persistCreateUiState = () => { - if (typeof window === "undefined") { - return; - } - try { - const values = form.getFieldsValue(true); - const uiState = { - modalVisible: isModalVisible, - formValues: values, - transportType, - costConfig, - allowedTools, - hasToolAllowlistInteraction, - searchValue, - aliasManuallyEdited, - logoUrl, - // Persist the identity so invalidation stays armed across the OAuth redirect round trip: a - // post-restore url/mode edit must still discard the held token instead of silently keeping it. - authorizedIdentity, - }; - setSecureItem(CREATE_OAUTH_UI_STATE_KEY, JSON.stringify(uiState)); - } catch (err) { - console.warn("Failed to persist MCP create state", err); - } + writeCreateUiSnapshot({ + modalVisible: isModalVisible, + formValues: form.getFieldsValue(true), + transportType, + costConfig, + allowedTools, + hasToolAllowlistInteraction, + searchValue, + aliasManuallyEdited, + logoUrl, + authorizedIdentity, + }); }; const { @@ -308,59 +293,39 @@ const CreateMCPServer: React.FC = ({ }; React.useEffect(() => { - if (typeof window === "undefined") { + const restored = readCreateUiSnapshot(); + if (!restored) { return; } - const storedState = getSecureItem(CREATE_OAUTH_UI_STATE_KEY); - if (!storedState) { - return; + if (restored.modalVisible) { + setModalVisible(true); } - - try { - const parsed = JSON.parse(storedState); - if (parsed.modalVisible) { - setModalVisible(true); - } - const restoredTransport = parsed.formValues?.transport || parsed.transportType || ""; - if (restoredTransport) { - setTransportType(restoredTransport); - } - if (parsed.formValues) { - // Assign the cleaned credentials (strip minted token material so a stale token never rehydrates); - // the declared app the admin typed is kept. Create has no server-side stored app to merge. - const restoredValues = { - ...parsed.formValues, - credentials: withoutMintedTokenCredentials(parsed.formValues.credentials), - }; - setPendingRestoredValues({ values: restoredValues, transport: restoredTransport }); - } - if (typeof parsed.authorizedIdentity === "string") { - // Re-arm invalidation: without this the remounted form has authorizedIdentity=undefined, so a - // post-restore mode/url edit would never fire the stale-token discard. - setAuthorizedIdentity(parsed.authorizedIdentity); - } - if (parsed.costConfig) { - setCostConfig(parsed.costConfig); - } - if (parsed.allowedTools) { - setAllowedTools(parsed.allowedTools); - } - if (typeof parsed.hasToolAllowlistInteraction === "boolean") { - setHasToolAllowlistInteraction(parsed.hasToolAllowlistInteraction); - } - if (parsed.searchValue) { - setSearchValue(parsed.searchValue); - } - if (typeof parsed.aliasManuallyEdited === "boolean") { - setAliasManuallyEdited(parsed.aliasManuallyEdited); - } - if (parsed.logoUrl) { - setLogoUrl(parsed.logoUrl); - } - } catch (err) { - console.error("Failed to restore MCP create state", err); - } finally { - window.sessionStorage.removeItem(CREATE_OAUTH_UI_STATE_KEY); + if (restored.transportType) { + setTransportType(restored.transportType); + } + if (restored.formValues) { + setPendingRestoredValues({ values: restored.formValues, transport: restored.transportType }); + } + if (restored.authorizedIdentity !== undefined) { + setAuthorizedIdentity(restored.authorizedIdentity); + } + if (restored.costConfig) { + setCostConfig(restored.costConfig); + } + if (restored.allowedTools) { + setAllowedTools([...restored.allowedTools]); + } + if (restored.hasToolAllowlistInteraction !== undefined) { + setHasToolAllowlistInteraction(restored.hasToolAllowlistInteraction); + } + if (restored.searchValue) { + setSearchValue(restored.searchValue); + } + if (restored.aliasManuallyEdited !== undefined) { + setAliasManuallyEdited(restored.aliasManuallyEdited); + } + if (restored.logoUrl) { + setLogoUrl(restored.logoUrl); } }, [form, setModalVisible]); @@ -422,169 +387,25 @@ const CreateMCPServer: React.FC = ({ setAliasManuallyEdited(false); }, [isModalVisible, prefillData, form]); - const handleCreate = async (values: Record) => { - const invalidDisplayName = Object.entries(toolNameToDisplayName).find( - ([, displayName]) => displayName && !TOOL_DISPLAY_NAME_PATTERN.test(displayName), - ); - if (invalidDisplayName) { - NotificationsManager.fromBackend( - `Tool display name "${invalidDisplayName[1]}" is invalid. Only letters, digits, underscores, and hyphens are allowed (no spaces).`, - ); + const handleCreate = async (values: Record) => { + const built = buildCreateServerPayload(values, { + transportType, + costConfig, + allowedTools, + hasToolAllowlistInteraction, + toolNameToDisplayName, + toolNameToDescription, + logoUrl, + dcrClient: dcrClientRef.current, + }); + if (built.kind !== "ok") { + NotificationsManager.fromBackend(payloadErrorMessage(built)); return; } + const payload = built.payload; + setIsLoading(true); try { - const { - static_headers: staticHeadersList, - env_vars: envVarsList, - stdio_config: rawStdioConfig, - credentials: credentialValues, - allow_all_keys: allowAllKeysRaw, - available_on_public_internet: availableOnPublicInternetRaw, - delegate_auth_to_upstream: delegateAuthToUpstreamRaw, - oauth_passthrough: oauthPassthroughRaw, - dcr_bridge: dcrBridgeRaw, - token_validation_json: rawTokenValidationJson, - ...restValues - } = values; - - // Transform access groups into objects with name property - const accessGroups = restValues.mcp_access_groups; - - const staticHeaders = reduceStaticHeaders(staticHeadersList); - const envVars = normalizeEnvVars(envVarsList); - - const credentialsPayload = - credentialValues && typeof credentialValues === "object" - ? Object.entries(credentialValues).reduce((acc: Record, [key, value]) => { - if (value === undefined || value === null || value === "") { - return acc; - } - if (key === "scopes") { - if (Array.isArray(value)) { - const filteredScopes = value.filter((scope) => scope != null && scope !== ""); - if (filteredScopes.length > 0) { - acc[key] = filteredScopes; - } - } - } else { - acc[key] = value; - } - return acc; - }, {}) - : undefined; - - // Process stdio configuration if present - let stdioFields = {}; - if (rawStdioConfig && transportType === "stdio") { - try { - const stdioConfig = JSON.parse(rawStdioConfig); - - // Handle both formats: - // 1. Full mcpServers structure: {"mcpServers": {"server-name": {...}}} - // 2. Direct config: {"command": "...", "args": [...], "env": {...}} - - let actualConfig = stdioConfig; - - // If it's the full mcpServers structure, extract the first server config - if (stdioConfig.mcpServers && typeof stdioConfig.mcpServers === "object") { - const serverNames = Object.keys(stdioConfig.mcpServers); - if (serverNames.length > 0) { - const firstServerName = serverNames[0]; - actualConfig = stdioConfig.mcpServers[firstServerName]; - - // If no alias is provided, use the server name from the JSON - if (!restValues.server_name) { - restValues.server_name = firstServerName.replace(/-/g, "_"); // Replace hyphens with underscores - } - } - } - - stdioFields = { - command: actualConfig.command, - args: actualConfig.args, - env: actualConfig.env, - }; - } catch (error) { - NotificationsManager.fromBackend("Invalid JSON in stdio configuration"); - return; - } - } - - // Map "openapi" transport to "http" for the backend - if (restValues.transport === TRANSPORT.OPENAPI) { - restValues.transport = "http"; - } - - // Parse token_validation JSON if provided - let tokenValidation: Record | null = null; - if (rawTokenValidationJson && rawTokenValidationJson.trim() !== "") { - try { - tokenValidation = JSON.parse(rawTokenValidationJson); - } catch { - NotificationsManager.fromBackend("Invalid JSON in Token Validation Rules"); - setIsLoading(false); - return; - } - } - - // Prepare the payload with cost configuration and allowed tools - const payload: Record = { - ...restValues, - ...stdioFields, - // Remove the raw stdio_config field as we've extracted its components - stdio_config: undefined, - mcp_info: { - server_name: restValues.server_name || restValues.url, - description: restValues.description, - logo_url: logoUrl || undefined, - mcp_server_cost_info: Object.keys(costConfig).length > 0 ? costConfig : null, - tool_allowlist_enforced: hasToolAllowlistInteraction || allowedTools.length > 0, - }, - mcp_access_groups: accessGroups, - alias: restValues.alias, - allowed_tools: allowedTools, - tool_name_to_display_name: toolNameToDisplayName, - tool_name_to_description: toolNameToDescription, - allow_all_keys: Boolean(allowAllKeysRaw), - available_on_public_internet: Boolean(availableOnPublicInternetRaw), - delegate_auth_to_upstream: Boolean(delegateAuthToUpstreamRaw), - oauth_passthrough: Boolean(oauthPassthroughRaw), - // ``dcr_bridge`` is only meaningful for the client-forwarded token - // modes (true_passthrough / oauth_delegate) and defaults on when the - // toggle is shown; force false for any other auth type so a stale - // ``true`` is never persisted. Mirrors the sibling flags above. - dcr_bridge: isClientForwardedTokenMode(restValues.auth_type) ? Boolean(dcrBridgeRaw ?? true) : false, - ...(restValues.auth_type === AUTH_TYPE.OAUTH2 - ? { - oauth2_flow: - values.oauth_flow_type === OAUTH_FLOW.M2M ? MCP_OAUTH2_FLOW_M2M : MCP_OAUTH2_FLOW_INTERACTIVE, - } - : {}), - static_headers: staticHeaders, - env_vars: envVars, - ...(tokenValidation !== null && { token_validation: tokenValidation }), - }; - - const includeCredentials = - restValues.auth_type && AUTH_TYPES_REQUIRING_CREDENTIALS.includes(restValues.auth_type); - - // Client-forwarded rows persist ONLY the declared app; strip any token material that lingered in - // the form (e.g. from a prior oauth2 authorize on the same session) so it can never reach the row. - const submitCredentials = isClientForwardedTokenMode(restValues.auth_type) - ? preservedAdminCredentials(credentialsPayload) - : credentialsPayload; - - if (includeCredentials && submitCredentials && Object.keys(submitCredentials).length > 0) { - payload.credentials = submitCredentials; - } - - // An interactive (oauth2) create persists its DCR-minted client from the ref (kept out of the - // form store); reuse a re-authorize's registered client instead of re-registering. - if (restValues.auth_type === AUTH_TYPE.OAUTH2 && dcrClientRef.current) { - payload.credentials = { ...(payload.credentials ?? {}), ...dcrClientRef.current }; - } - if (accessToken != null) { const response = isAdmin ? await createMCPServer(accessToken, payload) @@ -596,9 +417,9 @@ const CreateMCPServer: React.FC = ({ // forwards a browser-held token, so it stays in sessionStorage only. if (oauthTokenResponse?.access_token && response?.server_id) { const oauthMode = getMcpOAuthMode({ - auth_type: restValues.auth_type, + auth_type: values.auth_type as string | undefined, oauth2_flow: values.oauth_flow_type === OAUTH_FLOW.M2M ? MCP_OAUTH2_FLOW_M2M : null, - delegate_auth_to_upstream: Boolean(delegateAuthToUpstreamRaw), + delegate_auth_to_upstream: Boolean(values.delegate_auth_to_upstream), }); if (oauthMode === "authorization_code") { const scope = oauthTokenResponse.scope; @@ -953,94 +774,7 @@ const CreateMCPServer: React.FC = ({ )} {/* BYOK toggle - only for OpenAPI */} - {transportType === TRANSPORT.OPENAPI && ( - <> - - BYOK (Bring Your Own Key) - - - - - } - name="is_byok" - valuePropName="checked" - > - - - - prev.is_byok !== cur.is_byok || prev.auth_type !== cur.auth_type} - > - {({ getFieldValue }) => - getFieldValue("is_byok") ? ( - <> - {/* Auth format hint */} - {getFieldValue("auth_type") && getFieldValue("auth_type") !== "none" && ( -
- - - User keys will be sent as:{" "} - - {getFieldValue("auth_type") === "bearer_token" && "Authorization: Bearer {key}"} - {getFieldValue("auth_type") === "token" && "Authorization: token {key}"} - {getFieldValue("auth_type") === "api_key" && "x-api-key: {key}"} - {getFieldValue("auth_type") === "basic" && "Authorization: Basic {key}"} - {getFieldValue("auth_type") === "authorization" && "Authorization: {key}"} - - {!getFieldValue("auth_type") && "Set Authentication Type below to specify the format."} - -
- )} - {!getFieldValue("auth_type") && ( -
- - - Set the Authentication Type below to specify how user keys are sent - (e.g., Bearer Token, API Key header). - -
- )} - - Access Description - - - - - } - name="byok_description" - > - - - - ) : null - } -
- - )} + {transportType === TRANSPORT.OPENAPI && } = ({ /> )} - {transportType !== "stdio" && transportType !== "" && isAwsSigV4AuthType && ( - <> -

- For MCP servers hosted on AWS Bedrock AgentCore.{" "} - - View docs → - -

- - AWS Region - - - - - } - name={["credentials", "aws_region_name"]} - rules={[{ required: true, message: "AWS region is required for SigV4 auth" }]} - > - - - - AWS Service Name - - - - - } - name={["credentials", "aws_service_name"]} - > - - - - AWS Access Key ID - - - - - } - name={["credentials", "aws_access_key_id"]} - dependencies={[["credentials", "aws_secret_access_key"]]} - rules={[ - ({ getFieldValue }) => ({ - validator(_, value) { - const secretKey = getFieldValue(["credentials", "aws_secret_access_key"]); - if (secretKey && !value) { - return Promise.reject( - new Error("Access Key ID is required when Secret Access Key is provided"), - ); - } - return Promise.resolve(); - }, - }), - ]} - > - - - - AWS Secret Access Key - - - - - } - name={["credentials", "aws_secret_access_key"]} - dependencies={[["credentials", "aws_access_key_id"]]} - rules={[ - ({ getFieldValue }) => ({ - validator(_, value) { - const accessKeyId = getFieldValue(["credentials", "aws_access_key_id"]); - if (accessKeyId && !value) { - return Promise.reject( - new Error("Secret Access Key is required when Access Key ID is provided"), - ); - } - return Promise.resolve(); - }, - }), - ]} - > - - - - AWS Session Token - - - - - } - name={["credentials", "aws_session_token"]} - > - - - - AWS Role ARN - - - - - } - name={["credentials", "aws_role_name"]} - > - - - - AWS Session Name - - - - - } - name={["credentials", "aws_session_name"]} - > - - - - )} + {transportType !== "stdio" && transportType !== "" && isAwsSigV4AuthType && } {/* Stdio Configuration - only show for stdio transport */} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx new file mode 100644 index 00000000000..2ac4279e20a --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx @@ -0,0 +1,91 @@ +import React from "react"; +import { Form, Input, Select, Switch, Tooltip } from "antd"; +import { InfoCircleOutlined } from "@ant-design/icons"; + +const OpenApiByokFields: React.FC = () => ( + <> + + BYOK (Bring Your Own Key) + + + + + } + name="is_byok" + valuePropName="checked" + > + + + + prev.is_byok !== cur.is_byok || prev.auth_type !== cur.auth_type}> + {({ getFieldValue }) => + getFieldValue("is_byok") ? ( + <> + {/* Auth format hint */} + {getFieldValue("auth_type") && getFieldValue("auth_type") !== "none" && ( +
+ + + User keys will be sent as:{" "} + + {getFieldValue("auth_type") === "bearer_token" && "Authorization: Bearer {key}"} + {getFieldValue("auth_type") === "token" && "Authorization: token {key}"} + {getFieldValue("auth_type") === "api_key" && "x-api-key: {key}"} + {getFieldValue("auth_type") === "basic" && "Authorization: Basic {key}"} + {getFieldValue("auth_type") === "authorization" && "Authorization: {key}"} + + {!getFieldValue("auth_type") && "Set Authentication Type below to specify the format."} + +
+ )} + {!getFieldValue("auth_type") && ( +
+ + + Set the Authentication Type below to specify how user keys are sent (e.g., Bearer + Token, API Key header). + +
+ )} + + Access Description + + + + + } + name="byok_description" + > + + + + ) : null + } +
+ +); + +export default OpenApiByokFields; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.ts new file mode 100644 index 00000000000..f6475b97830 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.ts @@ -0,0 +1,96 @@ +import { MCPServerCostInfo, withoutMintedTokenCredentials } from "@/components/mcp_tools/types"; +import { getSecureItem, setSecureItem } from "@/utils/secureStorage"; + +const CREATE_OAUTH_UI_STATE_KEY = "litellm-mcp-oauth-create-state"; + +// Everything the create modal needs to look untouched after the OAuth authorize redirect reloads the +// page. `authorizedIdentity` is part of it so invalidation stays armed across the round trip: without +// it the remounted form starts with no identity, and a post-restore url/mode edit would never fire the +// stale-token discard. +export interface CreateUiSnapshot { + readonly modalVisible: boolean; + readonly formValues: Record; + readonly transportType: string; + readonly costConfig: MCPServerCostInfo; + readonly allowedTools: readonly string[]; + readonly hasToolAllowlistInteraction: boolean; + readonly searchValue: string; + readonly aliasManuallyEdited: boolean; + readonly logoUrl: string | undefined; + readonly authorizedIdentity: string | undefined; +} + +// Only the fields that survived their own presence check. A key absent here means "leave the freshly +// mounted state alone", which is why every field is optional rather than defaulted. +export type RestoredUiSnapshot = { + readonly modalVisible?: boolean; + readonly formValues?: Record; + readonly transportType?: string; + readonly costConfig?: MCPServerCostInfo; + readonly allowedTools?: readonly string[]; + readonly hasToolAllowlistInteraction?: boolean; + readonly searchValue?: string; + readonly aliasManuallyEdited?: boolean; + readonly logoUrl?: string; + readonly authorizedIdentity?: string; +}; + +export const writeCreateUiSnapshot = (snapshot: CreateUiSnapshot): void => { + if (typeof window === "undefined") { + return; + } + try { + setSecureItem(CREATE_OAUTH_UI_STATE_KEY, JSON.stringify(snapshot)); + } catch (err) { + console.warn("Failed to persist MCP create state", err); + } +}; + +/** + * Read and validate the snapshot left before the authorize redirect, then drop it so a later mount + * cannot replay it. Returns null when there is nothing to restore (or the payload was unparseable), + * in which case the stored value is left in place for an in-flight flow to time out naturally. + */ +export const readCreateUiSnapshot = (): RestoredUiSnapshot | null => { + if (typeof window === "undefined") { + return null; + } + const storedState = getSecureItem(CREATE_OAUTH_UI_STATE_KEY); + if (!storedState) { + return null; + } + + try { + const parsed = JSON.parse(storedState); + const restoredTransport = parsed.formValues?.transport || parsed.transportType || ""; + + return { + ...(parsed.modalVisible ? { modalVisible: true } : {}), + ...(restoredTransport ? { transportType: restoredTransport } : {}), + ...(parsed.formValues + ? { + // Strip minted token material so a stale token never rehydrates; the declared app the + // admin typed is kept. Create has no server-side stored app to merge. + formValues: { + ...parsed.formValues, + credentials: withoutMintedTokenCredentials(parsed.formValues.credentials), + }, + } + : {}), + ...(typeof parsed.authorizedIdentity === "string" ? { authorizedIdentity: parsed.authorizedIdentity } : {}), + ...(parsed.costConfig ? { costConfig: parsed.costConfig } : {}), + ...(parsed.allowedTools ? { allowedTools: parsed.allowedTools } : {}), + ...(typeof parsed.hasToolAllowlistInteraction === "boolean" + ? { hasToolAllowlistInteraction: parsed.hasToolAllowlistInteraction } + : {}), + ...(parsed.searchValue ? { searchValue: parsed.searchValue } : {}), + ...(typeof parsed.aliasManuallyEdited === "boolean" ? { aliasManuallyEdited: parsed.aliasManuallyEdited } : {}), + ...(parsed.logoUrl ? { logoUrl: parsed.logoUrl } : {}), + }; + } catch (err) { + console.error("Failed to restore MCP create state", err); + return null; + } finally { + window.sessionStorage.removeItem(CREATE_OAUTH_UI_STATE_KEY); + } +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createServerPayload.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createServerPayload.ts new file mode 100644 index 00000000000..f45857fcc8b --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createServerPayload.ts @@ -0,0 +1,233 @@ +import { + AUTH_TYPE, + MCPServerCostInfo, + MCP_OAUTH2_FLOW_INTERACTIVE, + MCP_OAUTH2_FLOW_M2M, + OAUTH_FLOW, + TRANSPORT, + isClientForwardedTokenMode, + preservedAdminCredentials, +} from "@/components/mcp_tools/types"; +import { TOOL_DISPLAY_NAME_PATTERN, normalizeEnvVars } from "./utils"; + +export const AUTH_TYPES_REQUIRING_AUTH_VALUE = [ + AUTH_TYPE.API_KEY, + AUTH_TYPE.BEARER_TOKEN, + AUTH_TYPE.TOKEN, + AUTH_TYPE.BASIC, +]; + +export const AUTH_TYPES_REQUIRING_CREDENTIALS = [ + ...AUTH_TYPES_REQUIRING_AUTH_VALUE, + AUTH_TYPE.OAUTH2, + AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE, + AUTH_TYPE.OAUTH2_ID_JAG, + AUTH_TYPE.AWS_SIGV4, + AUTH_TYPE.TRUE_PASSTHROUGH, + AUTH_TYPE.OAUTH_DELEGATE, +]; + +export interface DcrClient { + readonly client_id: string; + readonly client_secret?: string; +} + +export interface CreateServerUiState { + readonly transportType: string; + readonly costConfig: MCPServerCostInfo; + readonly allowedTools: readonly string[]; + readonly hasToolAllowlistInteraction: boolean; + readonly toolNameToDisplayName: Readonly>; + readonly toolNameToDescription: Readonly>; + readonly logoUrl: string | undefined; + readonly dcrClient: DcrClient | null; +} + +export type BuildCreatePayloadResult = + | { readonly kind: "ok"; readonly payload: Record } + | { readonly kind: "invalid_tool_display_name"; readonly displayName: string } + | { readonly kind: "invalid_stdio_json" } + | { readonly kind: "invalid_token_validation_json" }; + +export type StdioParseResult = + | { readonly kind: "ok"; readonly fields: Record; readonly derivedServerName?: string } + | { readonly kind: "invalid" }; + +type JsonParseResult = + | { readonly kind: "ok"; readonly value: Record | null } + | { readonly kind: "invalid" }; + +const tryParseJson = (raw: string): JsonParseResult => { + try { + return { kind: "ok", value: JSON.parse(raw) }; + } catch { + return { kind: "invalid" }; + } +}; + +export const reduceStaticHeaders = (list: unknown): Record => { + if (!Array.isArray(list)) return {}; + return list.reduce((acc: Record, entry: Record) => { + const header = entry?.header?.trim(); + if (header) acc[header] = (entry?.value ?? "").trim(); + return acc; + }, {}); +}; + +// Accepts both the full `{"mcpServers": {"name": {...}}}` shape a user copies out of a client config +// and a bare `{"command": ..., "args": ..., "env": ...}`. A non-object JSON body (null, a number) +// falls through to the invalid branch, which is what the caller surfaces to the admin. +export const parseStdioConfig = (raw: string): StdioParseResult => { + try { + const stdioConfig = JSON.parse(raw); + const nestedName = + stdioConfig.mcpServers && typeof stdioConfig.mcpServers === "object" + ? Object.keys(stdioConfig.mcpServers)[0] + : undefined; + const actualConfig = nestedName === undefined ? stdioConfig : stdioConfig.mcpServers[nestedName]; + + return { + kind: "ok", + fields: { command: actualConfig.command, args: actualConfig.args, env: actualConfig.env }, + // The JSON's own server key is the fallback name when the admin left the field blank. + ...(nestedName === undefined ? {} : { derivedServerName: nestedName.replace(/-/g, "_") }), + }; + } catch { + return { kind: "invalid" }; + } +}; + +const filterCredentials = (credentialValues: unknown): Record | undefined => { + if (!credentialValues || typeof credentialValues !== "object") return undefined; + return Object.entries(credentialValues as Record).reduce( + (acc: Record, [key, value]) => { + if (value === undefined || value === null || value === "") { + return acc; + } + if (key === "scopes") { + if (Array.isArray(value)) { + const filteredScopes = value.filter((scope) => scope != null && scope !== ""); + if (filteredScopes.length > 0) { + acc[key] = filteredScopes; + } + } + } else { + acc[key] = value; + } + return acc; + }, + {}, + ); +}; + +const firstInvalidToolDisplayName = (toolNameToDisplayName: Readonly>): string | undefined => + Object.entries(toolNameToDisplayName).find( + ([, displayName]) => displayName && !TOOL_DISPLAY_NAME_PATTERN.test(displayName), + )?.[1]; + +export const buildCreateServerPayload = ( + values: Record, + ui: CreateServerUiState, +): BuildCreatePayloadResult => { + const badDisplayName = firstInvalidToolDisplayName(ui.toolNameToDisplayName); + if (badDisplayName !== undefined) { + return { kind: "invalid_tool_display_name", displayName: badDisplayName }; + } + + const { + static_headers: staticHeadersList, + env_vars: envVarsList, + stdio_config: rawStdioConfig, + credentials: credentialValues, + allow_all_keys: allowAllKeysRaw, + available_on_public_internet: availableOnPublicInternetRaw, + delegate_auth_to_upstream: delegateAuthToUpstreamRaw, + oauth_passthrough: oauthPassthroughRaw, + dcr_bridge: dcrBridgeRaw, + token_validation_json: rawTokenValidationJson, + ...restValues + } = values; + + const stdio: StdioParseResult = + rawStdioConfig && ui.transportType === "stdio" + ? parseStdioConfig(rawStdioConfig as string) + : { kind: "ok", fields: {} }; + if (stdio.kind === "invalid") { + return { kind: "invalid_stdio_json" }; + } + + const rawTokenValidation = rawTokenValidationJson as string | undefined; + const tokenValidationResult: JsonParseResult = + rawTokenValidation && rawTokenValidation.trim() !== "" + ? tryParseJson(rawTokenValidation) + : { kind: "ok", value: null }; + if (tokenValidationResult.kind === "invalid") { + return { kind: "invalid_token_validation_json" }; + } + const tokenValidation = tokenValidationResult.value; + + const serverName = (restValues.server_name as string | undefined) || stdio.derivedServerName; + // "openapi" is a UI-only transport; the backend stores those servers as plain http. + const transport = restValues.transport === TRANSPORT.OPENAPI ? "http" : restValues.transport; + const authType = restValues.auth_type as string | undefined; + + const credentialsPayload = filterCredentials(credentialValues); + const includeCredentials = authType !== undefined && AUTH_TYPES_REQUIRING_CREDENTIALS.includes(authType); + // Client-forwarded rows persist ONLY the declared app; strip any token material that lingered in + // the form (e.g. from a prior oauth2 authorize on the same session) so it can never reach the row. + const submitCredentials = isClientForwardedTokenMode(authType) + ? preservedAdminCredentials(credentialsPayload) + : credentialsPayload; + const persistedCredentials = + includeCredentials && submitCredentials && Object.keys(submitCredentials).length > 0 + ? submitCredentials + : undefined; + // An interactive (oauth2) create persists its DCR-minted client from the ref (kept out of the + // form store); reuse a re-authorize's registered client instead of re-registering. + const credentials = + authType === AUTH_TYPE.OAUTH2 && ui.dcrClient + ? { ...(persistedCredentials ?? {}), ...ui.dcrClient } + : persistedCredentials; + + return { + kind: "ok", + payload: { + ...restValues, + ...stdio.fields, + ...(serverName === restValues.server_name ? {} : { server_name: serverName }), + ...(transport === restValues.transport ? {} : { transport }), + // Remove the raw stdio_config field as we've extracted its components + stdio_config: undefined, + mcp_info: { + server_name: serverName || restValues.url, + description: restValues.description, + logo_url: ui.logoUrl || undefined, + mcp_server_cost_info: Object.keys(ui.costConfig).length > 0 ? ui.costConfig : null, + tool_allowlist_enforced: ui.hasToolAllowlistInteraction || ui.allowedTools.length > 0, + }, + mcp_access_groups: restValues.mcp_access_groups, + alias: restValues.alias, + allowed_tools: [...ui.allowedTools], + tool_name_to_display_name: ui.toolNameToDisplayName, + tool_name_to_description: ui.toolNameToDescription, + allow_all_keys: Boolean(allowAllKeysRaw), + available_on_public_internet: Boolean(availableOnPublicInternetRaw), + delegate_auth_to_upstream: Boolean(delegateAuthToUpstreamRaw), + oauth_passthrough: Boolean(oauthPassthroughRaw), + // ``dcr_bridge`` is only meaningful for the client-forwarded token + // modes (true_passthrough / oauth_delegate) and defaults on when the + // toggle is shown; force false for any other auth type so a stale + // ``true`` is never persisted. Mirrors the sibling flags above. + dcr_bridge: isClientForwardedTokenMode(authType) ? Boolean(dcrBridgeRaw ?? true) : false, + ...(authType === AUTH_TYPE.OAUTH2 + ? { + oauth2_flow: values.oauth_flow_type === OAUTH_FLOW.M2M ? MCP_OAUTH2_FLOW_M2M : MCP_OAUTH2_FLOW_INTERACTIVE, + } + : {}), + static_headers: reduceStaticHeaders(staticHeadersList), + env_vars: normalizeEnvVars(envVarsList), + ...(tokenValidation !== null && { token_validation: tokenValidation }), + ...(credentials === undefined ? {} : { credentials }), + }, + }; +}; From a6d4654261d97d7757721b1ff5af6e9d38279ad6 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 3 Aug 2026 14:10:35 -0700 Subject: [PATCH 094/123] fix(openai): restore httpx client union type on owns_wrapped_http_client (#35706) PR #35492 was authored before the ruff sweep removed Union from the typing imports in litellm/llms/openai/common_utils.py, so the merge landed an annotation referencing Union without an import. The annotation is evaluated at class-definition time, so importing litellm raises NameError and every test shard on litellm_internal_staging fails at collection. Rewrites the annotation (and the same latent one in openai.py) as httpx.Client | httpx.AsyncClient | None, matching the file's PEP 604 style, so no typing import is needed at all. --- litellm/llms/openai/common_utils.py | 2 +- litellm/llms/openai/openai.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 082764df208..808998ddaf6 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -135,7 +135,7 @@ class BaseOpenAILLM: return _cached_client @staticmethod - def owns_wrapped_http_client(http_client: Optional[Union[httpx.Client, httpx.AsyncClient]]) -> bool: + def owns_wrapped_http_client(http_client: httpx.Client | httpx.AsyncClient | None) -> bool: """Whether litellm may close an SDK client built around ``http_client``. ``_get_async_http_client`` / ``_get_sync_http_client`` hand back diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 7096cdbf8fd..f01730a06a5 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -366,7 +366,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): if cached_client: if isinstance(cached_client, OpenAI) or isinstance(cached_client, AsyncOpenAI): return cached_client - http_client: Optional[Union[httpx.Client, httpx.AsyncClient]] = ( + http_client: httpx.Client | httpx.AsyncClient | None = ( OpenAIChatCompletion._get_async_http_client(shared_session=shared_session) if is_async else OpenAIChatCompletion._get_sync_http_client() From 32eb0720e3b6d9277e500555ba04ee2f5f1fcdff Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:14:58 -0700 Subject: [PATCH 095/123] fix(openai): drop the undefined Union from owns_wrapped_http_client (#35704) Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> From b7843193a0a1aa355105069b06d6bd969652d2b0 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 3 Aug 2026 14:27:40 -0700 Subject: [PATCH 096/123] chore(ui): note Google's Agent Platform rename in vector store setup (#28076) Google Cloud has renamed Vertex AI RAG Engine to "RAG Engine" and Vertex AI Search to "Agent Search" in its console. Users following our setup instructions hit a naming mismatch when they cross-reference the GCP console. Keep "Vertex AI" as the primary term (the generic new names would make our provider UI ambiguous) and surface the new names as secondary asides only where users leave the UI for the console. Resolves LIT-3081 --- .../_components/VectorStoreForm.tsx | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx index 6cdb895b98d..67ba469f68d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx @@ -176,6 +176,10 @@ const VectorStoreForm: React.FC = ({ description={

To use Vertex AI RAG Engine:

+

+ Note: Google Cloud has renamed this to "RAG Engine" in its console — the steps below still + apply. +

  1. Set up your Vertex AI RAG Engine corpus following the guide:{" "} @@ -188,7 +192,9 @@ const VectorStoreForm: React.FC = ({
  2. Create a corpus in your Google Cloud project
  3. -
  4. Note the corpus ID from the Vertex AI console
  5. +
  6. + Note the corpus ID from the Vertex AI console (now labeled "RAG Engine" in Google Cloud) +
  7. Enter the corpus ID in the Vector Store ID field below
@@ -206,6 +212,10 @@ const VectorStoreForm: React.FC = ({ description={

To use Vertex AI Search (Discovery Engine):

+

+ Note: Google Cloud has renamed this to "Agent Search" in its console — the steps below still + apply. +

  1. Enable the Discovery Engine API on your Google Cloud project and create a data store following the @@ -254,11 +264,11 @@ const VectorStoreForm: React.FC = ({ From 8cf2e2eb4385d1b3ea7232865596b81a1522e9de Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Mon, 3 Aug 2026 15:09:47 -0700 Subject: [PATCH 097/123] fix(proxy): apply key/team router_settings.model_group_alias (#35486) Key and team `router_settings.model_group_alias` was accepted, persisted and echoed back by `/key/info`, but never applied at request time, so the request ran on the group the caller asked for. `route_request` forwards only the settings the Router accepts as per-request kwargs, and `model_group_alias` is not one of them: the Router resolves aliases from its own instance attribute, which holds the global config map and is shared across requests. Resolve the alias in the proxy instead, alongside the existing model-alias rewrites and ahead of the pre-call hooks, so per-model limits and guardrails key off the group that actually serves the request. Authorize the alias target before the rewrite; model access was checked against the requested group, so a key whose alias points at a group it cannot call gets the usual 403 rather than being quietly served it. Resolves LIT-4879 --- litellm/proxy/common_request_processing.py | 81 +++++-- litellm/router.py | 12 +- litellm/router_utils/common_utils.py | 21 ++ .../proxy/proxy_server/test_proxy_config.py | 42 ++++ .../proxy/test_common_request_processing.py | 229 ++++++++++++++++++ .../proxy/test_model_level_guardrails.py | 59 +++-- .../test_router_utils_common_utils.py | 42 ++++ 7 files changed, 427 insertions(+), 59 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index cb688860280..bc3bcd233f0 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -43,6 +43,7 @@ from litellm.litellm_core_utils.llm_response_utils.get_headers import ( ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import ProxyException, UserAPIKeyAuth +from litellm.proxy.auth.auth_checks import can_key_call_resolved_model from litellm.proxy.auth.auth_utils import check_response_size_is_safe from litellm.proxy.common_utils.callback_utils import ( get_logging_caching_headers, @@ -53,6 +54,7 @@ from litellm.proxy.route_llm_request import route_request from litellm.proxy.utils import ProxyLogging, _check_and_merge_model_level_guardrails from litellm.router import Router from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict +from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.types.guardrails import GuardrailEventHooks from litellm.types.router import RouterRateLimitError from litellm.types.utils import ServerToolUse @@ -384,6 +386,39 @@ async def _authorize_response_file_search_vector_stores( ) +async def _resolve_per_request_model_group_alias( + requested_model: object, + router_settings: Mapping[str, object], + user_api_key_dict: UserAPIKeyAuth, + llm_router: Router, +) -> str | None: + """ + Resolve ``router_settings.model_group_alias`` coming from a key or team. + + The Router only ever resolves aliases from its own instance attribute, which + holds the global config map and is shared across requests, so a per-request + map has to be applied here instead of being forwarded to the Router. + + Model access was authorized against the requested group, so the target is + authorized in its own right before the rewrite; a key that may not call the + target gets the usual 403 rather than being quietly served it. + + Returns the target model group, or None when no alias applies. + """ + if not isinstance(requested_model, str): + return None + target = resolve_model_group_alias(router_settings.get("model_group_alias"), requested_model) + if target is None or target == requested_model: + return None + await can_key_call_resolved_model( + model=target, + llm_model_list=llm_router.model_list, + valid_token=user_api_key_dict, + llm_router=llm_router, + ) + return target + + async def _parse_event_data_for_error(event_line: str | bytes) -> int | None: """Parses an event line and returns an error code if present, else None.""" event_line = event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line @@ -1285,6 +1320,35 @@ class ProxyBaseLLMRequestProcessing: ): self.data["model"] = user_api_key_dict.aliases[self.data["model"]] + # Apply hierarchical router_settings (Key > Team) + # Global router_settings are already on the Router object itself. + # This sits with the other alias rewrites, and ahead of the guardrail + # merge and the pre-call hooks, so everything that keys off the model + # group -- model-level guardrails, per-model budgets and rate limits, + # the logging object -- sees the group that will actually serve. + if llm_router is not None and proxy_config is not None: + from litellm.proxy.proxy_server import prisma_client + + router_settings = await proxy_config._get_hierarchical_router_settings( + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + ) + + # If router_settings found (from key or team), apply them + # Pass settings as per-request overrides instead of creating a new Router + # This avoids expensive Router instantiation on each request + if router_settings is not None: + self.data["router_settings_override"] = router_settings + alias_target = await _resolve_per_request_model_group_alias( + requested_model=self.data.get("model"), + router_settings=router_settings, + user_api_key_dict=user_api_key_dict, + llm_router=llm_router, + ) + if alias_target is not None: + self.data["model"] = alias_target + self.data["litellm_call_id"] = request.headers.get("x-litellm-call-id", str(uuid.uuid4())) DDSpanTagger.tag_call_id(self.data.get("litellm_call_id")) DDSpanTagger.tag_request( @@ -1339,23 +1403,6 @@ class ProxyBaseLLMRequestProcessing: call_type=route_type, # type: ignore ) - # Apply hierarchical router_settings (Key > Team) - # Global router_settings are already on the Router object itself. - if llm_router is not None and proxy_config is not None: - from litellm.proxy.proxy_server import prisma_client - - router_settings = await proxy_config._get_hierarchical_router_settings( - user_api_key_dict=user_api_key_dict, - prisma_client=prisma_client, - proxy_logging_obj=proxy_logging_obj, - ) - - # If router_settings found (from key or team), apply them - # Pass settings as per-request overrides instead of creating a new Router - # This avoids expensive Router instantiation on each request - if router_settings is not None: - self.data["router_settings_override"] = router_settings - if "messages" in self.data and self.data["messages"]: logging_obj.update_messages(self.data["messages"]) diff --git a/litellm/router.py b/litellm/router.py index e6613e1d302..6bf1bdfc670 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -115,6 +115,7 @@ from litellm.router_utils.common_utils import ( _is_proxy_admin_request, filter_team_based_models, filter_web_search_deployments, + resolve_model_group_alias, ) from litellm.router_utils.cooldown_cache import CooldownCache from litellm.router_utils.cooldown_handlers import ( @@ -10331,16 +10332,7 @@ class Router: - str, the litellm model name - None, if model is not in model group alias """ - if model not in self.model_group_alias: - return None - - _item = self.model_group_alias[model] - if isinstance(_item, str): - model = _item - else: - model = _item["model"] - - return model + return resolve_model_group_alias(self.model_group_alias, model) def _get_deployment_by_litellm_model(self, model: str) -> list: """ diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 189296a8955..bf1c814c049 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -22,6 +22,27 @@ def _is_proxy_admin_request(request_kwargs: Mapping[str, object] | None) -> bool return getattr(user_api_key_auth, "user_role", None) == "proxy_admin" +def resolve_model_group_alias(model_group_alias: object, model: str) -> str | None: + """ + Resolve ``model`` through a ``model_group_alias`` map. + + Handles both supported entry shapes, the plain string form + ``{"alias": "target"}`` and the item form + ``{"alias": {"model": "target", "hidden": true}}``, and tolerates malformed + entries: the map can come from a key or team row rather than from validated + config, so a bad value must not raise mid-request. + + Returns the target model group, or None when the map does not rewrite ``model``. + """ + if not isinstance(model_group_alias, Mapping): + return None + entry = model_group_alias.get(model) + target = entry.get("model") if isinstance(entry, Mapping) else entry + if not isinstance(target, str) or not target: + return None + return target + + def get_litellm_params_sensitive_credential_hash(litellm_params: dict) -> str: """ Hash of the credential params, used for mapping the file id to the right model diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 9a3702d2355..28d4d87e26f 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -2016,6 +2016,48 @@ async def test_ProxyConfig__get_hierarchical_router_settings_missing_returns_non assert out is None +@pytest.mark.asyncio +async def test_ProxyConfig__get_hierarchical_router_settings_falls_back_to_team(monkeypatch): + """A key with no router_settings inherits the team's, so a team-level + model_group_alias reaches the request path at all.""" + pc = ProxyConfig() + fake_key = SimpleNamespace(router_settings=None, team_id="team-1") + team_settings = {"model_group_alias": {"group-a": "group-b"}} + monkeypatch.setattr( + "litellm.proxy.proxy_server.get_team_object", + AsyncMock(return_value=SimpleNamespace(router_settings=team_settings)), + ) + + out = await pc._get_hierarchical_router_settings( + user_api_key_dict=fake_key, + prisma_client=None, + proxy_logging_obj=None, + ) + + assert out == team_settings + + +@pytest.mark.asyncio +async def test_ProxyConfig__get_hierarchical_router_settings_key_shadows_team_entirely(monkeypatch): + """Resolution returns whichever object it finds first, it does not merge + per field, so a key that sets any router setting hides every team setting + including an alias the key itself never set.""" + pc = ProxyConfig() + fake_key = SimpleNamespace(router_settings={"num_retries": 3}, team_id="team-1") + team_lookup = AsyncMock(return_value=SimpleNamespace(router_settings={"model_group_alias": {"group-a": "group-b"}})) + monkeypatch.setattr("litellm.proxy.proxy_server.get_team_object", team_lookup) + + out = await pc._get_hierarchical_router_settings( + user_api_key_dict=fake_key, + prisma_client=None, + proxy_logging_obj=None, + ) + + assert out == {"num_retries": 3} + assert "model_group_alias" not in out + team_lookup.assert_not_called() + + # --------------------------------------------------------------------------- # ProxyConfig._add_router_settings_from_db_config # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 3bb84e095a0..4d98a05da8d 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -1,6 +1,7 @@ import asyncio import copy import datetime +from types import SimpleNamespace from typing import AsyncGenerator, Callable, Optional from unittest.mock import AsyncMock, MagicMock, patch @@ -28,11 +29,14 @@ from litellm.proxy.common_request_processing import ( _is_azure_model_router_request, _override_openai_response_model, _parse_event_data_for_error, + _resolve_per_request_model_group_alias, _should_return_raw_model_name, _UpstreamClosingStreamingResponse, create_response, ) from litellm.proxy.dd_span_tagger import DDSpanTagger +from litellm.proxy._types import ProxyException +from litellm.proxy._types import UserAPIKeyAuth as ProxyUserAPIKeyAuth from litellm.proxy.utils import ProxyLogging @@ -5354,3 +5358,228 @@ class TestModelDeploymentsSupportStreamOptions: def test_non_string_model_is_not_injected(self): assert self._support(None, None) is False + + +class TestPerRequestModelGroupAlias: + """``router_settings.model_group_alias`` on a key or team has to be resolved + by the proxy: the Router resolves aliases from its own shared instance + attribute, which only ever holds the global config map.""" + + @staticmethod + def _router() -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": "group-a", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"}, + }, + { + "model_name": "group-b", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-fake"}, + }, + ] + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "alias_map, expected", + [ + ({"group-a": "group-b"}, "group-b"), + ({"group-a": {"model": "group-b", "hidden": True}}, "group-b"), + ({"group-b": "group-a"}, None), + ({"group-a": "group-a"}, None), + ({"group-a": {"hidden": True}}, None), + ({}, None), + (None, None), + ], + ) + async def test_resolves_alias_for_the_requested_model_group(self, alias_map, expected): + resolved = await _resolve_per_request_model_group_alias( + requested_model="group-a", + router_settings={"model_group_alias": alias_map}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash", models=[]), + llm_router=self._router(), + ) + + assert resolved == expected + + @pytest.mark.asyncio + async def test_alias_target_outside_the_key_allowlist_is_rejected(self): + """Access was authorized against the requested group, so a rewrite that + the key could not have requested directly must not be served.""" + with pytest.raises(ProxyException) as exc_info: + await _resolve_per_request_model_group_alias( + requested_model="group-a", + router_settings={"model_group_alias": {"group-a": "group-b"}}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash", models=["group-a"]), + llm_router=self._router(), + ) + + assert exc_info.value.code == "403" + assert "group-b" in exc_info.value.message + + @pytest.mark.asyncio + async def test_alias_target_inside_the_key_allowlist_resolves(self): + resolved = await _resolve_per_request_model_group_alias( + requested_model="group-a", + router_settings={"model_group_alias": {"group-a": "group-b"}}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash", models=["group-a", "group-b"]), + llm_router=self._router(), + ) + + assert resolved == "group-b" + + @pytest.mark.asyncio + @pytest.mark.parametrize("requested_model", [None, ["group-a", "group-b"]]) + async def test_non_string_requested_model_is_left_alone(self, requested_model): + """The routed model is not always a string (a batch request carries a + list), and an unhashable one must not blow up the alias lookup.""" + resolved = await _resolve_per_request_model_group_alias( + requested_model=requested_model, + router_settings={"model_group_alias": {"group-a": "group-b"}}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash", models=[]), + llm_router=self._router(), + ) + + assert resolved is None + + @pytest.mark.asyncio + async def test_pre_call_logic_rewrites_the_requested_model(self, monkeypatch): + """End to end through the request path: a key carrying the alias must + leave pre-call processing pointing at the alias target, not at the + group the caller asked for.""" + processing_obj = ProxyBaseLLMRequestProcessing(data={"model": "group-a"}) + mock_request = MagicMock(spec=Request) + mock_request.headers = {} + + async def mock_add_litellm_data_to_request(*args, **kwargs): + return kwargs.get("data", {}) + + async def passthrough_pre_call_hook(user_api_key_dict, data, call_type): + return copy.deepcopy(data) + + mock_proxy_logging_obj = MagicMock(spec=ProxyLogging) + mock_proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=passthrough_pre_call_hook) + monkeypatch.setattr( + litellm.proxy.common_request_processing, + "add_litellm_data_to_request", + mock_add_litellm_data_to_request, + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", MagicMock()) + + mock_proxy_config = MagicMock(spec=ProxyConfig) + mock_proxy_config._get_hierarchical_router_settings = AsyncMock( + return_value={"model_group_alias": {"group-a": "group-b"}} + ) + + returned_data, _ = await processing_obj.common_processing_pre_call_logic( + request=mock_request, + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash", models=[]), + proxy_logging_obj=mock_proxy_logging_obj, + proxy_config=mock_proxy_config, + route_type="acompletion", + llm_router=self._router(), + ) + + assert returned_data["model"] == "group-b" + assert returned_data["router_settings_override"] == {"model_group_alias": {"group-a": "group-b"}} + # The rewrite has to land before the pre-call hooks: they are where + # per-model budgets and rate limits are enforced, so resolving later + # applies the requested group's limits to a call the target serves. + assert mock_proxy_logging_obj.pre_call_hook.call_args.kwargs["data"]["model"] == "group-b" + + @pytest.mark.asyncio + async def test_team_level_alias_rewrites_the_requested_model(self, monkeypatch): + """The team path is separate resolution, not a variant of the key path: + settings are looked up on the team only when the key carries none. Runs + the real hierarchical lookup rather than mocking it, so this covers the + team half of the fix end to end.""" + from litellm.proxy.proxy_server import ProxyConfig as RealProxyConfig + + processing_obj = ProxyBaseLLMRequestProcessing(data={"model": "group-a"}) + mock_request = MagicMock(spec=Request) + mock_request.headers = {} + + async def mock_add_litellm_data_to_request(*args, **kwargs): + return kwargs.get("data", {}) + + async def passthrough_pre_call_hook(user_api_key_dict, data, call_type): + return copy.deepcopy(data) + + mock_proxy_logging_obj = MagicMock(spec=ProxyLogging) + mock_proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=passthrough_pre_call_hook) + monkeypatch.setattr( + litellm.proxy.common_request_processing, + "add_litellm_data_to_request", + mock_add_litellm_data_to_request, + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", MagicMock()) + monkeypatch.setattr( + "litellm.proxy.proxy_server.get_team_object", + AsyncMock(return_value=SimpleNamespace(router_settings={"model_group_alias": {"group-a": "group-b"}})), + ) + + returned_data, _ = await processing_obj.common_processing_pre_call_logic( + request=mock_request, + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash", models=[], team_id="team-1"), + proxy_logging_obj=mock_proxy_logging_obj, + proxy_config=RealProxyConfig(), + route_type="acompletion", + llm_router=self._router(), + ) + + assert returned_data["model"] == "group-b" + + @pytest.mark.asyncio + async def test_model_level_guardrails_resolve_against_the_alias_target(self, monkeypatch): + """Model-level guardrails are merged by model group name, so the merge + must see the target rather than the group the caller named.""" + processing_obj = ProxyBaseLLMRequestProcessing(data={"model": "group-a"}) + mock_request = MagicMock(spec=Request) + mock_request.headers = {} + + async def mock_add_litellm_data_to_request(*args, **kwargs): + return kwargs.get("data", {}) + + async def passthrough_pre_call_hook(user_api_key_dict, data, call_type): + return copy.deepcopy(data) + + mock_proxy_logging_obj = MagicMock(spec=ProxyLogging) + mock_proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=passthrough_pre_call_hook) + monkeypatch.setattr( + litellm.proxy.common_request_processing, + "add_litellm_data_to_request", + mock_add_litellm_data_to_request, + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", MagicMock()) + + merged_for: list = [] + + def recording_merge(data, llm_router, trust_client_model_info=True): + merged_for.append(data.get("model")) + return data + + monkeypatch.setattr( + litellm.proxy.common_request_processing, + "_check_and_merge_model_level_guardrails", + recording_merge, + ) + + mock_proxy_config = MagicMock(spec=ProxyConfig) + mock_proxy_config._get_hierarchical_router_settings = AsyncMock( + return_value={"model_group_alias": {"group-a": "group-b"}} + ) + + await processing_obj.common_processing_pre_call_logic( + request=mock_request, + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash", models=[]), + proxy_logging_obj=mock_proxy_logging_obj, + proxy_config=mock_proxy_config, + route_type="acompletion", + llm_router=self._router(), + ) + + assert merged_for == ["group-b"] diff --git a/tests/test_litellm/proxy/test_model_level_guardrails.py b/tests/test_litellm/proxy/test_model_level_guardrails.py index 48163bf5ed5..a1278e399b5 100644 --- a/tests/test_litellm/proxy/test_model_level_guardrails.py +++ b/tests/test_litellm/proxy/test_model_level_guardrails.py @@ -598,10 +598,15 @@ async def test_pre_call_merges_model_level_guardrails_before_pre_call_hook(): } ) - captured_pre_call_data: dict = {} + captured_pre_call_guardrails: list = [] async def fake_pre_call_hook(*, user_api_key_dict, data, call_type): - captured_pre_call_data.update(data) + # Snapshot the list rather than the dict: metadata is shared by + # reference, so a merge that happens after this point would otherwise + # show up here retroactively and the assertion would pass either way. + captured_pre_call_guardrails.extend( + (data.get("metadata") or {}).get("guardrails") or data.get("guardrails") or [] + ) return data proxy_logging = MagicMock() @@ -616,13 +621,9 @@ async def test_pre_call_merges_model_level_guardrails_before_pre_call_hook(): proxy_config = MagicMock() proxy_config._get_hierarchical_router_settings = AsyncMock(return_value=None) - # Stop the function before any post-pre_call_hook logic so we can keep - # the test focused. Raising _StopAfterPreCall in the next await fires - # right after the guardrail merge + pre_call_hook complete. - class _StopAfterPreCall(Exception): - pass - - proxy_config._get_hierarchical_router_settings.side_effect = _StopAfterPreCall() + # Assert on what pre_call_hook was handed rather than short-circuiting the + # function part way through: a sentinel keyed to one particular later call + # silently stops testing the ordering as soon as that call moves. with ( patch( @@ -640,30 +641,24 @@ async def test_pre_call_merges_model_level_guardrails_before_pre_call_hook(): ): from litellm.proxy._types import UserAPIKeyAuth - try: - await processing.common_processing_pre_call_logic( - request=MagicMock(headers={}, url=MagicMock(path="/v1/chat/completions")), - general_settings={}, - user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), - proxy_logging_obj=proxy_logging, - proxy_config=proxy_config, - route_type="acompletion", - version=None, - user_model=None, - user_temperature=None, - user_request_timeout=None, - user_max_tokens=None, - user_api_base=None, - model=None, - llm_router=mock_router, - ) - except _StopAfterPreCall: - pass + await processing.common_processing_pre_call_logic( + request=MagicMock(headers={}, url=MagicMock(path="/v1/chat/completions")), + general_settings={}, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + proxy_logging_obj=proxy_logging, + proxy_config=proxy_config, + route_type="acompletion", + version=None, + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + model=None, + llm_router=mock_router, + ) # The pre_call_hook must have received data with the model-level # guardrail already merged in. Before the fix, this assertion fails # because pre_call_hook saw the original data without merge. - merged = (captured_pre_call_data.get("metadata") or {}).get("guardrails") or ( - captured_pre_call_data.get("guardrails") or [] - ) - assert "my-pre-call-guardrail" in merged + assert "my-pre-call-guardrail" in captured_pre_call_guardrails diff --git a/tests/test_litellm/router_utils/test_router_utils_common_utils.py b/tests/test_litellm/router_utils/test_router_utils_common_utils.py index efa5f2382dc..7d453c72652 100644 --- a/tests/test_litellm/router_utils/test_router_utils_common_utils.py +++ b/tests/test_litellm/router_utils/test_router_utils_common_utils.py @@ -10,6 +10,7 @@ from litellm.router_utils.common_utils import ( add_model_file_id_mappings, filter_team_based_models, filter_web_search_deployments, + resolve_model_group_alias, ) @@ -516,3 +517,44 @@ class TestAddModelFileIdMappings: def test_should_return_empty_mapping_when_given_empty_list(self): result = add_model_file_id_mappings([], []) assert result == {} + + +class TestResolveModelGroupAlias: + """``model_group_alias`` maps reach this helper from validated config and + from key/team rows, so both entry shapes must resolve and malformed entries + must not raise mid-request.""" + + @pytest.mark.parametrize( + "alias_map, expected", + [ + ({"group-a": "group-b"}, "group-b"), + ({"group-a": {"model": "group-b", "hidden": True}}, "group-b"), + ({"group-a": {"model": "group-b"}}, "group-b"), + ({"other": "group-b"}, None), + ({}, None), + (None, None), + ("not-a-map", None), + ({"group-a": {"hidden": True}}, None), + ({"group-a": {"model": 5}}, None), + ({"group-a": 5}, None), + ({"group-a": None}, None), + ({"group-a": ""}, None), + ], + ) + def test_resolves_both_entry_shapes_and_tolerates_malformed_entries(self, alias_map, expected): + assert resolve_model_group_alias(alias_map, "group-a") == expected + + def test_router_alias_resolution_uses_the_shared_helper(self): + router = Router( + model_list=[ + { + "model_name": "group-b", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"}, + } + ], + model_group_alias={"group-a": "group-b", "group-item": {"model": "group-b", "hidden": True}}, + ) + + assert router._get_model_from_alias("group-a") == "group-b" + assert router._get_model_from_alias("group-item") == "group-b" + assert router._get_model_from_alias("group-b") is None From 8ad5d144a1fe0c6b06b35a0777e871277c8f8a47 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 3 Aug 2026 15:18:27 -0700 Subject: [PATCH 098/123] feat(complexity_router): default session affinity off and expose it in the UI (#35714) * feat(ui): expose an Auto-Router session affinity toggle session_affinity on ComplexityRouterConfig defaults to True, and neither the create form nor the edit modal ever emitted the key, so every auto-router built in the UI silently pinned each session to its first turn's model for an hour with no way to see or change that. Adds an "Advanced: Session Affinity" switch to both surfaces, defaulted on to match the backend field. Both paths now write the key explicitly instead of falling through to the backend default, so a stored config states what the router actually does. A stored config with the key absent hydrates as on, since those routers are running with affinity enabled today; showing them as off would report the opposite of reality and persist it on the next save. * feat(complexity_router): default session affinity off and expose it in the UI session_affinity defaulted to True and the Auto-Router UI never emitted the key, so every router built there silently pinned each session to whatever model its first turn classified into for an hour, refreshed on every hit. There was no way to see that from the UI and no way to change it without hand-editing config.yaml. The default flips to False, so every turn is classified on its own merits and lands on the cheapest adequate tier. Pinning is now opt-in. The toggle added in the previous commit follows the field: it renders off, and both the create tab and the edit modal keep writing the key explicitly, so a stored config states what the router does instead of inheriting a default that can move under it. Behavior change for existing routers: those created before this have no session_affinity key stored, so they pick up the new default and start reclassifying every turn. That gives up the provider prompt cache the pin was preserving, and a multi-turn session can now change model between turns. Set session_affinity: true to keep the old behavior. --- .../complexity_router/config.py | 8 +- .../router_strategy/test_complexity_router.py | 26 +++---- .../add_model/ComplexityRouterConfig.tsx | 28 +++++++ .../add_model/add_auto_router_tab.test.tsx | 36 +++++++++ .../add_model/add_auto_router_tab.tsx | 3 + .../build_complexity_router_config.test.ts | 18 ++++- .../build_complexity_router_config.ts | 4 + ...d_updated_complexity_router_config.test.ts | 25 +++++++ .../edit_auto_router_modal.test.ts | 2 + .../edit_auto_router_modal.test.tsx | 73 +++++++++++++++++++ .../edit_auto_router_modal.tsx | 7 ++ 11 files changed, 209 insertions(+), 21 deletions(-) diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 970d8de8575..1fa98f13c25 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -426,13 +426,13 @@ class ComplexityRouterConfig(BaseModel): # Session affinity: pin the first turn's routed model for the rest of the session session_affinity: bool = Field( - default=True, + default=False, description=( "When True and a session_id is resolvable on the request, pin the model chosen on the " "session's first turn and reuse it for every later turn, skipping re-classification. " - "On by default so multi-turn sessions stay on one model, preserving provider prompt " - "caches and avoiding cross-model conversation-history errors. Set False to reclassify " - "every turn." + "Off by default so every turn is classified on its own merits and routed to the cheapest " + "adequate tier. Set True to keep a multi-turn session on one model, which preserves " + "provider prompt caches and avoids cross-model conversation-history errors." ), ) session_affinity_ttl_seconds: int = Field( diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index cc73273450a..3a94b1e0f85 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -2903,7 +2903,7 @@ class TestRoutingDecisionCauseLogging: class TestSessionAffinity: - """Test the session_affinity sticky-routing behavior (on by default).""" + """Test the session_affinity sticky-routing behavior (off by default).""" REASONING_MESSAGE = [ { @@ -2917,18 +2917,14 @@ class TestSessionAffinity: def session_affinity_config(self, basic_config) -> Dict: return {**basic_config, "session_affinity": True} - @pytest.fixture - def session_affinity_disabled_config(self, basic_config) -> Dict: - return {**basic_config, "session_affinity": False} - @staticmethod def _request_kwargs(session_id: str) -> Dict: return {"metadata": {"session_id": session_id}} @pytest.mark.asyncio - async def test_enabled_by_default_pins_model(self, mock_router_instance, basic_config): - """Regression: session_affinity defaults to True, so a shared session_id pins the - first turn's model and later turns reuse it instead of reclassifying.""" + async def test_disabled_by_default_reclassifies_every_turn(self, mock_router_instance, basic_config): + """Regression: session_affinity defaults to False, so a shared session_id must NOT + pin the first turn's model; every turn is classified on its own merits.""" assert "session_affinity" not in basic_config mock_router_instance.cache = DualCache() router = ComplexityRouter( @@ -2944,19 +2940,17 @@ class TestSessionAffinity: model="test-model", request_kwargs=request_kwargs, messages=self.SIMPLE_MESSAGE ) assert first.model == "o1-preview" - assert second.model == "o1-preview" + assert second.model == "gpt-4o-mini" @pytest.mark.asyncio - async def test_can_be_disabled_reclassifies_every_turn( - self, mock_router_instance, session_affinity_disabled_config - ): - """Regression: session_affinity=False must still reclassify every turn even when a - shared session_id is present, so the opt-out keeps working.""" + async def test_can_be_enabled_to_pin_every_later_turn(self, mock_router_instance, session_affinity_config): + """Regression: session_affinity=True is the opt-in, so a shared session_id reuses the + first turn's model instead of reclassifying.""" mock_router_instance.cache = DualCache() router = ComplexityRouter( model_name="test-router", litellm_router_instance=mock_router_instance, - complexity_router_config=session_affinity_disabled_config, + complexity_router_config=session_affinity_config, ) request_kwargs = self._request_kwargs("session-1") first = await router.async_pre_routing_hook( @@ -2966,7 +2960,7 @@ class TestSessionAffinity: model="test-model", request_kwargs=request_kwargs, messages=self.SIMPLE_MESSAGE ) assert first.model == "o1-preview" - assert second.model == "gpt-4o-mini" + assert second.model == "o1-preview" @pytest.mark.asyncio async def test_pins_model_after_first_turn(self, mock_router_instance, session_affinity_config): diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 0503c0c9c6d..b83808b0728 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -14,6 +14,7 @@ export const DEFAULT_CLASSIFIER_TIMEOUT_MS = 3000; export const DEFAULT_TIER_DISTANCE_PENALTY = 0.5; export const DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE = 3; export const DEFAULT_CLASSIFIER_CONTEXT_PER_TURN_CHARS = 200; +export const DEFAULT_SESSION_AFFINITY = false; export interface ComplexityTiers { SIMPLE: string[]; @@ -45,6 +46,7 @@ export interface ComplexityRouterConfigValue { classifier_context_window_size?: number; classifier_context_per_turn_chars?: number; classifier_context_include_assistant_turns?: boolean; + session_affinity?: boolean; adaptive?: boolean; adaptive_weights?: AdaptiveRouterWeights; tier_distance_penalty?: number; @@ -224,6 +226,32 @@ const ComplexityRouterConfig: React.FC = ({ ), children: , }, + { + key: "session-affinity", + label: ( + + Advanced: Session Affinity + + ), + children: ( + <> +
    + onChange({ ...value, session_affinity: sessionAffinity })} + aria-label="Pin a session to its first model" + /> + Pin a session to its first model +
    + + Off by default: every turn is classified on its own merits and routed to the cheapest adequate tier. + Turn this on to reuse the model chosen on a session's first turn for every later turn, which + preserves provider prompt caches and avoids cross-model conversation-history errors, at the cost of + keeping the whole session on the first turn's tier. + + + ), + }, { key: "response", label: ( diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index fa31c7d9c9d..f7cc9a1deae 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -111,4 +111,40 @@ describe("AddAutoRouterTab", () => { expect(await screen.findByText("Please select a team to continue")).toBeInTheDocument(); expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled(); }); + + it("defaults a new router to session affinity off, matching the backend field default", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "affinity-router"); + await user.click(screen.getByText("Advanced: Session Affinity")); + expect(await screen.findByRole("switch", { name: "Pin a session to its first model" })).not.toBeChecked(); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({ + session_affinity: false, + }); + }); + + it("carries session affinity turned on through to the create payload", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "affinity-router"); + await user.click(screen.getByText("Advanced: Session Affinity")); + await user.click(await screen.findByRole("switch", { name: "Pin a session to its first model" })); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({ + session_affinity: true, + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 4593f6a6a2e..ea75bd8e283 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -10,6 +10,7 @@ import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_m import ComplexityRouterConfig, { ComplexityRouterConfigValue, DEFAULT_ADAPTIVE_WEIGHTS, + DEFAULT_SESSION_AFFINITY, DEFAULT_TIER_DISTANCE_PENALTY, } from "./ComplexityRouterConfig"; import { KeywordTierRule } from "./KeywordTierRules"; @@ -102,6 +103,7 @@ const AddAutoRouterTab: React.FC = ({ classifier_context_window_size: classifierContextWindowSize, classifier_context_per_turn_chars: classifierContextPerTurnChars, classifier_context_include_assistant_turns: classifierContextIncludeAssistantTurns, + session_affinity: sessionAffinity = DEFAULT_SESSION_AFFINITY, adaptive = false, adaptive_weights: adaptiveWeights = DEFAULT_ADAPTIVE_WEIGHTS, tier_distance_penalty: tierDistancePenalty = DEFAULT_TIER_DISTANCE_PENALTY, @@ -148,6 +150,7 @@ const AddAutoRouterTab: React.FC = ({ classifierContextWindowSize, classifierContextPerTurnChars, classifierContextIncludeAssistantTurns, + sessionAffinity, customTechnicalKeywords, keywordTierRules, semanticMatchingEnabled, diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index e939ce12904..9d784b57903 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -19,6 +19,7 @@ const baseParams: BuildComplexityRouterConfigParams = { classifierContextWindowSize: undefined, classifierContextPerTurnChars: undefined, classifierContextIncludeAssistantTurns: undefined, + sessionAffinity: false, customTechnicalKeywords: [], keywordTierRules: [], semanticMatchingEnabled: false, @@ -35,7 +36,12 @@ const baseParams: BuildComplexityRouterConfigParams = { describe("buildComplexityRouterConfig", () => { it("emits tiers, classifier_type, and escalation_keywords when nothing else is configured", () => { const config = buildComplexityRouterConfig(baseParams); - expect(config).toEqual({ tiers, classifier_type: "heuristic", escalation_keywords: ["LITELLM ESCALATE"] }); + expect(config).toEqual({ + tiers, + classifier_type: "heuristic", + session_affinity: false, + escalation_keywords: ["LITELLM ESCALATE"], + }); }); it("trims escalation keywords and drops blank entries", () => { @@ -219,6 +225,16 @@ describe("buildComplexityRouterConfig", () => { expect(config.return_raw_model_name).toBeUndefined(); }); + it("writes session_affinity=true so turning the toggle on overrides the backend's off-by-default", () => { + const config = buildComplexityRouterConfig({ ...baseParams, sessionAffinity: true }); + expect(config.session_affinity).toBe(true); + }); + + it("writes session_affinity explicitly when off, so the stored config never relies on the backend default", () => { + const config = buildComplexityRouterConfig({ ...baseParams, sessionAffinity: false }); + expect(config.session_affinity).toBe(false); + }); + it("includes return_raw_model_name when enabled", () => { const config = buildComplexityRouterConfig({ ...baseParams, returnRawModelName: true }); expect(config.return_raw_model_name).toBe(true); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 192e71b4597..cd6c697b377 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -15,6 +15,7 @@ export interface BuildComplexityRouterConfigParams { classifierContextWindowSize: number | undefined; classifierContextPerTurnChars: number | undefined; classifierContextIncludeAssistantTurns: boolean | undefined; + sessionAffinity: boolean; customTechnicalKeywords: string[]; keywordTierRules: KeywordTierRule[]; semanticMatchingEnabled: boolean; @@ -35,6 +36,7 @@ export interface ComplexityRouterConfigPayload { classifier_context_window_size?: number; classifier_context_per_turn_chars?: number; classifier_context_include_assistant_turns?: boolean; + session_affinity: boolean; custom_technical_keywords?: string[]; keyword_tier_rules?: { keywords: string[]; tier: KeywordTierRule["tier"] }[]; semantic_keyword_matching?: boolean; @@ -78,6 +80,7 @@ export const buildComplexityRouterConfig = ({ classifierContextWindowSize, classifierContextPerTurnChars, classifierContextIncludeAssistantTurns, + sessionAffinity, customTechnicalKeywords, keywordTierRules, semanticMatchingEnabled, @@ -110,6 +113,7 @@ export const buildComplexityRouterConfig = ({ classifierContextIncludeAssistantTurns !== undefined && { classifier_context_include_assistant_turns: classifierContextIncludeAssistantTurns, }), + session_affinity: sessionAffinity, ...(customTechnicalKeywords.length > 0 && { custom_technical_keywords: customTechnicalKeywords }), ...(cleanedKeywordTierRules.length > 0 && { keyword_tier_rules: cleanedKeywordTierRules }), escalation_keywords: cleanedEscalationKeywords, diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index f8d46f9ddd6..971c833a0de 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -199,3 +199,28 @@ describe("buildUpdatedComplexityRouterConfig assistant turns", () => { expect(result.classifier_context_include_assistant_turns).toBeUndefined(); }); }); + +describe("buildUpdatedComplexityRouterConfig session affinity", () => { + it("writes session_affinity=false when the toggle is off", () => { + const result = buildUpdatedComplexityRouterConfig(STORED, { ...FORM_VALUE, session_affinity: false }); + expect(result.session_affinity).toBe(false); + }); + + it("writes session_affinity=true when the toggle is on", () => { + const result = buildUpdatedComplexityRouterConfig(STORED, { ...FORM_VALUE, session_affinity: true }); + expect(result.session_affinity).toBe(true); + }); + + it("re-asserts the backend's off-by-default when the form value is absent, rather than dropping the key", () => { + const result = buildUpdatedComplexityRouterConfig({ ...STORED, session_affinity: true }, FORM_VALUE); + expect(result.session_affinity).toBe(false); + }); + + it("stops a stored session_affinity=true from surviving a save that turned the toggle back off", () => { + const result = buildUpdatedComplexityRouterConfig( + { ...STORED, session_affinity: true }, + { ...FORM_VALUE, session_affinity: false }, + ); + expect(result.session_affinity).toBe(false); + }); +}); diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts index 17fa810b529..eb5bb46f0e8 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts @@ -47,6 +47,7 @@ const expectedClassifiedTierConfig = { semantic_keyword_matching: true, embedding_model: "voyage-4-large", match_threshold: 0.65, + session_affinity: false, adaptive: true, adaptive_weights: { quality: 0.4, cost: 0.6 }, adaptive_eligible: "classified_tier", @@ -66,6 +67,7 @@ const expectedAdaptiveDisabledConfig = { semantic_keyword_matching: true, embedding_model: "voyage-4-large", match_threshold: 0.65, + session_affinity: false, }; describe("buildUpdatedComplexityRouterConfig", () => { diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx index 98b7ac519f5..c0806befa52 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx @@ -243,3 +243,76 @@ describe("EditAutoRouterModal assistant turns", () => { expect(savedConfig().classifier_context_include_assistant_turns).toBe(false); }); }); + +describe("EditAutoRouterModal session affinity", () => { + beforeEach(() => { + modelPatchUpdateCall.mockClear(); + }); + + const renderWithStoredConfig = (complexity_router_config: Record) => + renderWithProviders( + , + ); + + // A stored config with no session_affinity key now runs with affinity OFF, because the backend + // field defaults to False. The toggle has to render what the router actually does, and an + // untouched save must not flip it. + it("shows a stored config with no session_affinity key as off", async () => { + const user = userEvent.setup(); + renderWithStoredConfig(STORED_CONFIG); + + await user.click(await screen.findByText("Advanced: Session Affinity")); + expect(await screen.findByRole("switch", { name: "Pin a session to its first model" })).not.toBeChecked(); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().session_affinity).toBe(false); + }); + + it("shows a stored session_affinity=true as on and preserves it through an untouched save", async () => { + const user = userEvent.setup(); + renderWithStoredConfig({ ...STORED_CONFIG, session_affinity: true }); + + await user.click(await screen.findByText("Advanced: Session Affinity")); + expect(await screen.findByRole("switch", { name: "Pin a session to its first model" })).toBeChecked(); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().session_affinity).toBe(true); + }); + + it("persists turning session affinity on", async () => { + const user = userEvent.setup(); + renderWithStoredConfig(STORED_CONFIG); + + await user.click(await screen.findByText("Advanced: Session Affinity")); + await user.click(await screen.findByRole("switch", { name: "Pin a session to its first model" })); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().session_affinity).toBe(true); + }); + + it("persists turning session affinity back off", async () => { + const user = userEvent.setup(); + renderWithStoredConfig({ ...STORED_CONFIG, session_affinity: true }); + + await user.click(await screen.findByText("Advanced: Session Affinity")); + await user.click(await screen.findByRole("switch", { name: "Pin a session to its first model" })); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().session_affinity).toBe(false); + }); +}); diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index 99b5ff178b1..a70fc31d6fe 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -13,6 +13,7 @@ import { hydrateKeywordTierRules, serializeKeywordTierRules } from "../add_model import ComplexityRouterConfig, { ComplexityRouterConfigValue, DEFAULT_ADAPTIVE_WEIGHTS, + DEFAULT_SESSION_AFFINITY, DEFAULT_TIER_DISTANCE_PENALTY, } from "../add_model/ComplexityRouterConfig"; import NotificationsManager from "../molecules/notifications_manager"; @@ -36,6 +37,7 @@ const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "classifier_context_window_size", "classifier_context_per_turn_chars", "classifier_context_include_assistant_turns", + "session_affinity", "adaptive", "adaptive_weights", "tier_distance_penalty", @@ -101,6 +103,7 @@ export const buildUpdatedComplexityRouterConfig = ( value.classifier_context_include_assistant_turns !== undefined && { classifier_context_include_assistant_turns: value.classifier_context_include_assistant_turns, }), + session_affinity: value.session_affinity ?? DEFAULT_SESSION_AFFINITY, ...(customTechnicalKeywords && customTechnicalKeywords.length > 0 && { custom_technical_keywords: customTechnicalKeywords, @@ -218,6 +221,10 @@ const EditAutoRouterModal: React.FC = ({ typeof parsedConfig.classifier_context_include_assistant_turns === "boolean" ? parsedConfig.classifier_context_include_assistant_turns : undefined, + session_affinity: + typeof parsedConfig.session_affinity === "boolean" + ? parsedConfig.session_affinity + : DEFAULT_SESSION_AFFINITY, adaptive: parsedConfig.adaptive || false, adaptive_weights: parsedConfig.adaptive_weights, tier_distance_penalty: parsedConfig.tier_distance_penalty, From 7dab1ff75f8fb105bc4a3f783742798ebb5b61cd Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 3 Aug 2026 16:19:56 -0700 Subject: [PATCH 099/123] fix(datadog): read team callback dd_* params from kwargs instead of blocked dynamic params (#35115) (#35687) Team-scoped DD credentials (dd_api_key, dd_site) set via POST /team/{id}/callback were silently dropped because _request_blocked_callback_params blocks them from standard_callback_dynamic_params. The security block is correct for request-level injection, but team callback_vars are admin-configured and trusted. Store the raw init kwargs on the Logging instance and read dd_* params from there in _process_dynamic_callback_list instead of from standard_callback_dynamic_params. Adds an integration test that exercises the full Logging.__init__ flow with team callback_vars to prevent regression. Co-authored-by: Aanchal Khandelwal --- .../initialize_dynamic_callback_params.py | 38 ++-- litellm/litellm_core_utils/litellm_logging.py | 11 +- litellm/proxy/litellm_pre_call_utils.py | 45 ++++- litellm/types/utils.py | 7 + .../datadog/test_datadog_team_handler.py | 102 ++++++++++- .../proxy/test_litellm_pre_call_utils.py | 169 ++++++++++++++++++ 6 files changed, 354 insertions(+), 18 deletions(-) diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index 11668acb21e..171165d01be 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -1,7 +1,7 @@ -from collections.abc import Iterator +from collections.abc import Iterator, Mapping from typing import Any -from litellm.types.utils import StandardCallbackDynamicParams +from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, StandardCallbackDynamicParams _CLIENT_CALLBACK_METADATA_SLOTS: tuple[str, ...] = ("litellm_metadata", "metadata") @@ -75,14 +75,32 @@ _supported_callback_params = [ "turn_off_message_logging", ] -_request_blocked_callback_params = { - "gcs_bucket_name", - "gcs_path_service_account", - "dd_api_key", - "dd_site", - "dd_agent_host", - "dd_agent_port", -} +_request_blocked_callback_params = frozenset( + { + "gcs_bucket_name", + "gcs_path_service_account", + "dd_api_key", + "dd_site", + "dd_agent_host", + "dd_agent_port", + } +) + + +def get_trusted_callback_params(kwargs: Mapping[str, Any] | None) -> tuple[tuple[str, str], ...]: + """ + Read callback params the proxy itself stamped from admin-configured team/key callback settings. + + Request-body values never reach this field: the proxy strips it from client input before + setting it, so callbacks can consume credentials and destinations here without re-validating. + + Returned as pairs rather than a mapping because the caller keeps this on the Logging object, + which the proxy deep-copies; a mappingproxy is not copyable and a dict would be mutable. + """ + trusted_vars = kwargs.get(TRUSTED_CALLBACK_VARS_FIELD) if kwargs else None + if not isinstance(trusted_vars, Mapping): + return () + return tuple((key, str(value)) for key, value in trusted_vars.items() if isinstance(key, str)) def initialize_standard_callback_dynamic_params( diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index b00130653c5..66d82bd18f1 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -166,6 +166,9 @@ from ..integrations.s3_v2 import S3Logger as S3V2Logger from ..integrations.supabase import Supabase from ..integrations.traceloop import TraceloopLogger from .exception_mapping_utils import _get_response_headers +from .initialize_dynamic_callback_params import ( + get_trusted_callback_params, +) from .initialize_dynamic_callback_params import ( initialize_standard_callback_dynamic_params as _initialize_standard_callback_dynamic_params, ) @@ -362,6 +365,7 @@ class Logging(LiteLLMLoggingBaseClass): self.standard_callback_dynamic_params: StandardCallbackDynamicParams = ( self.initialize_standard_callback_dynamic_params(kwargs) ) + self._trusted_callback_vars: tuple[tuple[str, str], ...] = get_trusted_callback_params(kwargs) # Process dynamic callbacks (after standard_callback_dynamic_params is initialized, # so team-scoped credentials are available for callback initialization) @@ -459,9 +463,10 @@ class Logging(LiteLLMLoggingBaseClass): # pass only the relevant dynamic params as custom_logger_init_args. _custom_logger_init_args: dict | None = None if callback == "datadog": - _custom_logger_init_args = { - k: v for k, v in self.standard_callback_dynamic_params.items() if k.startswith("dd_") - } + # dd_* params are blocked from standard_callback_dynamic_params + # (request-level security); only the proxy-stamped team/key + # callback vars are admin-configured and trusted. + _custom_logger_init_args = {k: v for k, v in self._trusted_callback_vars if k.startswith("dd_")} callback_class = _init_custom_logger_compatible_class( callback, # type: ignore[arg-type] diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index d58f953d2ef..13eb41af751 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -20,6 +20,8 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + TRUSTED_CALLBACK_VARS_FIELD, + _request_blocked_callback_params, iter_client_callback_metadata_dicts, ) from litellm.litellm_core_utils.safe_json_loads import safe_json_loads @@ -356,6 +358,39 @@ def _strip_client_message_redaction_opt_out(data: dict[str, Any]) -> None: ) +def _strip_client_callback_credentials( + data: dict[str, Any], # mutable-ok: strips in place on the request body the pre-call pipeline threads through +) -> None: + """Drop callback credentials and destinations supplied by the caller. + + ``_request_blocked_callback_params`` (Datadog + GCS credentials, sites and agent + hosts) are already ignored when building ``standard_callback_dynamic_params``. + Strip them from the body and every client metadata slot as well, so a caller + cannot pair its own ``dd_site``/``dd_agent_host`` with the team's admin-configured + ``dd_api_key`` and have the resulting logs shipped to a host it controls. + + ``TRUSTED_CALLBACK_VARS_FIELD`` is proxy-owned; it is cleared here and repopulated + from team/key callback settings in ``add_litellm_data_to_request``. + """ + containers = (("body", data), *iter_client_callback_metadata_dicts(data)) + stripped = tuple( + f"{label}.{field}" + for label, container in containers + for field in _request_blocked_callback_params + if field in container + ) + for _, container in containers: + for field in _request_blocked_callback_params: + container.pop(field, None) + data.pop(TRUSTED_CALLBACK_VARS_FIELD, None) + if stripped: + verbose_proxy_logger.debug( + "Stripped client-supplied callback credentials from request: %s. " + "Configure these on the team or key callback settings instead.", + ", ".join(sorted(stripped)), + ) + + def _strip_client_pricing_overrides(data: dict[str, Any]) -> None: """Drop pricing overrides from the request body and any metadata variant. @@ -524,7 +559,8 @@ def safe_add_api_version_from_query_params(data: dict, request: Request): def convert_key_logging_metadata_to_callback( - data: AddTeamCallback, team_callback_settings_obj: TeamCallbackMetadata | None + data: AddTeamCallback, + team_callback_settings_obj: TeamCallbackMetadata | None, ) -> TeamCallbackMetadata: if team_callback_settings_obj is None: team_callback_settings_obj = TeamCallbackMetadata() @@ -1563,6 +1599,10 @@ async def add_litellm_data_to_request( if not _key_or_team_allows_client_pricing_override(user_api_key_dict): _strip_client_pricing_overrides(data) + # Same reason as the strips above: runs after the metadata string-to-dict parse + # so JSON-string metadata cannot smuggle callback credentials past the dict guard. + _strip_client_callback_credentials(data) + if not _allow_client_message_redaction_opt_out and litellm.turn_off_message_logging is True: _strip_client_message_redaction_opt_out(data) @@ -1771,6 +1811,9 @@ async def add_litellm_data_to_request( # unpack callback_vars in data for k, v in callback_settings_obj.callback_vars.items(): data[k] = v + # Callbacks that must not honour request-supplied credentials read this + # proxy-owned field instead of the raw request kwargs. + data[TRUSTED_CALLBACK_VARS_FIELD] = callback_settings_obj.callback_vars # Add disabled callbacks from key metadata if user_api_key_dict.metadata and "litellm_disabled_callbacks" in user_api_key_dict.metadata: diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 18991f53e6f..3539ac0f27a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3286,8 +3286,15 @@ agentic_loop_internal_litellm_params = [ "_code_interpreter_interception_converted_stream", ] +# Proxy-owned callback credentials, stamped from admin-configured team/key callback +# settings. Listed in all_litellm_params for the same reason as the agentic-loop +# fields above: an unrecognized top-level key is swept into extra_body and sent to +# the provider. +TRUSTED_CALLBACK_VARS_FIELD = "litellm_trusted_callback_vars" + all_litellm_params = ( agentic_loop_internal_litellm_params + + [TRUSTED_CALLBACK_VARS_FIELD] + [ "metadata", "litellm_metadata", diff --git a/tests/test_litellm/integrations/datadog/test_datadog_team_handler.py b/tests/test_litellm/integrations/datadog/test_datadog_team_handler.py index 772e993c132..09d6f51e0a8 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_team_handler.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_team_handler.py @@ -6,6 +6,7 @@ Verifies that DataDogLogger can be instantiated with per-team credentials and that the DataDogHandler correctly resolves and caches per-team loggers. """ +import copy from unittest.mock import patch import pytest @@ -13,7 +14,9 @@ import pytest from litellm.integrations.datadog.datadog import DataDogLogger from litellm.integrations.datadog.datadog_team_handler import ( DataDogHandler, - DatadogLoggingConfig, +) +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + TRUSTED_CALLBACK_VARS_FIELD, ) from litellm.litellm_core_utils.specialty_caches.dynamic_logging_cache import ( DynamicLoggingCache, @@ -94,9 +97,7 @@ class TestDataDogLoggerCredentialKwargs: assert logger.DD_API_KEY is None assert "attacker.example.com" in logger.intake_url - def test_direct_api_mode_does_not_leak_env_api_key_when_disallowed( - self, datadog_env - ): + def test_direct_api_mode_does_not_leak_env_api_key_when_disallowed(self, datadog_env): """With allow_env_credentials=False and no explicit key, init must fail rather than reuse env key.""" with pytest.raises(Exception, match="DD_API_KEY"): with patch("asyncio.create_task"): @@ -261,3 +262,96 @@ class TestStandardCallbackDynamicParamsIncludesDatadog: assert "dd_site" in annotations assert "dd_agent_host" in annotations assert "dd_agent_port" in annotations + + +def _build_logging_obj(kwargs: dict, *, with_datadog_callback: bool = True): + from litellm.litellm_core_utils.litellm_logging import Logging + + with patch("asyncio.create_task"): + return Logging( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="completion", + start_time="2026-01-01", + litellm_call_id="test-call-id", + function_id="test-func", + dynamic_success_callbacks=["datadog"] if with_datadog_callback else None, + kwargs=kwargs, + ) + + +def _dd_loggers(logging_obj) -> list[DataDogLogger]: + return [cb for cb in (logging_obj.dynamic_success_callbacks or []) if isinstance(cb, DataDogLogger)] + + +class TestTeamCallbackFlowPassesDDCredentials: + """ + dd_* credentials reach DataDogHandler only from the proxy-stamped trusted field. + + Team callback_vars are admin-configured, so they must survive + _request_blocked_callback_params; anything the caller put in the request body + must not, or a caller could pair its own dd_site with the team's dd_api_key. + """ + + def test_trusted_callback_vars_reach_datadog_handler(self, datadog_env): + trusted_vars = {"dd_api_key": "team-dd-key-123", "dd_site": "us5.datadoghq.com"} + logging_obj = _build_logging_obj( + { + TRUSTED_CALLBACK_VARS_FIELD: trusted_vars, + "model": "gpt-4", + "litellm_params": {"metadata": {}}, + } + ) + + dd_loggers = _dd_loggers(logging_obj) + assert len(dd_loggers) == 1, "DataDogLogger should be initialized from team callback_vars" + assert dd_loggers[0].DD_API_KEY == "team-dd-key-123" + assert "us5.datadoghq.com" in dd_loggers[0].intake_url + + def test_request_kwargs_dd_params_are_ignored(self, datadog_env): + """Top-level dd_* in the call kwargs are caller-controlled and must never be honoured.""" + logging_obj = _build_logging_obj( + { + "dd_api_key": "caller-dd-key", + "dd_site": "attacker.example.com", + "dd_agent_host": "attacker.example.com", + "model": "gpt-4", + "litellm_params": {"metadata": {}}, + } + ) + + dd_loggers = _dd_loggers(logging_obj) + assert len(dd_loggers) == 1 + assert dd_loggers[0].DD_API_KEY == "global_api_key" + assert "attacker.example.com" not in dd_loggers[0].intake_url + assert "us1.datadoghq.com" in dd_loggers[0].intake_url + + def test_logging_object_stays_deepcopyable(self): + """The proxy deep-copies request data, and the Logging object rides along in it.""" + logging_obj = _build_logging_obj( + { + TRUSTED_CALLBACK_VARS_FIELD: {"dd_api_key": "team-dd-key-123", "dd_site": "us5.datadoghq.com"}, + "model": "gpt-4", + "litellm_params": {"metadata": {}}, + }, + with_datadog_callback=False, + ) + + assert copy.deepcopy(logging_obj)._trusted_callback_vars == logging_obj._trusted_callback_vars + + def test_caller_cannot_redirect_team_credentials(self, datadog_env): + """The exfil shape: caller's dd_site paired with the team's dd_api_key.""" + logging_obj = _build_logging_obj( + { + TRUSTED_CALLBACK_VARS_FIELD: {"dd_api_key": "team-dd-key-123"}, + "dd_site": "attacker.example.com", + "model": "gpt-4", + "litellm_params": {"metadata": {}}, + } + ) + + dd_loggers = _dd_loggers(logging_obj) + assert len(dd_loggers) == 1 + assert dd_loggers[0].DD_API_KEY == "team-dd-key-123" + assert "attacker.example.com" not in dd_loggers[0].intake_url diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index bceefae3a9f..37642605088 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -28,6 +28,9 @@ from litellm.proxy.litellm_pre_call_utils import ( check_if_token_is_service_account, clean_headers, ) +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + TRUSTED_CALLBACK_VARS_FIELD, +) from litellm.types.utils import CredentialItem sys.path.insert( @@ -5554,3 +5557,169 @@ def test_warn_stale_team_alias_once_evicts_oldest_key_beyond_cap(monkeypatch): pre_call_utils._warn_stale_team_alias_once("key-3", "stale alias") assert list(pre_call_utils._STALE_TEAM_ALIAS_WARNING_KEYS) == ["key-2", "key-3"] + + +def _callback_credential_request_mock() -> MagicMock: + request_mock = MagicMock(spec=Request) + request_mock.url = MagicMock() + request_mock.url.path = "/v1/chat/completions" + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + return request_mock + + +_DATADOG_TEAM_KEY = UserAPIKeyAuth( + api_key="hashed-key", + team_id="team-1", + team_metadata={ + "logging": [ + { + "callback_name": "datadog", + "callback_type": "success", + "callback_vars": {"dd_api_key": "team-dd-key"}, + } + ] + }, +) + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_strips_caller_supplied_callback_credentials(): + """ + The team admin sets dd_api_key only; a caller pairing its own dd_site with that key + would ship the team's Datadog credential to a host it controls. + """ + caller_destinations = {"dd_site": "attacker.example.com", "dd_agent_host": "attacker.example.com"} + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hello"}], + **caller_destinations, + "gcs_bucket_name": "attacker-bucket", + TRUSTED_CALLBACK_VARS_FIELD: {"dd_site": "smuggled.example.com"}, + "metadata": {**caller_destinations, "safe_user_metadata": "kept"}, + "litellm_metadata": dict(caller_destinations), + "litellm_params": {"metadata": dict(caller_destinations)}, + } + + updated = await add_litellm_data_to_request( + data=data, + request=_callback_credential_request_mock(), + user_api_key_dict=_DATADOG_TEAM_KEY, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert "dd_site" not in updated + assert "dd_agent_host" not in updated + assert "gcs_bucket_name" not in updated + assert updated["dd_api_key"] == "team-dd-key" + assert updated[TRUSTED_CALLBACK_VARS_FIELD] == {"dd_api_key": "team-dd-key"} + for metadata_key in ("metadata", "litellm_metadata"): + assert "dd_site" not in updated[metadata_key] + assert "dd_agent_host" not in updated[metadata_key] + assert "dd_site" not in updated["litellm_params"]["metadata"] + assert updated["metadata"]["safe_user_metadata"] == "kept" + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_strips_caller_supplied_callback_credentials_with_clientside_creds_allowed(): + """`allow_client_side_credentials` opens the auth-layer ban; the strip must still hold.""" + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hello"}], + "dd_site": "attacker.example.com", + } + + updated = await add_litellm_data_to_request( + data=data, + request=_callback_credential_request_mock(), + user_api_key_dict=_DATADOG_TEAM_KEY, + proxy_config=MagicMock(), + general_settings={"allow_client_side_credentials": True}, + version="test-version", + ) + + assert "dd_site" not in updated + assert updated[TRUSTED_CALLBACK_VARS_FIELD] == {"dd_api_key": "team-dd-key"} + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_omits_trusted_callback_vars_without_team_callbacks(): + """Without team/key callback settings the trusted field must not exist for a callback to read.""" + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hello"}], + TRUSTED_CALLBACK_VARS_FIELD: {"dd_api_key": "caller-key", "dd_site": "attacker.example.com"}, + } + + updated = await add_litellm_data_to_request( + data=data, + request=_callback_credential_request_mock(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert TRUSTED_CALLBACK_VARS_FIELD not in updated + + +def test_trusted_callback_vars_never_reach_the_provider(): + """ + The stamped field rides the request body, so it has to be a recognised litellm param; + otherwise the OpenAI param builder sweeps it into extra_body and the provider 400s. + """ + from litellm.utils import get_non_default_completion_params + + non_default = get_non_default_completion_params( + { + "model": "gpt-4", + TRUSTED_CALLBACK_VARS_FIELD: {"dd_api_key": "team-dd-key"}, + "some_provider_param": "kept", + } + ) + + assert TRUSTED_CALLBACK_VARS_FIELD not in non_default + assert non_default["some_provider_param"] == "kept" + + +@pytest.mark.asyncio +async def test_key_level_callback_vars_survive_the_strip(): + """ + Key-level callbacks configure their own destination and credentials, and they replace + team settings rather than merging with them, so only the request body is untrusted. + """ + key_with_datadog_callback = UserAPIKeyAuth( + api_key="hashed-key", + metadata={ + "logging": [ + { + "callback_name": "datadog", + "callback_type": "success", + "callback_vars": {"dd_api_key": "key-dd-key", "dd_site": "us5.datadoghq.com"}, + } + ] + }, + ) + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hello"}], + "dd_site": "attacker.example.com", + } + + updated = await add_litellm_data_to_request( + data=data, + request=_callback_credential_request_mock(), + user_api_key_dict=key_with_datadog_callback, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert updated[TRUSTED_CALLBACK_VARS_FIELD] == {"dd_api_key": "key-dd-key", "dd_site": "us5.datadoghq.com"} + assert updated["dd_site"] == "us5.datadoghq.com" From cd3b7ef4271c622e55dfcdc4c5c4ed0b6744bb79 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 3 Aug 2026 13:31:45 -0700 Subject: [PATCH 100/123] test(ui): tier the MCP create tests into unit and integration Adds 61 unit tests on the modules #35694 extracted: 46 on the payload builder, 15 on the OAuth redirect snapshot. They run in 9ms against 240s for the 77 full-render tests they partly replace. Nine of nine mutants were killed when the extracted logic was deliberately broken, so the speed does not come at the cost of signal. Deletes six cases across four blocks that rendered the whole modal to assert one payload key belonging to a field they never touched. Every test that proves a form field reaches the right payload key stays; those cover field to form value to payload, which a unit test cannot reach. Replaces "should not render when user is not an admin", which asserted the admin title was absent and so passed for the wrong reason: the modal does render for a non-admin, retitled. registerMCPServer was mocked but never asserted anywhere, leaving the whole non-admin submission path uncovered. It now drives a real submit and asserts the call lands there and never on createMCPServer. Renames the slow file to CreateMCPServer.integration.test.tsx and documents the three tiers in the dashboard CLAUDE.md. No production code changes. --- ui/litellm-dashboard/CLAUDE.md | 6 + ...x => CreateMCPServer.integration.test.tsx} | 144 +++------ .../_components/createOAuthUiState.test.ts | 126 ++++++++ .../_components/createServerPayload.test.ts | 296 ++++++++++++++++++ 4 files changed, 476 insertions(+), 96 deletions(-) rename ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/{CreateMCPServer.test.tsx => CreateMCPServer.integration.test.tsx} (96%) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createServerPayload.test.ts diff --git a/ui/litellm-dashboard/CLAUDE.md b/ui/litellm-dashboard/CLAUDE.md index 5ec9392d2b0..701b37ec6aa 100644 --- a/ui/litellm-dashboard/CLAUDE.md +++ b/ui/litellm-dashboard/CLAUDE.md @@ -3,3 +3,9 @@ Never put LiteLLM tokens or API keys in `localStorage`. `localStorage` survives When you fix lint violations that are grandfathered in `eslint-suppressions.json`, run `eslint . --prune-suppressions` and commit the updated baseline so the gate ratchets down instead of leaving a stale suppression `src/lib/http/schema.d.ts` is generated from the proxy's OpenAPI spec; never hand-edit it. After changing a backend route or response model that the dashboard consumes, run `npm run gen:api` and commit the result (CI `Check UI API Types Sync` enforces this) + +Tests come in three tiers, named by the standard definitions. `Foo.test.tsx` is a unit test: one module, collaborators replaced by doubles, no multi-component tree, and it should run in milliseconds. `Foo.integration.test.tsx` renders a real component tree with real children and only stubs the network boundary; it costs seconds per case, so it earns its place by proving wiring that a unit test cannot reach. Browser-level tests live in `tests/e2e/ui/` as Playwright specs against a live proxy + +When a component holds logic worth asserting, extract the logic and unit-test it there rather than driving it through a render. `CreateMCPServer` is the worked example: its payload building lives in `createServerPayload.ts` with 46 unit tests that run in single-digit milliseconds, while `CreateMCPServer.integration.test.tsx` keeps only the cases that prove a form field reaches the right payload key. A test that renders a whole modal to assert the shape of one object belongs in the first category, not the second + +Most of the suite predates this split and is not yet classified, so an unsuffixed `*.test.tsx` is not evidence that a file is really a unit test. Classify what you touch diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.integration.test.tsx similarity index 96% rename from ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.integration.test.tsx index 45da71ed301..c9007e29c3e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.integration.test.tsx @@ -120,10 +120,48 @@ describe("CreateMCPServer", () => { expect(screen.getByText("Add New MCP Server")).toBeInTheDocument(); }); - it("should not render when user is not an admin", () => { + // The modal DOES render for a non-admin; it retitles and routes the submit to the review endpoint. + // The assertion this replaced only checked that the admin title was absent, which passed for the + // wrong reason and left the whole non-admin submission path uncovered. + it("routes a non-admin submission to the review endpoint instead of creating the server", async () => { render(); + expect(screen.getByText("Submit MCP Server for Review")).toBeInTheDocument(); expect(screen.queryByText("Add New MCP Server")).not.toBeInTheDocument(); + + await selectAntOption("Transport Type", "Streamable HTTP"); + await waitFor(() => { + expect(screen.getByPlaceholderText("https://your-mcp-server.com")).toBeInTheDocument(); + }); + await act(async () => { + fireEvent.change(getServerNameInput(), { target: { value: "Submitted_Server" } }); + }); + await act(async () => { + fireEvent.change(screen.getByPlaceholderText("https://your-mcp-server.com"), { + target: { value: "https://example.com/mcp" }, + }); + }); + await selectAntOption("Authentication", "None"); + + vi.mocked(networking.registerMCPServer).mockResolvedValue({ + server_id: "submitted-1", + server_name: "Submitted_Server", + alias: "Submitted_Server", + url: "https://example.com/mcp", + transport: "http", + auth_type: "none", + created_at: "2024-01-01T00:00:00Z", + created_by: "user-1", + updated_at: "2024-01-01T00:00:00Z", + updated_by: "user-1", + }); + + await act(async () => { + fireEvent.click(screen.getByRole("button", { name: "Add MCP Server" })); + }); + + await waitFor(() => expect(networking.registerMCPServer).toHaveBeenCalledTimes(1)); + expect(networking.createMCPServer).not.toHaveBeenCalled(); }); it("should show transport type options", async () => { @@ -1591,44 +1629,8 @@ describe("CreateMCPServer", () => { expect(payload.credentials?.client_secret).toBeUndefined(); }); - it("omits token_validation from payload when token_validation_json is empty", async () => { - vi.mocked(networking.createMCPServer).mockResolvedValue({ - server_id: "new-server-oauth", - server_name: "OAuth_Server", - alias: "OAuth_Server", - url: "https://example.com/mcp", - transport: "http", - auth_type: "oauth2", - created_at: "2024-01-01T00:00:00Z", - created_by: "user-1", - updated_at: "2024-01-01T00:00:00Z", - updated_by: "user-1", - }); - - await setupOAuthInteractive(); - - const nameInput = document.getElementById("server_name") as HTMLInputElement; - await act(async () => { - fireEvent.change(nameInput, { target: { value: "OAuth_Server" } }); - }); - const urlInput = screen.getByPlaceholderText("https://your-mcp-server.com"); - await act(async () => { - fireEvent.change(urlInput, { target: { value: "https://example.com/mcp" } }); - }); - - // Leave token_validation_json empty - const submitButton = screen.getByRole("button", { name: "Add MCP Server" }); - await act(async () => { - fireEvent.click(submitButton); - }); - - await waitFor(() => { - expect(networking.createMCPServer).toHaveBeenCalledTimes(1); - }); - - const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0]; - expect(payload.token_validation).toBeUndefined(); - }); + // Empty/whitespace token_validation is covered in createServerPayload.test.ts; the sibling + // test above still proves the textarea reaches token_validation_json. it("includes credentials.token_endpoint_auth_method in payload when client_secret_basic is selected", async () => { vi.mocked(networking.createMCPServer).mockResolvedValue({ @@ -1670,43 +1672,8 @@ describe("CreateMCPServer", () => { expect(payload.credentials?.token_endpoint_auth_method).toBe("client_secret_basic"); }); - it("omits token_endpoint_auth_method from credentials when left blank", async () => { - vi.mocked(networking.createMCPServer).mockResolvedValue({ - server_id: "new-server-oauth", - server_name: "OAuth_Server", - alias: "OAuth_Server", - url: "https://example.com/mcp", - transport: "http", - auth_type: "oauth2", - created_at: "2024-01-01T00:00:00Z", - created_by: "user-1", - updated_at: "2024-01-01T00:00:00Z", - updated_by: "user-1", - }); - - await setupOAuthInteractive(); - - const nameInput = document.getElementById("server_name") as HTMLInputElement; - await act(async () => { - fireEvent.change(nameInput, { target: { value: "OAuth_Server" } }); - }); - const urlInput = screen.getByPlaceholderText("https://your-mcp-server.com"); - await act(async () => { - fireEvent.change(urlInput, { target: { value: "https://example.com/mcp" } }); - }); - - const submitButton = screen.getByRole("button", { name: "Add MCP Server" }); - await act(async () => { - fireEvent.click(submitButton); - }); - - await waitFor(() => { - expect(networking.createMCPServer).toHaveBeenCalledTimes(1); - }); - - const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0]; - expect(payload.credentials?.token_endpoint_auth_method).toBeUndefined(); - }); + // Blank credential keys are dropped by the shared filter, covered in createServerPayload.test.ts; + // the sibling test above still proves the select reaches credentials.token_endpoint_auth_method. it("persists access + refresh token to the DB on submit for OBO mode", async () => { // "Authorize & Fetch" produced a token before submit. @@ -2052,14 +2019,8 @@ describe("CreateMCPServer oauth2_flow persistence", () => { expect(payload.oauth2_flow).toBe("client_credentials"); }); - it("sends no oauth2_flow for a non-oauth2 create", async () => { - vi.mocked(networking.createMCPServer).mockResolvedValue({ ...createdServer, auth_type: "none" }); - await setupHttpServerForm(); - await selectAntOption("Authentication", "None"); - - const payload = await submitCreate(); - expect(payload.oauth2_flow).toBeUndefined(); - }); + // oauth2_flow branch coverage lives in createServerPayload.test.ts; the two cases above keep + // the dropdown-to-payload wiring they uniquely prove. }); describe("CreateMCPServer dcr_bridge toggle", () => { @@ -2185,18 +2146,9 @@ describe("CreateMCPServer dcr_bridge toggle", () => { expect(payload.dcr_bridge).toBe(false); }); - it.each([ - ["none", "None"], - ["api_key", "API Key"], - ["oauth2", "OAuth"], - ])("forces an explicit dcr_bridge: false for %s", async (authType, optionLabel) => { - vi.mocked(networking.createMCPServer).mockResolvedValue({ ...createdServer, auth_type: authType }); - await setupHttpServerForm(); - await selectAntOption("Authentication", optionLabel); - - const payload = await submitCreate(); - expect(payload.dcr_bridge).toBe(false); - }); + // Forcing dcr_bridge false for every non-client-forwarded auth type is covered in + // createServerPayload.test.ts. The two form-state cases below stay: they prove the Form.Item + // unmounts on a switch away, and that the live value survives a client-forwarded swap. it("forces dcr_bridge: false when the auth type is switched away after toggling", async () => { vi.mocked(networking.createMCPServer).mockResolvedValue({ ...createdServer, auth_type: "none" }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.test.ts new file mode 100644 index 00000000000..e2e7814f0d7 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.test.ts @@ -0,0 +1,126 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { setSecureItem } from "@/utils/secureStorage"; +import { CreateUiSnapshot, readCreateUiSnapshot, writeCreateUiSnapshot } from "./createOAuthUiState"; + +const STORAGE_KEY = "litellm-mcp-oauth-create-state"; + +const fullSnapshot: CreateUiSnapshot = { + modalVisible: true, + formValues: { url: "https://example.com/mcp", auth_type: "oauth2", credentials: { client_id: "app-id" } }, + transportType: "http", + costConfig: { default_cost_per_query: 0.02 }, + allowedTools: ["search"], + hasToolAllowlistInteraction: true, + searchValue: "group-a", + aliasManuallyEdited: true, + logoUrl: "https://cdn/logo.png", + authorizedIdentity: "identity-abc", +}; + +const seedRaw = (value: unknown) => setSecureItem(STORAGE_KEY, JSON.stringify(value)); + +describe("createOAuthUiState", () => { + beforeEach(() => { + window.sessionStorage.clear(); + vi.restoreAllMocks(); + }); + + it("returns null and leaves storage untouched when nothing was persisted", () => { + expect(readCreateUiSnapshot()).toBeNull(); + }); + + it("round-trips a full snapshot through the redirect", () => { + writeCreateUiSnapshot(fullSnapshot); + expect(readCreateUiSnapshot()).toEqual(fullSnapshot); + }); + + it("does not store the snapshot in plaintext", () => { + writeCreateUiSnapshot(fullSnapshot); + // secureStorage base64-encodes; a readable url in the raw value would mean the encoding was lost. + expect(window.sessionStorage.getItem(STORAGE_KEY)).not.toContain("https://example.com/mcp"); + }); + + it("consumes the snapshot so a second mount cannot replay it", () => { + writeCreateUiSnapshot(fullSnapshot); + expect(readCreateUiSnapshot()).not.toBeNull(); + expect(readCreateUiSnapshot()).toBeNull(); + expect(window.sessionStorage.getItem(STORAGE_KEY)).toBeNull(); + }); + + it("strips minted token material so a stale token never rehydrates", () => { + writeCreateUiSnapshot({ + ...fullSnapshot, + formValues: { + url: "https://example.com/mcp", + credentials: { + client_id: "app-id", + client_secret: "app-secret", + access_token: "stale-tok", + refresh_token: "stale-refresh", + expires_in: 3600, + scope: "read", + }, + }, + }); + + const restored = readCreateUiSnapshot(); + expect(restored?.formValues?.credentials).toEqual({ client_id: "app-id", client_secret: "app-secret" }); + expect(JSON.stringify(restored)).not.toContain("stale-tok"); + expect(JSON.stringify(restored)).not.toContain("stale-refresh"); + }); + + it("re-arms invalidation by restoring the authorized identity", () => { + writeCreateUiSnapshot(fullSnapshot); + expect(readCreateUiSnapshot()?.authorizedIdentity).toBe("identity-abc"); + }); + + it("prefers the persisted form transport over the standalone transportType", () => { + seedRaw({ formValues: { transport: "sse" }, transportType: "http" }); + expect(readCreateUiSnapshot()?.transportType).toBe("sse"); + }); + + it("omits falsy scalars so a restore never blanks freshly mounted state", () => { + seedRaw({ searchValue: "", logoUrl: "", transportType: "", modalVisible: false }); + const restored = readCreateUiSnapshot(); + expect(restored).not.toHaveProperty("searchValue"); + expect(restored).not.toHaveProperty("logoUrl"); + expect(restored).not.toHaveProperty("transportType"); + expect(restored).not.toHaveProperty("modalVisible"); + }); + + it("restores an explicitly empty tool allowlist, which is a real admin choice", () => { + seedRaw({ allowedTools: [], hasToolAllowlistInteraction: true }); + const restored = readCreateUiSnapshot(); + expect(restored?.allowedTools).toEqual([]); + expect(restored?.hasToolAllowlistInteraction).toBe(true); + }); + + it.each([ + ["hasToolAllowlistInteraction", false], + ["aliasManuallyEdited", false], + ])("restores %s when it was persisted as false", (key, value) => { + seedRaw({ [key]: value }); + expect(readCreateUiSnapshot()).toHaveProperty(key, value); + }); + + it.each([["hasToolAllowlistInteraction"], ["aliasManuallyEdited"]])( + "ignores a non-boolean %s rather than coercing it", + (key) => { + seedRaw({ [key]: "yes" }); + expect(readCreateUiSnapshot()).not.toHaveProperty(key); + }, + ); + + it("ignores a non-string authorizedIdentity", () => { + seedRaw({ authorizedIdentity: 42 }); + expect(readCreateUiSnapshot()).not.toHaveProperty("authorizedIdentity"); + }); + + it("returns null on a corrupted payload but still clears it", () => { + vi.spyOn(console, "error").mockImplementation(() => {}); + setSecureItem(STORAGE_KEY, "{not json"); + + expect(readCreateUiSnapshot()).toBeNull(); + expect(window.sessionStorage.getItem(STORAGE_KEY)).toBeNull(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createServerPayload.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createServerPayload.test.ts new file mode 100644 index 00000000000..4573069ff07 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createServerPayload.test.ts @@ -0,0 +1,296 @@ +import { describe, expect, it } from "vitest"; +import { + BuildCreatePayloadResult, + CreateServerUiState, + buildCreateServerPayload, + parseStdioConfig, + reduceStaticHeaders, +} from "./createServerPayload"; + +const baseUi: CreateServerUiState = { + transportType: "http", + costConfig: {}, + allowedTools: [], + hasToolAllowlistInteraction: false, + toolNameToDisplayName: {}, + toolNameToDescription: {}, + logoUrl: undefined, + dcrClient: null, +}; + +/** Narrow to the success branch so a regression surfaces as a failed assertion, not a type error. */ +const payloadOf = (result: BuildCreatePayloadResult): Record => { + expect(result.kind).toBe("ok"); + if (result.kind !== "ok") throw new Error("unreachable"); + return result.payload; +}; + +const build = (values: Record, ui: Partial = {}) => + buildCreateServerPayload(values, { ...baseUi, ...ui }); + +describe("reduceStaticHeaders", () => { + it("returns an empty map for a non-array", () => { + expect(reduceStaticHeaders(undefined)).toEqual({}); + expect(reduceStaticHeaders("X-Api-Key: v")).toEqual({}); + }); + + it("trims header and value and drops rows with a blank header", () => { + expect( + reduceStaticHeaders([ + { header: " X-Api-Key ", value: " secret " }, + { header: " ", value: "orphaned" }, + { header: "X-Empty" }, + ]), + ).toEqual({ "X-Api-Key": "secret", "X-Empty": "" }); + }); + + it("keeps the last value when a header repeats", () => { + expect( + reduceStaticHeaders([ + { header: "X-Dup", value: "first" }, + { header: "X-Dup", value: "second" }, + ]), + ).toEqual({ "X-Dup": "second" }); + }); +}); + +describe("parseStdioConfig", () => { + it("reads a direct command/args/env config", () => { + const result = parseStdioConfig('{"command":"npx","args":["-y","srv"],"env":{"TOKEN":"t"}}'); + expect(result).toEqual({ + kind: "ok", + fields: { command: "npx", args: ["-y", "srv"], env: { TOKEN: "t" } }, + }); + }); + + it("unwraps the mcpServers form and derives the server name with underscores", () => { + const result = parseStdioConfig('{"mcpServers":{"my-github-server":{"command":"npx","args":["-y"]}}}'); + expect(result).toEqual({ + kind: "ok", + fields: { command: "npx", args: ["-y"], env: undefined }, + derivedServerName: "my_github_server", + }); + }); + + it("takes the first server when mcpServers holds several", () => { + const result = parseStdioConfig('{"mcpServers":{"first":{"command":"a"},"second":{"command":"b"}}}'); + expect(result).toMatchObject({ kind: "ok", fields: { command: "a" }, derivedServerName: "first" }); + }); + + it("treats an empty mcpServers object as a direct config rather than deriving a name", () => { + const result = parseStdioConfig('{"mcpServers":{},"command":"direct"}'); + expect(result).toEqual({ kind: "ok", fields: { command: "direct", args: undefined, env: undefined } }); + }); + + it.each([["not json{"], ["null"]])("reports %s as invalid", (raw) => { + expect(parseStdioConfig(raw)).toEqual({ kind: "invalid" }); + }); +}); + +describe("buildCreateServerPayload validation", () => { + it("rejects a tool display name containing a space and names the offender", () => { + const result = build({ auth_type: "none" }, { toolNameToDisplayName: { search: "My Tool" } }); + expect(result).toEqual({ kind: "invalid_tool_display_name", displayName: "My Tool" }); + }); + + it("accepts letters, digits, underscores and hyphens in a display name", () => { + const result = build({ auth_type: "none" }, { toolNameToDisplayName: { search: "my-tool_2" } }); + expect(result.kind).toBe("ok"); + }); + + it("rejects unparseable stdio JSON only when the stdio transport is selected", () => { + expect(build({ stdio_config: "{oops" }, { transportType: "stdio" })).toEqual({ kind: "invalid_stdio_json" }); + // The same bad string on an http server is an inert leftover field, not a submit blocker. + expect(build({ stdio_config: "{oops" }, { transportType: "http" }).kind).toBe("ok"); + }); + + it("rejects unparseable token validation JSON", () => { + expect(build({ token_validation_json: "not-valid-json{" })).toEqual({ kind: "invalid_token_validation_json" }); + }); + + it("ignores a whitespace-only token validation body", () => { + const payload = payloadOf(build({ token_validation_json: " " })); + expect(payload).not.toHaveProperty("token_validation"); + }); + + it("includes parsed token validation rules when the JSON is valid", () => { + const payload = payloadOf(build({ token_validation_json: '{"organization":"my-org","team.id":"42"}' })); + expect(payload.token_validation).toEqual({ organization: "my-org", "team.id": "42" }); + }); +}); + +describe("buildCreateServerPayload transport and naming", () => { + it("maps the UI-only openapi transport to http for the backend", () => { + const payload = payloadOf(build({ transport: "openapi", spec_path: "https://api.example.com/openapi.json" })); + expect(payload.transport).toBe("http"); + }); + + it("leaves http and sse transports untouched", () => { + expect(payloadOf(build({ transport: "sse" })).transport).toBe("sse"); + }); + + it("falls back to the stdio JSON's server key when the name field is blank", () => { + const payload = payloadOf( + build( + { transport: "stdio", stdio_config: '{"mcpServers":{"my-server":{"command":"npx"}}}' }, + { transportType: "stdio" }, + ), + ); + expect(payload.server_name).toBe("my_server"); + expect(payload.command).toBe("npx"); + }); + + it("keeps an explicit server name over the stdio JSON's key", () => { + const payload = payloadOf( + build( + { server_name: "Chosen", transport: "stdio", stdio_config: '{"mcpServers":{"my-server":{"command":"npx"}}}' }, + { transportType: "stdio" }, + ), + ); + expect(payload.server_name).toBe("Chosen"); + }); + + it("falls back to the url for mcp_info.server_name when no name is given", () => { + const payload = payloadOf(build({ url: "https://example.com/mcp" })); + expect((payload.mcp_info as Record).server_name).toBe("https://example.com/mcp"); + }); +}); + +describe("buildCreateServerPayload credentials", () => { + it("drops empty, null and undefined credential entries", () => { + const payload = payloadOf( + build({ auth_type: "api_key", credentials: { auth_value: "secret", client_id: "", client_secret: null } }), + ); + expect(payload.credentials).toEqual({ auth_value: "secret" }); + }); + + it("filters blank scopes and omits the key when none survive", () => { + expect( + payloadOf(build({ auth_type: "oauth2", credentials: { client_id: "c", scopes: ["read", "", null] } })) + .credentials, + ).toEqual({ client_id: "c", scopes: ["read"] }); + expect(payloadOf(build({ auth_type: "oauth2", credentials: { client_id: "c", scopes: [] } })).credentials).toEqual({ + client_id: "c", + }); + }); + + it("omits credentials entirely for an auth type that needs none", () => { + const payload = payloadOf(build({ auth_type: "none", credentials: { auth_value: "stale" } })); + expect(payload).not.toHaveProperty("credentials"); + }); + + it.each([["true_passthrough"], ["oauth_delegate"]])( + "persists only the declared app for %s, never minted token material", + (authType) => { + const payload = payloadOf( + build({ + auth_type: authType, + credentials: { + client_id: "org-app", + client_secret: "org-secret", + access_token: "upstream-tok", + refresh_token: "refresh-tok", + expires_in: 3600, + scope: "read", + }, + }), + ); + expect(payload.credentials).toEqual({ client_id: "org-app", client_secret: "org-secret" }); + expect(JSON.stringify(payload)).not.toContain("upstream-tok"); + expect(JSON.stringify(payload)).not.toContain("refresh-tok"); + }, + ); + + it("merges the DCR-minted client into an oauth2 payload", () => { + const payload = payloadOf( + build( + { auth_type: "oauth2", credentials: { access_token: "tok" } }, + { dcrClient: { client_id: "dcr-id", client_secret: "dcr-secret" } }, + ), + ); + expect(payload.credentials).toMatchObject({ + client_id: "dcr-id", + client_secret: "dcr-secret", + access_token: "tok", + }); + }); + + it("never leaks the DCR-minted client onto a non-oauth2 server", () => { + const payload = payloadOf( + build({ auth_type: "true_passthrough" }, { dcrClient: { client_id: "dcr-id", client_secret: "dcr-secret" } }), + ); + expect(JSON.stringify(payload)).not.toContain("dcr-id"); + }); +}); + +describe("buildCreateServerPayload flags", () => { + it.each([["true_passthrough"], ["oauth_delegate"]])("defaults dcr_bridge on for %s", (authType) => { + expect(payloadOf(build({ auth_type: authType })).dcr_bridge).toBe(true); + }); + + it.each([["true_passthrough"], ["oauth_delegate"]])("honours an explicit dcr_bridge false for %s", (authType) => { + expect(payloadOf(build({ auth_type: authType, dcr_bridge: false })).dcr_bridge).toBe(false); + }); + + it.each([["none"], ["api_key"], ["oauth2"]])( + "forces dcr_bridge off for %s even when the form still holds true", + (authType) => { + expect(payloadOf(build({ auth_type: authType, dcr_bridge: true })).dcr_bridge).toBe(false); + }, + ); + + it("stamps the interactive oauth2 flow by default", () => { + expect(payloadOf(build({ auth_type: "oauth2" })).oauth2_flow).toBe("authorization_code"); + }); + + it("stamps client_credentials for an M2M oauth2 server", () => { + expect(payloadOf(build({ auth_type: "oauth2", oauth_flow_type: "m2m" })).oauth2_flow).toBe("client_credentials"); + }); + + it("sends no oauth2_flow for a non-oauth2 server", () => { + expect(payloadOf(build({ auth_type: "api_key", oauth_flow_type: "m2m" }))).not.toHaveProperty("oauth2_flow"); + }); + + it.each([["allow_all_keys"], ["available_on_public_internet"], ["delegate_auth_to_upstream"], ["oauth_passthrough"]])( + "coerces %s to a boolean", + (key) => { + expect(payloadOf(build({ auth_type: "none" }))[key]).toBe(false); + expect(payloadOf(build({ auth_type: "none", [key]: true }))[key]).toBe(true); + }, + ); +}); + +describe("buildCreateServerPayload tool allowlist", () => { + it("marks the allowlist enforced once the admin has touched it, even with nothing selected", () => { + const payload = payloadOf(build({ auth_type: "none" }, { hasToolAllowlistInteraction: true })); + expect((payload.mcp_info as Record).tool_allowlist_enforced).toBe(true); + expect(payload.allowed_tools).toEqual([]); + }); + + it("marks the allowlist enforced when tools are selected without an explicit interaction", () => { + const payload = payloadOf(build({ auth_type: "none" }, { allowedTools: ["search"] })); + expect((payload.mcp_info as Record).tool_allowlist_enforced).toBe(true); + expect(payload.allowed_tools).toEqual(["search"]); + }); + + it("leaves the allowlist unenforced when untouched and empty", () => { + const payload = payloadOf(build({ auth_type: "none" })); + expect((payload.mcp_info as Record).tool_allowlist_enforced).toBe(false); + }); +}); + +describe("buildCreateServerPayload mcp_info", () => { + it("sends a null cost map when nothing is configured and the map when it is", () => { + expect( + (payloadOf(build({ auth_type: "none" })).mcp_info as Record).mcp_server_cost_info, + ).toBeNull(); + const priced = payloadOf(build({ auth_type: "none" }, { costConfig: { default_cost_per_query: 0.01 } })); + expect((priced.mcp_info as Record).mcp_server_cost_info).toEqual({ default_cost_per_query: 0.01 }); + }); + + it("carries the selected logo and drops the raw stdio_config field", () => { + const payload = payloadOf(build({ auth_type: "none", stdio_config: "{}" }, { logoUrl: "https://cdn/logo.png" })); + expect((payload.mcp_info as Record).logo_url).toBe("https://cdn/logo.png"); + expect(payload.stdio_config).toBeUndefined(); + }); +}); From 9c2c79f976bf78a7572c198b2840c62ca7620d3b Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 3 Aug 2026 16:25:35 -0700 Subject: [PATCH 101/123] fix(ui): render Responses API request and response in the logs drawer The Pretty view only parsed the Chat Completions shape (messages / choices[0].message), so any spend log storing the Responses API shape (input / output) rendered an empty Input card and the literal text "No response data available" even though the row held the full request and response. This also hit plain /v1/chat/completions callers, because litellm may route those over the Responses bridge and then store the upstream Responses-shaped body. Parsing now branches on a tagged union covering both shapes, which also replaces the any-typed key sniffing and the role guessing it relied on. --- ui/litellm-dashboard/eslint-suppressions.json | 5 - .../PrettyMessagesView.test.tsx | 120 +++++++++ .../LogDetailsDrawer/prettyMessagesTypes.ts | 16 +- .../LogDetailsDrawer/prettyMessagesUtils.ts | 236 ++++++++++++------ 4 files changed, 299 insertions(+), 78 deletions(-) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 107f66b8f1a..2602b8fa7ad 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -4142,11 +4142,6 @@ "count": 1 } }, - "src/components/view_logs/LogDetailsDrawer/prettyMessagesUtils.ts": { - "no-nested-ternary": { - "count": 1 - } - }, "src/components/view_logs/LogDetailsDrawer/useKeyboardNavigation.ts": { "react-hooks/immutability": { "count": 2 diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.test.tsx index e7295ed7a72..104c421acbf 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.test.tsx @@ -76,6 +76,126 @@ describe("PrettyMessagesView", () => { expect(modelElements.length).toBeGreaterThanOrEqual(1); }); + it("renders a Responses API log, whose body uses input/output instead of messages/choices", () => { + const request = { + model: "gpt-5.6", + input: [{ role: "user", content: "Reply with exactly: hello from responses api" }], + }; + const response = { + output: [ + { + id: "msg_070989277645d4ae", + role: "assistant", + type: "message", + status: "completed", + content: [{ text: "hello from responses api", type: "output_text", annotations: [] }], + }, + ], + }; + + render(); + expect(screen.getByText("Reply with exactly: hello from responses api")).toBeInTheDocument(); + expect(screen.getByText("hello from responses api")).toBeInTheDocument(); + expect(screen.queryByText("No response data available")).not.toBeInTheDocument(); + }); + + it("renders a Responses API tool call, whose output item is a function_call", () => { + const request = { + model: "gpt-5.6", + input: [{ role: "user", content: "What is the weather in San Francisco? Use the tool." }], + }; + const response = { + output: [ + { + id: "fc_08edf6c2312f1485", + name: "get_weather", + type: "function_call", + status: "completed", + call_id: "call_AtO0J9eNy5jgECXzBicMJM8W", + arguments: '{"city":"San Francisco"}', + }, + ], + }; + + render(); + expect(screen.getByText("What is the weather in San Francisco? Use the tool.")).toBeInTheDocument(); + expect(screen.getByText("get_weather")).toBeInTheDocument(); + expect(screen.queryByText("No response data available")).not.toBeInTheDocument(); + }); + + it("renders instructions as the system turn and a bare string input", () => { + const request = { model: "gpt-5.6", instructions: "You are terse.", input: "Say A" }; + const response = { + output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text: "A" }] }], + }; + + render(); + expect(screen.getByText("You are terse.")).toBeInTheDocument(); + expect(screen.getByText("Say A")).toBeInTheDocument(); + expect(screen.getByText("A")).toBeInTheDocument(); + }); + + it("skips reasoning output items rather than rendering them as empty turns", () => { + const request = { input: [{ role: "user", content: "Think then answer" }] }; + const response = { + output: [ + { type: "reasoning", id: "rs_1", summary: [] }, + { type: "message", role: "assistant", content: [{ type: "output_text", text: "answered" }] }, + ], + }; + + render(); + expect(screen.getByText("answered")).toBeInTheDocument(); + expect(screen.queryByText("No response data available")).not.toBeInTheDocument(); + }); + + it("renders a Responses API follow-up turn carrying a prior function_call and its output", () => { + const request = { + input: [ + { role: "user", content: "What is the weather in San Francisco? Use the tool." }, + { + type: "function_call", + name: "get_weather", + call_id: "call_AtO0J9eNy5jgECXzBicMJM8W", + arguments: '{"city":"San Francisco"}', + }, + { type: "function_call_output", call_id: "call_AtO0J9eNy5jgECXzBicMJM8W", output: '{"temp":18}' }, + ], + }; + const response = { + output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text: "It is 18 degrees." }] }], + }; + + render(); + expect(screen.getByText("It is 18 degrees.")).toBeInTheDocument(); + expect(screen.getByText('{"temp":18}')).toBeInTheDocument(); + expect(screen.getByText("TOOL")).toBeInTheDocument(); + }); + + it("maps the developer and legacy function roles onto the roles the drawer renders", () => { + const request = { + messages: [ + { role: "developer", content: "Stay terse." }, + { role: "user", content: "Weather?" }, + { role: "function", name: "get_weather", content: '{"temp":18}' }, + ], + }; + const response = { choices: [{ message: { role: "assistant", content: "18 degrees." } }] }; + + render(); + expect(screen.getByText("Stay terse.")).toBeInTheDocument(); + expect(screen.getByText("TOOL")).toBeInTheDocument(); + expect(screen.queryByText("FUNCTION")).not.toBeInTheDocument(); + }); + + it("still reports missing output when a Responses API log has an empty output array", () => { + const request = { input: [{ role: "user", content: "Hello" }] }; + + render(); + expect(screen.getByText("Hello")).toBeInTheDocument(); + expect(screen.getByText("No response data available")).toBeInTheDocument(); + }); + it("should render standard view when response has results but no realtime events", () => { const request = { messages: [{ role: "user", content: "Test" }], diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesTypes.ts b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesTypes.ts index da5c492e60f..463ba65d6ff 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesTypes.ts +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesTypes.ts @@ -2,17 +2,29 @@ * Type definitions for pretty messages view */ +export type MessageRole = "system" | "user" | "assistant" | "tool"; + export interface ParsedMessage { - role: "system" | "user" | "assistant" | "tool"; + role: MessageRole; content: string; toolCalls?: ToolCall[]; toolCallId?: string; } +export type RequestPayload = + | { kind: "chat"; messages: readonly unknown[] } + | { kind: "responses"; instructions: string; input: string | readonly unknown[] } + | { kind: "unknown" }; + +export type ResponsePayload = + | { kind: "chat"; choices: readonly unknown[] } + | { kind: "responses"; output: readonly unknown[] } + | { kind: "unknown" }; + export interface ToolCall { id: string; name: string; - arguments: Record; + arguments: Record; } export interface ParsedMessages { diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesUtils.ts b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesUtils.ts index 09b8f551c1d..1f73da1d30e 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesUtils.ts +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesUtils.ts @@ -2,7 +2,15 @@ * Utility functions for parsing and formatting messages for pretty view */ -import { ParsedMessage, ParsedMessages, RoleStyle } from "./prettyMessagesTypes"; +import { + MessageRole, + ParsedMessage, + ParsedMessages, + RequestPayload, + ResponsePayload, + RoleStyle, + ToolCall, +} from "./prettyMessagesTypes"; /** * Role color styles for message cards - minimal, professional design @@ -35,102 +43,188 @@ export const ROLE_STYLES: Record = { }, }; +type UnknownRecord = Record; + +const isRecord = (value: unknown): value is UnknownRecord => + typeof value === "object" && value !== null && !Array.isArray(value); + +const asString = (value: unknown): string => (typeof value === "string" ? value : ""); + +const ROLES: readonly MessageRole[] = ["system", "user", "assistant", "tool"]; + +const toRole = (value: unknown, fallback: MessageRole): MessageRole => { + if (value === "developer") return "system"; + if (value === "function") return "tool"; + return ROLES.includes(value as MessageRole) ? (value as MessageRole) : fallback; +}; + +const classifyRequest = (request: unknown): RequestPayload => { + if (Array.isArray(request)) return { kind: "chat", messages: request }; + if (!isRecord(request)) return { kind: "unknown" }; + if (Array.isArray(request.messages)) return { kind: "chat", messages: request.messages }; + const { input } = request; + if (typeof input === "string" || Array.isArray(input)) { + return { kind: "responses", instructions: asString(request.instructions), input }; + } + return { kind: "unknown" }; +}; + +const classifyResponse = (response: unknown): ResponsePayload => { + if (!isRecord(response)) return { kind: "unknown" }; + if (Array.isArray(response.choices)) return { kind: "chat", choices: response.choices }; + if (Array.isArray(response.output)) return { kind: "responses", output: response.output }; + return { kind: "unknown" }; +}; + /** * Parse request messages and response message from log data */ -export const parseMessages = (request: any, response: any): ParsedMessages => { - // Parse request messages. `request` is either the raw request body - // ({ messages: [...] }) or, when prompts come from cold storage, the bare - // messages array itself. - const requestMessages: ParsedMessage[] = []; +export const parseMessages = (request: unknown, response: unknown): ParsedMessages => ({ + requestMessages: parseRequestMessages(classifyRequest(request)), + responseMessage: parseResponseMessage(classifyResponse(response)), +}); - const requestMessageList = Array.isArray(request) - ? request - : Array.isArray(request?.messages) - ? request.messages - : []; - - requestMessageList.forEach((msg: any) => { - requestMessages.push({ - role: msg.role || "user", - content: parseMessageContent(msg.content), - toolCallId: msg.tool_call_id, - }); - }); - - // Parse response message - let responseMessage: ParsedMessage | null = null; - const responseMsg = response?.choices?.[0]?.message; - - if (responseMsg) { - responseMessage = { - role: responseMsg.role || "assistant", - content: responseMsg.content || "", - toolCalls: parseToolCalls(responseMsg.tool_calls), - }; +const parseRequestMessages = (payload: RequestPayload): ParsedMessage[] => { + switch (payload.kind) { + case "chat": + return payload.messages.map(parseChatMessage); + case "responses": { + const instructions: ParsedMessage[] = payload.instructions + ? [{ role: "system", content: payload.instructions }] + : []; + const input: ParsedMessage[] = + typeof payload.input === "string" + ? [{ role: "user", content: payload.input }] + : payload.input.flatMap(parseResponsesInputItem); + return [...instructions, ...input]; + } + case "unknown": + return []; } - - return { requestMessages, responseMessage }; }; +const parseResponseMessage = (payload: ResponsePayload): ParsedMessage | null => { + switch (payload.kind) { + case "chat": { + const choice = payload.choices[0]; + const message = isRecord(choice) ? choice.message : undefined; + if (!isRecord(message)) return null; + return { + role: toRole(message.role, "assistant"), + content: parseMessageContent(message.content), + toolCalls: parseChatToolCalls(message.tool_calls), + }; + } + case "responses": { + const content = payload.output + .filter((item): item is UnknownRecord => isRecord(item) && item.type === "message") + .map((item) => parseMessageContent(item.content)) + .filter((text) => text.length > 0) + .join("\n"); + const toolCalls = payload.output.filter(isResponsesFunctionCall).map(parseResponsesFunctionCall); + if (content.length === 0 && toolCalls.length === 0) return null; + return { role: "assistant", content, toolCalls: toolCalls.length > 0 ? toolCalls : undefined }; + } + case "unknown": + return null; + } +}; + +const parseChatMessage = (message: unknown): ParsedMessage => { + if (!isRecord(message)) return { role: "user", content: parseMessageContent(message) }; + return { + role: toRole(message.role, "user"), + content: parseMessageContent(message.content), + toolCalls: parseChatToolCalls(message.tool_calls), + toolCallId: typeof message.tool_call_id === "string" ? message.tool_call_id : undefined, + }; +}; + +const parseResponsesInputItem = (item: unknown): ParsedMessage[] => { + if (typeof item === "string") return [{ role: "user", content: item }]; + if (!isRecord(item)) return []; + if (item.type === "function_call") { + return [{ role: "assistant", content: "", toolCalls: [parseResponsesFunctionCall(item)] }]; + } + if (item.type === "function_call_output") { + return [{ role: "tool", content: parseMessageContent(item.output), toolCallId: asString(item.call_id) }]; + } + if (item.type === "reasoning") return []; + if ("role" in item || "content" in item) { + return [{ role: toRole(item.role, "user"), content: parseMessageContent(item.content) }]; + } + return []; +}; + +const isResponsesFunctionCall = (item: unknown): item is UnknownRecord => + isRecord(item) && item.type === "function_call"; + +const parseResponsesFunctionCall = (item: UnknownRecord): ToolCall => ({ + id: asString(item.call_id) || asString(item.id), + name: asString(item.name) || "unknown", + arguments: parseToolArguments(item.arguments), +}); + /** * Parse message content - handle strings and content arrays (for vision, etc.) */ -const parseMessageContent = (content: any): string => { - if (typeof content === "string") { - return content; - } - - if (Array.isArray(content)) { - // Handle content arrays (vision API format) - return content - .map((item) => { - if (typeof item === "string") return item; - if (item.type === "text") return item.text; - if (item.type === "image_url") return "[Image]"; - return JSON.stringify(item); - }) - .join("\n"); - } - - // Fallback to JSON string for complex content +const parseMessageContent = (content: unknown): string => { + if (typeof content === "string") return content; + if (content === null || content === undefined) return ""; + if (Array.isArray(content)) return content.map(parseContentPart).join("\n"); return JSON.stringify(content); }; +const parseContentPart = (part: unknown): string => { + if (typeof part === "string") return part; + if (!isRecord(part)) return JSON.stringify(part); + switch (part.type) { + case "text": + case "input_text": + case "output_text": + return asString(part.text); + case "refusal": + return asString(part.refusal); + case "image_url": + case "input_image": + return "[Image]"; + case "input_file": + return "[File]"; + case "input_audio": + return "[Audio]"; + default: + return JSON.stringify(part); + } +}; + /** * Parse tool calls from response message */ -const parseToolCalls = ( - toolCalls: any[], -): - | Array<{ - id: string; - name: string; - arguments: Record; - }> - | undefined => { - if (!toolCalls || !Array.isArray(toolCalls)) return undefined; - - return toolCalls.map((tc) => ({ - id: tc.id || "", - name: tc.function?.name || "unknown", - arguments: parseToolArguments(tc.function?.arguments), - })); +const parseChatToolCalls = (toolCalls: unknown): ToolCall[] | undefined => { + if (!Array.isArray(toolCalls)) return undefined; + return toolCalls.map((toolCall) => { + const call = isRecord(toolCall) ? toolCall : {}; + const fn = isRecord(call.function) ? call.function : {}; + return { + id: asString(call.id), + name: asString(fn.name) || "unknown", + arguments: parseToolArguments(fn.arguments), + }; + }); }; /** * Parse tool arguments - handle both string and object formats */ -const parseToolArguments = (args: any): Record => { +const parseToolArguments = (args: unknown): Record => { if (!args) return {}; - if (typeof args === "string") { try { - return JSON.parse(args); + const parsed: unknown = JSON.parse(args); + return isRecord(parsed) ? parsed : { raw: args }; } catch { return { raw: args }; } } - - return args; + return isRecord(args) ? args : {}; }; From bd8b377dd7f672cad6354c5d26a2c159a16d726c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 3 Aug 2026 23:30:31 +0000 Subject: [PATCH 102/123] chore(typing): clear basedpyright Any errors in budget reset, access groups, and cache settings Replace Any seams in three proxy modules with real types so the values keep their shape through the call graph: - reset_budget_job: Protocols for the Prisma spend-linked tables, the reset batcher, and each cascade row shape, with the per-table counter/cache key lambdas promoted to typed module functions so the row type is inferred - access_group_endpoints: Protocols for the access group record, the team and key tables, and the transaction handle; record to response conversion now goes through model_validate on the record dict - cache_settings_endpoints: the opaque cache settings blobs are Mapping[str, object] / dict[str, object] instead of Any, keeping Any only on the two returns that feed the dynamic litellm.Cache kwargs bag Whole-tree basedpyright: reportAny 19435 -> 19306, reportExplicitAny 6518 -> 6487, total errors 148372 -> 148117, with no rule above its baseline and no untouched file changed. No behavior changes. --- basedpyright-code-budget.json | 16 +- .../proxy/common_utils/reset_budget_job.py | 143 ++++++++++++++---- .../access_group_endpoints.py | 119 +++++++++++---- .../cache_settings_endpoints.py | 40 ++--- ruff-strict-budget.json | 4 +- type-discipline-budget.json | 4 +- 6 files changed, 240 insertions(+), 86 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index f6dd90077b1..acdf97cb386 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 29813 + "limit": 29684 }, "reportArgumentType": { "limit": 2645 @@ -24,7 +24,7 @@ "limit": 42 }, "reportExplicitAny": { - "limit": 9473 + "limit": 9442 }, "reportFunctionMemberAccess": { "limit": 11 @@ -54,10 +54,10 @@ "limit": 0 }, "reportMissingParameterType": { - "limit": 5855 + "limit": 5848 }, "reportMissingTypeArgument": { - "limit": 15852 + "limit": 15850 }, "reportMissingTypeStubs": { "limit": 41 @@ -99,19 +99,19 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 45324 + "limit": 45297 }, "reportUnknownLambdaType": { "limit": 113 }, "reportUnknownMemberType": { - "limit": 40452 + "limit": 40411 }, "reportUnknownParameterType": { - "limit": 20309 + "limit": 20301 }, "reportUnknownVariableType": { - "limit": 31978 + "limit": 31968 }, "reportUnnecessaryCast": { "limit": 177 diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 3087e356f99..6ec441a0e06 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -1,12 +1,13 @@ import asyncio import json import time -from collections.abc import Callable +from collections.abc import Callable, Mapping, Sequence from datetime import datetime, timezone -from typing import Any, Literal +from typing import Literal, Protocol, TypeVar import litellm from litellm._logging import verbose_proxy_logger +from litellm.caching.dual_cache import DualCache from litellm.constants import GLOBAL_PROXY_SPEND_CACHE_KEY, LITELLM_PROXY_BUDGET_NAME from litellm.proxy._types import ( LiteLLM_BudgetTableFull, @@ -33,6 +34,98 @@ from litellm.repositories.verification_token_repository import ( ) from litellm.types.services import ServiceTypes +_RowT = TypeVar("_RowT") +_RowT_co = TypeVar("_RowT_co", covariant=True) + + +class _PrismaRecord(Protocol): + def dict(self) -> Mapping[str, object]: ... + + +class _BatchTable(Protocol): + def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> None: ... + + +class _ResetBatcher(Protocol): + @property + def litellm_verificationtoken(self) -> _BatchTable: ... + + @property + def litellm_usertable(self) -> _BatchTable: ... + + @property + def litellm_teamtable(self) -> _BatchTable: ... + + async def commit(self) -> None: ... + + +class _EndUserTable(Protocol): + async def find_many(self, where: Mapping[str, object]) -> Sequence[_PrismaRecord]: ... + + +class _SpendLinkedTable(Protocol[_RowT_co]): + async def find_many(self, where: Mapping[str, object]) -> Sequence[_RowT_co]: ... + + async def update_many(self, where: Mapping[str, object], data: Mapping[str, object]) -> int: ... + + +class _TeamMembershipRow(Protocol): + @property + def user_id(self) -> str: ... + + @property + def team_id(self) -> str: ... + + +class _KeyRow(Protocol): + @property + def token(self) -> str: ... + + +class _OrgRow(Protocol): + @property + def organization_id(self) -> str: ... + + +class _TagRow(Protocol): + @property + def tag_name(self) -> str: ... + + +def _team_membership_counter_key(row: _TeamMembershipRow) -> str: + return f"spend:team_member:{row.user_id}:{row.team_id}" + + +def _team_membership_cache_key(row: _TeamMembershipRow) -> str: + return f"{row.team_id}_{row.user_id}" + + +def _key_counter_key(row: _KeyRow) -> str: + return f"spend:key:{row.token}" + + +def _key_cache_key(row: _KeyRow) -> str: + return row.token + + +def _org_counter_key(row: _OrgRow) -> str: + return f"spend:org:{row.organization_id}" + + +def _org_cache_keys(row: _OrgRow) -> Sequence[str]: + return [ + f"org_id:{row.organization_id}", + f"org_id:{row.organization_id}:with_budget", + ] + + +def _tag_counter_key(row: _TagRow) -> str: + return f"spend:tag:{row.tag_name}" + + +def _tag_cache_key(row: _TagRow) -> str: + return f"tag:{row.tag_name}" + class ResetBudgetJob: """ @@ -134,11 +227,11 @@ class ResetBudgetJob: async def _cascade_reset_spend_for_budget_link( self, budgets_to_reset: list[LiteLLM_BudgetTableFull], - table: Any, - counter_key_fn: Callable[[Any], str], + table: "_SpendLinkedTable[_RowT]", + counter_key_fn: Callable[[_RowT], str], log_subject: str, - extra_where: dict | None = None, - cache_key_fn: Callable[[Any], str | list[str]] | None = None, + extra_where: dict[str, object] | None = None, + cache_key_fn: Callable[[_RowT], str | Sequence[str]] | None = None, ): """ Generic cascade: zero spend on rows whose budget_id is in the reset set. @@ -151,14 +244,14 @@ class ResetBudgetJob: if not budget_ids: return - where: dict = {"budget_id": {"in": budget_ids}} + where: dict[str, object] = {"budget_id": {"in": budget_ids}} if extra_where: where.update(extra_where) try: - rows = await table.find_many(where=where) + rows: Sequence[_RowT] = await table.find_many(where=where) except Exception as e: - rows = [] + rows = () verbose_proxy_logger.warning("Failed to fetch %s for counter invalidation: %s", log_subject, e) update_result = await table.update_many(where=where, data={"spend": 0}) @@ -181,9 +274,9 @@ class ResetBudgetJob: return await self._cascade_reset_spend_for_budget_link( budgets_to_reset=budgets_to_reset, table=TeamMembershipRepository(self.prisma_client).table, - counter_key_fn=lambda m: f"spend:team_member:{m.user_id}:{m.team_id}", + counter_key_fn=_team_membership_counter_key, log_subject="team memberships", - cache_key_fn=lambda m: f"{m.team_id}_{m.user_id}", + cache_key_fn=_team_membership_cache_key, ) async def reset_budget_for_keys_linked_to_budgets(self, budgets_to_reset: list[LiteLLM_BudgetTableFull]): @@ -196,10 +289,10 @@ class ResetBudgetJob: return await self._cascade_reset_spend_for_budget_link( budgets_to_reset=budgets_to_reset, table=VerificationTokenRepository(self.prisma_client).table, - counter_key_fn=lambda k: f"spend:key:{k.token}", + counter_key_fn=_key_counter_key, log_subject="keys", extra_where={"budget_duration": None, "spend": {"gt": 0}}, - cache_key_fn=lambda k: k.token, + cache_key_fn=_key_cache_key, ) async def reset_budget_for_orgs_linked_to_budgets(self, budgets_to_reset: list[LiteLLM_BudgetTableFull]): @@ -209,13 +302,10 @@ class ResetBudgetJob: return await self._cascade_reset_spend_for_budget_link( budgets_to_reset=budgets_to_reset, table=OrganizationRepository(self.prisma_client).table, - counter_key_fn=lambda o: f"spend:org:{o.organization_id}", + counter_key_fn=_org_counter_key, log_subject="orgs", extra_where={"spend": {"gt": 0}}, - cache_key_fn=lambda o: [ - f"org_id:{o.organization_id}", - f"org_id:{o.organization_id}:with_budget", - ], + cache_key_fn=_org_cache_keys, ) async def reset_budget_for_tags_linked_to_budgets(self, budgets_to_reset: list[LiteLLM_BudgetTableFull]): @@ -233,10 +323,10 @@ class ResetBudgetJob: return await self._cascade_reset_spend_for_budget_link( budgets_to_reset=budgets_to_reset, table=TagRepository(self.prisma_client).table, - counter_key_fn=lambda t: f"spend:tag:{t.tag_name}", + counter_key_fn=_tag_counter_key, log_subject="tags", extra_where={"spend": {"gt": 0}}, - cache_key_fn=lambda t: f"tag:{t.tag_name}", + cache_key_fn=_tag_cache_key, ) async def reset_budget_for_litellm_budget_table(self): @@ -376,13 +466,14 @@ class ResetBudgetJob: rely on the default budget (litellm.max_end_user_budget_id) applied in-memory during auth checks. """ - rows = await EndUserRepository(self.prisma_client).table.find_many( + table: _EndUserTable = EndUserRepository(self.prisma_client).table + rows = await table.find_many( where={ "budget_id": None, "spend": {"gt": 0}, }, ) - return [LiteLLM_EndUserTable(**row.dict()) for row in rows] + return [LiteLLM_EndUserTable.model_validate(row.dict()) for row in rows] async def _write_key_reset_updates(self, updated_keys: list[LiteLLM_VerificationToken]) -> None: """ @@ -395,7 +486,7 @@ class ResetBudgetJob: aborts the entire batch — silently leaving spend over the cap and budget_reset_at unchanged forever. """ - batcher = self.prisma_client.db.batch_() + batcher: _ResetBatcher = self.prisma_client.db.batch_() for k in updated_keys: token = getattr(k, "token", None) if token is None: @@ -414,7 +505,7 @@ class ResetBudgetJob: that trips Prisma's DataError on rows carrying unrecognised fields (see #27730). """ - batcher = self.prisma_client.db.batch_() + batcher: _ResetBatcher = self.prisma_client.db.batch_() for u in updated_users: user_id = getattr(u, "user_id", None) if user_id is None: @@ -433,7 +524,7 @@ class ResetBudgetJob: that trips Prisma's DataError on rows carrying unrecognised fields (see #27730). """ - batcher = self.prisma_client.db.batch_() + batcher: _ResetBatcher = self.prisma_client.db.batch_() for t in updated_teams: team_id = getattr(t, "team_id", None) if team_id is None: @@ -688,7 +779,7 @@ class ResetBudgetJob: async def _reset_expired_window( window: dict, counter_key: str, - spend_counter_cache: Any, + spend_counter_cache: DualCache, now: datetime, reset_settings: BudgetResetSettings, ) -> bool: diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index 13e45b17090..4c870c3fb1d 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -1,3 +1,6 @@ +from collections.abc import Mapping, Sequence +from typing import Protocol + from fastapi import APIRouter, Depends, HTTPException, status from litellm._logging import verbose_proxy_logger @@ -29,6 +32,74 @@ router = APIRouter( ) +class _AccessGroupRecord(Protocol): + @property + def access_group_id(self) -> str: ... + + @property + def assigned_team_ids(self) -> Sequence[str] | None: ... + + @property + def assigned_key_ids(self) -> Sequence[str] | None: ... + + def dict(self) -> Mapping[str, object]: ... + + +class _TeamRecord(Protocol): + @property + def team_id(self) -> str: ... + + @property + def access_group_ids(self) -> Sequence[str] | None: ... + + +class _KeyRecord(Protocol): + @property + def token(self) -> str: ... + + @property + def access_group_ids(self) -> Sequence[str] | None: ... + + +class _AccessGroupTable(Protocol): + async def find_unique(self, where: Mapping[str, object]) -> _AccessGroupRecord | None: ... + + async def find_many(self, order: Mapping[str, object]) -> Sequence[_AccessGroupRecord]: ... + + async def create(self, data: Mapping[str, object]) -> _AccessGroupRecord: ... + + async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> _AccessGroupRecord: ... + + async def delete(self, where: Mapping[str, object]) -> object: ... + + +class _TeamTable(Protocol): + async def find_unique(self, where: Mapping[str, object]) -> _TeamRecord | None: ... + + async def find_many(self, where: Mapping[str, object]) -> Sequence[_TeamRecord]: ... + + async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> object: ... + + +class _KeyTable(Protocol): + async def find_unique(self, where: Mapping[str, object]) -> _KeyRecord | None: ... + + async def find_many(self, where: Mapping[str, object]) -> Sequence[_KeyRecord]: ... + + async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> object: ... + + +class _AccessGroupTx(Protocol): + @property + def litellm_accessgrouptable(self) -> _AccessGroupTable: ... + + @property + def litellm_teamtable(self) -> _TeamTable: ... + + @property + def litellm_verificationtoken(self) -> _KeyTable: ... + + def _require_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None: if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: raise HTTPException( @@ -48,29 +119,16 @@ def _require_admin_view(user_api_key_dict: UserAPIKeyAuth) -> None: ) -def _record_to_response(record) -> AccessGroupResponse: - return AccessGroupResponse( - access_group_id=record.access_group_id, - access_group_name=record.access_group_name, - description=record.description, - access_model_names=record.access_model_names, - access_mcp_server_ids=record.access_mcp_server_ids, - access_agent_ids=record.access_agent_ids, - assigned_team_ids=record.assigned_team_ids, - assigned_key_ids=record.assigned_key_ids, - created_at=record.created_at, - created_by=record.created_by, - updated_at=record.updated_at, - updated_by=record.updated_by, - ) +def _record_to_response(record: _AccessGroupRecord) -> AccessGroupResponse: + return AccessGroupResponse.model_validate(record.dict()) -def _record_to_access_group_table(record) -> LiteLLM_AccessGroupTable: +def _record_to_access_group_table(record: _AccessGroupRecord) -> LiteLLM_AccessGroupTable: """Convert a Prisma record to a LiteLLM_AccessGroupTable pydantic object for caching.""" - return LiteLLM_AccessGroupTable(**record.dict()) + return LiteLLM_AccessGroupTable.model_validate(record.dict()) -async def _cache_access_group_record(record) -> None: +async def _cache_access_group_record(record: _AccessGroupRecord) -> None: """ Cache an access group Prisma record in the user_api_key_cache. @@ -109,7 +167,7 @@ async def _invalidate_cache_access_group(access_group_id: str) -> None: # --------------------------------------------------------------------------- -async def _sync_add_access_group_to_teams(tx, team_ids: list[str], access_group_id: str) -> None: +async def _sync_add_access_group_to_teams(tx: _AccessGroupTx, team_ids: list[str], access_group_id: str) -> None: """Add access_group_id to each team's access_group_ids (idempotent).""" for team_id in team_ids: team = await tx.litellm_teamtable.find_unique(where={"team_id": team_id}) @@ -120,18 +178,18 @@ async def _sync_add_access_group_to_teams(tx, team_ids: list[str], access_group_ ) -async def _sync_remove_access_group_from_teams(tx, team_ids: list[str], access_group_id: str) -> None: +async def _sync_remove_access_group_from_teams(tx: _AccessGroupTx, team_ids: list[str], access_group_id: str) -> None: """Remove access_group_id from each team's access_group_ids (idempotent).""" for team_id in team_ids: team = await tx.litellm_teamtable.find_unique(where={"team_id": team_id}) if team is not None and access_group_id in (team.access_group_ids or []): await tx.litellm_teamtable.update( where={"team_id": team_id}, - data={"access_group_ids": [ag for ag in team.access_group_ids if ag != access_group_id]}, + data={"access_group_ids": [ag for ag in (team.access_group_ids or ()) if ag != access_group_id]}, ) -async def _sync_add_access_group_to_keys(tx, key_tokens: list[str], access_group_id: str) -> None: +async def _sync_add_access_group_to_keys(tx: _AccessGroupTx, key_tokens: list[str], access_group_id: str) -> None: """Add access_group_id to each key's access_group_ids (idempotent).""" for token in key_tokens: key = await tx.litellm_verificationtoken.find_unique(where={"token": token}) @@ -142,14 +200,14 @@ async def _sync_add_access_group_to_keys(tx, key_tokens: list[str], access_group ) -async def _sync_remove_access_group_from_keys(tx, key_tokens: list[str], access_group_id: str) -> None: +async def _sync_remove_access_group_from_keys(tx: _AccessGroupTx, key_tokens: list[str], access_group_id: str) -> None: """Remove access_group_id from each key's access_group_ids (idempotent).""" for token in key_tokens: key = await tx.litellm_verificationtoken.find_unique(where={"token": token}) if key is not None and access_group_id in (key.access_group_ids or []): await tx.litellm_verificationtoken.update( where={"token": token}, - data={"access_group_ids": [ag for ag in key.access_group_ids if ag != access_group_id]}, + data={"access_group_ids": [ag for ag in (key.access_group_ids or ()) if ag != access_group_id]}, ) @@ -280,6 +338,7 @@ async def create_access_group( prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) try: + tx: _AccessGroupTx async with prisma_client.db.tx() as tx: existing = await tx.litellm_accessgrouptable.find_unique( where={"access_group_name": data.access_group_name} @@ -347,7 +406,8 @@ async def list_access_groups( _require_admin_view(user_api_key_dict) prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) - records = await AccessGroupRepository(prisma_client).table.find_many(order={"created_at": "desc"}) + table: _AccessGroupTable = AccessGroupRepository(prisma_client).table + records = await table.find_many(order={"created_at": "desc"}) return [_record_to_response(r) for r in records] @@ -362,7 +422,8 @@ async def get_access_group( _require_admin_view(user_api_key_dict) prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) - record = await AccessGroupRepository(prisma_client).table.find_unique(where={"access_group_id": access_group_id}) + table: _AccessGroupTable = AccessGroupRepository(prisma_client).table + record = await table.find_unique(where={"access_group_id": access_group_id}) if record is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -408,6 +469,7 @@ async def update_access_group( keys_to_remove: list[str] = [] try: + tx: _AccessGroupTx async with prisma_client.db.tx() as tx: # Read inside the transaction so delta computation is consistent with the write, # avoiding a TOCTOU race where a concurrent update could make deltas stale. @@ -480,6 +542,7 @@ async def delete_access_group( affected_team_ids: list[str] = [] affected_key_tokens: list[str] = [] + tx: _AccessGroupTx async with prisma_client.db.tx() as tx: existing = await tx.litellm_accessgrouptable.find_unique(where={"access_group_id": access_group_id}) if existing is None: @@ -512,7 +575,7 @@ async def delete_access_group( for team in teams_with_group: await tx.litellm_teamtable.update( where={"team_id": team.team_id}, - data={"access_group_ids": [ag for ag in (team.access_group_ids or []) if ag != access_group_id]}, + data={"access_group_ids": [ag for ag in (team.access_group_ids or ()) if ag != access_group_id]}, ) # Use _sync_remove only for out-of-sync teams not found by the hasSome query. out_of_sync_team_ids = set(existing.assigned_team_ids or []) - {t.team_id for t in teams_with_group} @@ -522,7 +585,7 @@ async def delete_access_group( for key in keys_with_group: await tx.litellm_verificationtoken.update( where={"token": key.token}, - data={"access_group_ids": [ag for ag in (key.access_group_ids or []) if ag != access_group_id]}, + data={"access_group_ids": [ag for ag in (key.access_group_ids or ()) if ag != access_group_id]}, ) # Use _sync_remove only for out-of-sync keys not found by the hasSome query. out_of_sync_key_tokens = set(existing.assigned_key_ids or []) - {k.token for k in keys_with_group} diff --git a/litellm/proxy/management_endpoints/cache_settings_endpoints.py b/litellm/proxy/management_endpoints/cache_settings_endpoints.py index e08bc13a14d..439449403c4 100644 --- a/litellm/proxy/management_endpoints/cache_settings_endpoints.py +++ b/litellm/proxy/management_endpoints/cache_settings_endpoints.py @@ -63,7 +63,7 @@ _REDACTED_VALUE = "***REDACTED***" _URL_OVERRIDDEN_CONNECTION_FIELDS: frozenset = frozenset({"host", "port", "db", "password", "username"}) -def _resolve_cache_url_precedence(settings: Mapping[str, Any]) -> dict[str, Any]: +def _resolve_cache_url_precedence(settings: Mapping[str, object]) -> dict[str, Any]: """Return cache settings with the url-vs-discrete-fields ambiguity resolved. When a full ``url`` is supplied it wins: the discrete @@ -80,7 +80,7 @@ def _resolve_cache_url_precedence(settings: Mapping[str, Any]) -> dict[str, Any] return {k: v for k, v in settings.items() if k not in _URL_OVERRIDDEN_CONNECTION_FIELDS} -def _parse_stored_settings(cache_settings_value: object) -> dict[str, Any]: +def _parse_stored_settings(cache_settings_value: object) -> dict[str, object]: """Normalize a stored cache_settings blob to a dict. The prisma column comes back as either a JSON string or an already-parsed @@ -91,7 +91,7 @@ def _parse_stored_settings(cache_settings_value: object) -> dict[str, Any]: return parsed if isinstance(parsed, dict) else {} -def _overlay_environment(stored: Mapping[str, Any]) -> dict[str, Any]: +def _overlay_environment(stored: Mapping[str, object]) -> dict[str, object]: """Fill connection fields from the REDIS_* environment the cache actually reads. A response cache pointed at Redis resolves host/port/password/etc. from the @@ -113,7 +113,7 @@ def _overlay_environment(stored: Mapping[str, Any]) -> dict[str, Any]: return effective -def _redact_credentials(settings: Mapping[str, Any]) -> dict[str, Any]: +def _redact_credentials(settings: Mapping[str, object]) -> dict[str, object]: """Replace credential-bearing values with a fixed marker, keeping the rest. The marker is unambiguous on the way back in: an admin who edits an @@ -164,7 +164,7 @@ def _target_repr(value: object) -> str: return str(value) -def _saved_secret_is_reusable(incoming: Mapping[str, Any], saved: Mapping[str, Any]) -> bool: +def _saved_secret_is_reusable(incoming: Mapping[str, object], saved: Mapping[str, object]) -> bool: """Whether a stored credential may be restored for this request. A stored secret belongs to the stored connection target, so it is reused only @@ -197,7 +197,7 @@ def _saved_secret_is_reusable(incoming: Mapping[str, Any], saved: Mapping[str, A return True -def _merge_over_saved(incoming: Mapping[str, Any], saved: Mapping[str, Any]) -> dict[str, Any]: +def _merge_over_saved(incoming: Mapping[str, object], saved: Mapping[str, object]) -> dict[str, Any]: """Keep the stored secret behind any credential the caller echoed back redacted or omitted. GET returns credentials as the marker and the form never re-prefills a @@ -239,7 +239,7 @@ def _merge_over_saved(incoming: Mapping[str, Any], saved: Mapping[str, Any]) -> return merged -def _redact_settings(settings: Mapping[str, Any] | None) -> dict[str, Any]: +def _redact_settings(settings: Mapping[str, object] | None) -> dict[str, object]: """Replace every value in a settings map with a fixed marker. Cache config carries Redis credentials (passwords, connection strings). @@ -268,8 +268,8 @@ def _log_audit_task_exception(task: "asyncio.Task[None]") -> None: async def _emit_cache_settings_audit_log( *, action: AUDIT_ACTIONS, - before_settings: Mapping[str, Any] | None, - after_settings: Mapping[str, Any] | None, + before_settings: Mapping[str, object] | None, + after_settings: Mapping[str, object] | None, user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: str | None, ) -> None: @@ -313,17 +313,17 @@ class CacheSettingsManager: Tracks last cache params to avoid unnecessary reinitialization. """ - _last_cache_params: dict[str, Any] | None = None + _last_cache_params: dict[str, object] | None = None @staticmethod - def _cache_params_equal(params1: dict[str, Any], params2: dict[str, Any]) -> bool: + def _cache_params_equal(params1: dict[str, object], params2: dict[str, object]) -> bool: """ Compare two cache parameter dictionaries for equality. Normalizes values and filters out UI-only fields. """ # Normalize by removing None values and UI-only fields - def normalize(params: dict[str, Any]) -> dict[str, Any]: + def normalize(params: dict[str, object]) -> dict[str, object]: normalized = {} for k, v in params.items(): if k == "redis_type": # Skip UI-only field @@ -390,7 +390,7 @@ class CacheSettingsManager: ) @staticmethod - def update_cache_params(cache_params: dict[str, Any]): + def update_cache_params(cache_params: dict[str, object]): """ Update the last cache params after initialization. Called after cache settings are updated via the API. @@ -400,12 +400,12 @@ class CacheSettingsManager: class CacheSettingsResponse(BaseModel): fields: list[CacheSettingsField] = Field(description="List of all configurable cache settings with metadata") - current_values: dict[str, Any] = Field(description="Current values of cache settings") + current_values: dict[str, object] = Field(description="Current values of cache settings") redis_type_descriptions: dict[str, str] = Field(description="Descriptions for each Redis type option") class CacheTestRequest(BaseModel): - cache_settings: dict[str, Any] = Field(description="Cache settings to test connection with") + cache_settings: dict[str, object] = Field(description="Cache settings to test connection with") class CacheTestResponse(BaseModel): @@ -415,7 +415,7 @@ class CacheTestResponse(BaseModel): class CacheSettingsUpdateRequest(BaseModel): - cache_settings: dict[str, Any] = Field(description="Cache settings to save") + cache_settings: dict[str, object] = Field(description="Cache settings to save") @router.get( @@ -441,7 +441,7 @@ async def get_cache_settings( cache_fields = [field.model_copy(deep=True) for field in CACHE_SETTINGS_FIELDS] # Read the stored settings (decrypted); an env-only cache has none. - stored: dict[str, Any] = {} + stored: dict[str, object] = {} if prisma_client is not None: cache_config = await CacheConfigRepository(prisma_client).table.find_unique(where={"id": "cache_config"}) if cache_config is not None and cache_config.cache_settings: @@ -507,7 +507,7 @@ async def test_cache_connection( # A credential the form left untouched arrives redacted; resolve it back # to the stored secret so the test connects with the real password. A # lookup failure must not block the test, so fall back to no stored row. - saved_settings: dict[str, Any] = {} + saved_settings: dict[str, object] = {} if prisma_client is not None: try: existing_row = await CacheConfigRepository(prisma_client).table.find_unique( @@ -590,8 +590,8 @@ async def update_cache_settings( # Read the stored row first: its decrypted values back any credential the # caller echoed back redacted, and its key set drives the audit diff. existing_row = await CacheConfigRepository(prisma_client).table.find_unique(where={"id": "cache_config"}) - before_settings: dict[str, Any] | None = None - saved_settings: dict[str, Any] = {} + before_settings: dict[str, object] | None = None + saved_settings: dict[str, object] = {} if existing_row is not None and existing_row.cache_settings: before_settings = _parse_stored_settings(existing_row.cache_settings) saved_settings = proxy_config._decrypt_db_variables(variables_dict=before_settings) diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index d3ef01940bb..386108169bc 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -1,6 +1,6 @@ { "ANN001": { - "limit": 3104 + "limit": 3097 }, "ANN002": { "limit": 69 @@ -24,7 +24,7 @@ "limit": 130 }, "ANN401": { - "limit": 1851 + "limit": 1849 }, "ASYNC230": { "limit": 14 diff --git a/type-discipline-budget.json b/type-discipline-budget.json index f071c381916..a2cf4e351f5 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 23350 + "limit": 23349 }, "LIT002": { - "limit": 27256 + "limit": 27253 }, "LIT003": { "limit": 292 From c98d595359c7f58572c0d7cb37769fc85bfd419e Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 3 Aug 2026 17:07:05 -0700 Subject: [PATCH 103/123] fix(proxy): redact credential headers from request logging copies (#35678) * fix(proxy): redact credential headers from request logging copies clean_headers preserves an Anthropic subscription OAuth token, and other client-supplied provider credentials, so they can be forwarded upstream. The same dict was also stored as proxy_server_request["headers"] and metadata["headers"], so those credentials reached every logging callback and the SpendLogs proxy_server_request column that the Admin UI logs page renders. Build the observability facing copies through redact_credential_headers, and drop the transport-only keys (provider_specific_header, headers, api_key) from the request body snapshot since they have to keep the real values. * fix(proxy): use the redacted header copy in the request debug log The stdout secret filter matches Bearer and sk- shaped values, so an MCP auth token printed by the request-header debug line survived it in cleartext. * fix(proxy): resolve the configured MCP auth header name through the secret manager get_secret_str also consults a configured secret manager, so a deployment that stores the header name there now gets that header masked too. Drops the added comments in favour of a named constant. * perf(proxy): resolve the MCP auth header name once per process get_secret_str issues a blocking secret-manager SDK call when one is configured, and configured_credential_header_names runs on every proxied request. * fix(proxy): read the MCP auth header name live, cache only the secret manager The config reloader rewrites os.environ on an interval and after /config/update, and MCPRequestHandler resolves the same setting per request, so caching the env lookup left a renamed header logged in the clear until the process restarted. Only the blocking secret-manager call stays cached. * refactor(proxy): narrow header redaction to the reported credential set Drops the MCP header-name resolution, its per-request config and secret-manager lookups, and the x-mcp- prefix rule. Those cover a separate credential family than the one this ticket reports and carried their own config-reload staleness surface; they belong in their own change. --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/litellm_pre_call_utils.py | 42 ++++- .../proxy/test_litellm_pre_call_utils.py | 149 +++++++++++++++++- 2 files changed, 185 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 13eb41af751..6864caccea2 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -4,6 +4,7 @@ import json import re import time from collections import OrderedDict +from collections.abc import Mapping from typing import TYPE_CHECKING, Any from fastapi import HTTPException, Request @@ -48,6 +49,12 @@ from litellm.proxy.common_utils.http_parsing_utils import _safe_get_request_head # Cache special headers as a frozenset for O(1) lookup performance _SPECIAL_HEADERS_CACHE = frozenset(v.value.lower() for v in SpecialHeaders._member_map_.values()) +_REDACTED_HEADER_VALUE = "***REDACTED***" +_CREDENTIAL_HEADER_NAMES = SpecialHeaders.litellm_credential_header_names() | frozenset( + {"cookie", "proxy-authorization"} +) +_TRANSPORT_ONLY_CREDENTIAL_KEYS = frozenset({"provider_specific_header", "headers", "api_key"}) + # Matches any header of the form x--session-id (case-insensitive). # Excludes the two explicit litellm headers which are handled with higher priority. _GENERIC_SESSION_ID_HEADER_RE = re.compile(r"^x-.+-session-id$", re.IGNORECASE) @@ -747,6 +754,30 @@ def clean_headers( return clean_headers +def _is_credential_header(header: str) -> bool: + """Whether `header` carries a caller credential rather than request context.""" + return header.lower() in _CREDENTIAL_HEADER_NAMES + + +def redact_credential_headers(headers: Mapping[str, str]) -> Mapping[str, str]: + """Return a copy of `headers` with credential-bearing values masked. + + `clean_headers` deliberately preserves some credential headers so they can be + forwarded to the upstream provider; an Anthropic subscription OAuth token in + `Authorization`, or a client-supplied provider key in `x-api-key`. Those values + must never reach a logging callback or a spend log, so every observability-facing + copy of the header dict is built through this helper while the copy that is + forwarded upstream keeps the real values. + + The returned object is a plain dict; guardrail hooks stamp their own headers onto + the stored copy and the logging callbacks JSON-serialize it. + """ + return { + header: (_REDACTED_HEADER_VALUE if _is_credential_header(header) else value) + for header, value in headers.items() + } + + class LiteLLMProxyRequestSetup: @staticmethod def _get_timeout_from_request(headers: dict) -> float | None: @@ -1443,7 +1474,8 @@ async def add_litellm_data_to_request( _headers, allow_client_message_redaction_opt_out=_allow_client_message_redaction_opt_out, ) - verbose_proxy_logger.debug(f"Request Headers: {_headers}") + _logging_safe_headers = redact_credential_headers(_headers) + verbose_proxy_logger.debug(f"Request Headers: {_logging_safe_headers}") verbose_proxy_logger.debug(f"Raw Headers: {_raw_headers}") if forward_llm_auth and "x-api-key" in _headers: @@ -1464,7 +1496,7 @@ async def add_litellm_data_to_request( data["proxy_server_request"] = { "url": str(request.url), "method": request.method, - "headers": _headers, + "headers": _logging_safe_headers, "body": None, # filled in post-strip; see below "arrival_time": arrival_time, # Track when request arrived at proxy } @@ -1490,7 +1522,7 @@ async def add_litellm_data_to_request( # Expose request headers under the metadata field for guardrails (fixes #17477) if _metadata_variable_name in data and isinstance(data[_metadata_variable_name], dict): - data[_metadata_variable_name]["headers"] = _headers + data[_metadata_variable_name]["headers"] = _logging_safe_headers # check for forwardable headers data = LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group( @@ -1619,7 +1651,7 @@ async def add_litellm_data_to_request( # self-reference — body.proxy_server_request.body would be the same # dict as body, producing an infinite traversal loop for any consumer # that walks the structure. - _body_snapshot_exclude = {"secret_fields", "proxy_server_request"} + _body_snapshot_exclude = frozenset({"secret_fields", "proxy_server_request"}) | _TRANSPORT_ONLY_CREDENTIAL_KEYS _body_snapshot = {k: v for k, v in data.items() if k not in _body_snapshot_exclude} data["proxy_server_request"]["body"] = _body_snapshot @@ -1726,7 +1758,7 @@ async def add_litellm_data_to_request( data[_metadata_variable_name]["user_api_key_team_object_permission_id"] = getattr( user_api_key_dict, "team_object_permission_id", None ) - data[_metadata_variable_name]["headers"] = _headers + data[_metadata_variable_name]["headers"] = _logging_safe_headers data[_metadata_variable_name]["endpoint"] = str(request.url) # Carry the proxy-receive instant via metadata (like `endpoint`) so the # OTel layer can compute pre-request latency, including on the failure diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 37642605088..0e9aac7bf85 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -5559,6 +5559,153 @@ def test_warn_stale_team_alias_once_evicts_oldest_key_beyond_cap(monkeypatch): assert list(pre_call_utils._STALE_TEAM_ALIAS_WARNING_KEYS) == ["key-2", "key-3"] +_OAUTH_TOKEN = "Bearer sk-ant-oat01-regression-token-lit5108" + + +def _all_header_dicts(data: dict, metadata_variable_name: str) -> list[dict]: + metadata = data.get(metadata_variable_name) or {} + proxy_server_request = data["proxy_server_request"] + body = proxy_server_request["body"] + return [ + metadata.get("headers") or {}, + (metadata.get("requester_metadata") or {}).get("headers") or {}, + proxy_server_request["headers"], + (body.get(metadata_variable_name) or {}).get("headers") or {}, + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path, metadata_variable_name", + [ + ("/v1/messages", "litellm_metadata"), + ("/v1/chat/completions", "metadata"), + ], +) +async def test_add_litellm_data_to_request_redacts_oauth_header_from_logging_copies(path, metadata_variable_name): + """The Anthropic subscription token is forwarded upstream but never handed to logging.""" + request_mock = _make_request_mock( + path, + { + "Content-Type": "application/json", + "anthropic-version": "2023-06-01", + "Authorization": _OAUTH_TOKEN, + "x-litellm-api-key": "Bearer sk-virtual-key", + }, + ) + + updated = await add_litellm_data_to_request( + data={"model": "anthropic-claude", "messages": [{"role": "user", "content": "hello"}]}, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"forward_client_headers_to_llm_api": True}, + version="test-version", + ) + + for header_dict in _all_header_dicts(updated, metadata_variable_name): + assert header_dict.get("Authorization") != _OAUTH_TOKEN + assert "sk-ant-oat01" not in json.dumps(header_dict) + + assert "sk-ant-oat01" not in json.dumps(updated["proxy_server_request"], default=repr) + + assert updated["proxy_server_request"]["headers"] is updated[metadata_variable_name]["headers"] + + assert updated["provider_specific_header"]["extra_headers"]["Authorization"] == _OAUTH_TOKEN + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_keeps_every_forwarded_credential_out_of_logging_copies(): + """Credentials kept for transport must not survive anywhere under proxy_server_request.""" + secrets = { + "x-api-key": "sk-byok-provider-key-lit5108", + "cookie": "litellm_jwt=session-token-lit5108", + "proxy-authorization": "Bearer proxy-token-lit5108", + } + request_mock = _make_request_mock( + "/v1/chat/completions", + { + "Content-Type": "application/json", + "x-litellm-api-key": "Bearer sk-virtual-key", + **secrets, + }, + ) + + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": [{"role": "user", "content": "hello"}]}, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={ + "forward_llm_provider_auth_headers": True, + "forward_client_headers_to_llm_api": True, + }, + version="test-version", + ) + + assert updated["api_key"] == secrets["x-api-key"] + assert updated["headers"]["x-api-key"] == secrets["x-api-key"] + + logged = json.dumps(updated["proxy_server_request"], default=repr) + for value in secrets.values(): + assert value not in logged + + + +@pytest.mark.parametrize( + "header, expected_redacted", + [ + ("Authorization", True), + ("X-Api-Key", True), + ("x-goog-api-key", True), + ("Ocp-Apim-Subscription-Key", True), + ("API-Key", True), + ("Cookie", True), + ("Proxy-Authorization", True), + ("anthropic-version", False), + ("user-agent", False), + ], +) +def test_redact_credential_headers_classifies_each_header(header, expected_redacted): + from litellm.proxy.litellm_pre_call_utils import redact_credential_headers + + headers = {header: "secret-value"} + + redacted = redact_credential_headers(headers) + + assert redacted[header] == ("***REDACTED***" if expected_redacted else "secret-value") + assert headers[header] == "secret-value" + + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_debug_log_does_not_print_credentials(): + """The request-header debug line carries values the stdout secret filter does not match.""" + import litellm.proxy.litellm_pre_call_utils as pre_call_utils + + request_mock = _make_request_mock( + "/v1/chat/completions", + { + "Content-Type": "application/json", + "Ocp-Apim-Subscription-Key": "apim-plaintext-token-lit5108", + "x-litellm-api-key": "Bearer sk-virtual-key", + }, + ) + + with patch.object(pre_call_utils.verbose_proxy_logger, "debug") as mock_debug: + await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": [{"role": "user", "content": "hello"}]}, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"forward_llm_provider_auth_headers": True}, + version="test-version", + ) + + logged = " ".join(str(call) for call in mock_debug.call_args_list) + assert "apim-plaintext-token-lit5108" not in logged + + def _callback_credential_request_mock() -> MagicMock: request_mock = MagicMock(spec=Request) request_mock.url = MagicMock() @@ -5722,4 +5869,4 @@ async def test_key_level_callback_vars_survive_the_strip(): ) assert updated[TRUSTED_CALLBACK_VARS_FIELD] == {"dd_api_key": "key-dd-key", "dd_site": "us5.datadoghq.com"} - assert updated["dd_site"] == "us5.datadoghq.com" + assert updated["dd_site"] == "us5.datadoghq.com" \ No newline at end of file From ba1bde70e4b45d4f2bf5e9dd4b49858e7d9ac691 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 4 Aug 2026 00:20:25 +0000 Subject: [PATCH 104/123] feat(guardrails/rubrik): prompt moderation, response-text blocking, streaming buffer, failure logging (#35722) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(guardrails/rubrik): prompt moderation, response-text blocking, streaming buffer, failure logging (#34019) * feat(guardrails/rubrik): add prompt moderation, response-text blocking, streaming buffer, failure logging - Add `pre_call` prompt moderation via `/v1/before_prompt/openai/v1` webhook: structured messages are flattened and sent before the LLM is called; blocked prompts surface a `ModifyResponseException` with the refusal text. - Extend `post_call` response moderation to cover assistant text in addition to tool calls; text blocks (wholesale replacement) are distinguished from tool-block explanations (appended) via `startswith` diffing. - Add `streaming_end_of_stream_only = True` and `streaming_buffer_until_moderated = True` so streamed responses are withheld until end-of-stream moderation passes (requires litellm >= BerriAI/litellm#31389; older versions fall back to detect-only). - Add `_MalformedToolBlockingResponseError` for structurally invalid service responses; `_guarded` logs at CRITICAL so operators notice misconfiguration. - Add `max_queue_size = 10_000`, `_enforce_max_queue_size`, and drop-oldest backpressure so a webhook outage cannot grow the retry queue unboundedly. - Add `flush_queue` override that snapshots once for both send and drain, preventing duplicate delivery on concurrent flush calls. - Make `_log_batch_to_rubrik` re-raise on error so `flush_queue` preserves undelivered events for the next retry. - Add `async_post_call_failure_hook` to log blocked requests (`ModifyResponseException`) with a best-effort fallback payload for prompt blocks (where no `standard_logging_object` exists yet). - Add `_correlation_id` / `_apply_correlation_id` / `_prepend_system_prompt` helpers; `_prepare_log_payload` now applies them for all providers (not just Anthropic) so every log correlates by `litellm_call_id`. - Add `get_supported_event_hooks` classmethod advertising `[pre_call, post_call]`. - Use dedicated `httpx.AsyncClient` (`moderation_client`) for webhook calls with explicit pool limits, separate from the shared logging client. - Drop module-level `rubrik_handler` singleton (inappropriate for a library). - Update `initialize_guardrail` docstring to explain `pre_call` vs `post_call` mode. - Update tests: rename `tool_blocking_client` → `moderation_client`, `tool_blocking_endpoint` → `response_moderation_endpoint`, `_flush_task` → `_periodic_flush_task`; migrate `TestExtractBlockedTools` to `TestExtractResponseBlock` for the new combined text+tool block API; add tests for prompt moderation, text blocking, streaming flags, and failure payload construction. Co-Authored-By: Claude Sonnet 4.6 (1M context) * test(guardrails/rubrik): add tests to reach 100% coverage 50 new tests across 18 classes covering previously-untested paths: - Prompt moderation: passthrough, block, no-messages skip, message flattening (content-list → string), payload construction with tools/user/correlation_key/litellm_call_id fallback, refusal extraction - async_post_call_failure_hook: non-matching exception no-op, missing stash warning, valid stash → enqueue, AttributeError in payload build, flush exception handling - Block payload building: standard_logging_object present vs fallback path, missing start_time - async_log_success_event: _rubrik_blocked=True skip path - aclose: task cancel + moderation_client.aclose() - Edge cases: sampling rate clamp warning, unknown input_type passthrough, empty-inputs early return, model_call_details warning, _stash_block_context, duck-typed tool-call normalization, request_data["tools"] preference over optional_params, system-prompt exception handler, flush-at-batch-size, enqueue exception swallowing, queue empty/lock-None guards, non-dict JSON response TypeError Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): use get_async_httpx_client, ruff format - Replace bare httpx.AsyncClient with get_async_httpx_client (required by ensure_async_clients_test; avoids per-request client creation) - aclose() calls close() (AsyncHTTPHandler interface, not aclose()) - ruff format on rubrik.py and guardrail_hooks/rubrik/__init__.py - Update 3 tests for AsyncHTTPHandler type (isinstance check, close()) osv-scan and documentation CI failures are pre-existing on the base branch and unrelated to this PR. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): fix UP006 strict ruff violation get_supported_event_hooks return type used List[...] (UP006) instead of list[...]. Replace with the built-in generic and remove the now-unused List import from typing. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): fix 3 reportArgumentType basedpyright violations Use `# pyright: ignore[reportArgumentType]` (not `# type: ignore`) to suppress the three errors basedpyright reports in --outputjson mode: - convert_content_list_to_str call (dict vs AllMessageValues) - _apply_correlation_id call (StandardLoggingPayload vs dict[str, Any]) - _prepend_system_prompt call (same) Also tighten _apply_correlation_id and _prepend_system_prompt signatures from bare `dict` to `dict[str, Any]`. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): don't close shared HTTP client in aclose() moderation_client and async_httpx_client both come from LiteLLM's global HTTP-client cache (get_async_httpx_client keys on llm_provider + params). Two RubrikLogger instances with the same parameters share the same underlying AsyncHTTPHandler object. Calling close() in aclose() closed the shared connection pool for all instances, breaking any subsequent moderation request on other loggers. aclose() now only cancels the periodic flush task and lets LiteLLM manage the shared client lifecycle. Tests updated to assert close() is NOT called. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): use Counter for duplicate tool-call ID detection Set-based comparison lost ID multiplicity: two original tool calls with the same ID both appeared "allowed" even when the service returned only one (e.g. one allowed + one prohibited sharing an ID). Replace with Counter so returned_id_counts[id] >= required_id_counts[id] must hold for every ID. Matches the approach in the original _extract_blocked_tools. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): respect default_on=true when omitted from config LitellmParams.__init__ converts an omitted default_on to False before initialize_guardrail receives it, so litellm_params.default_on is always bool and never None. The is-None guard in RubrikLogger.__init__ therefore never fired on the proxy path, leaving prompt/response moderation inactive for any config that omitted default_on. Fix: read the raw guardrail dict (before LitellmParams coercion) to distinguish an explicit `default_on: false` from the absent-means-True default. When the key is absent from the raw config, default_on=True is used; when it is explicitly set (either True or False), that value wins. Co-Authored-By: Claude Sonnet 4.6 (1M context) * style: ruff format rubrik.py after Counter import addition Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): detect ID-less tool call removal; fix UP045 ID-less tool calls (tc.id is falsy) were excluded from required_id_counts, so the Counter comparison never caught their removal. Add a cardinality check (len(returned) < len(original)) that fires on any removal regardless of ID presence, combined with the Counter check for duplicate-ID attacks. Also fix 5 UP045 violations (Optional[X] → X | None) introduced by our new code against the daily-branch baseline. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): filter optional_params through ModelParamHelper in fallback payload _build_fallback_payload forwarded the raw optional_params dict as model_parameters. optional_params can contain extra_headers, api_key, and other upstream provider credentials that must not reach the Rubrik webhook. The normal standard_logging_object path already filters through ModelParamHelper.get_standard_logging_model_parameters(), which allowlists only safe LLM API parameters. Apply the same filter here. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): scope failure hook by guardrail_name; moderate text-completions Guard async_post_call_failure_hook by guardrail_name so multiple Rubrik instances don't cross-log: the failure hook is called for every registered callback; without the check the first instance pops the stash and the originating instance finds None and silently skips logging. Now each instance only handles blocks raised by itself. Also moderate /v1/completions prompts: _moderate_prompt returned early when structured_messages was absent. For text-completion requests litellm supplies inputs["texts"] with no structured_messages. Added a fallback that synthesises a user-message from texts so the before_prompt webhook can evaluate text-completion prompts. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(lint): add reason comments to pyright: ignore suppressions type-discipline budget requires each # pyright: ignore[...] to carry an explanatory comment. Add reasons to the three bare suppressions on lines 483, 651, 652. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): include tool-call arguments in prompt moderation _flatten_messages_for_moderation only sent the content field, silently dropping tool_calls[].function.arguments and function_call.arguments. An attacker could embed prohibited text in tool-call arguments inside assistant history turns and bypass prompt moderation entirely. Now collects all attacker-controlled text per message: text content via convert_content_list_to_str, plus all tool_calls[].function.arguments and the deprecated function_call.arguments, joined with newlines before being sent to the before_prompt webhook. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): tighten append detection to prevent prefix bypass startswith(sent_content) allowed any replacement whose text shares the original as a prefix (e.g. "Hello" → "Hello, blocked.") to be classified as a tool-block append rather than a text block, bypassing detection. Use startswith(f"{sent_content}\n\n") to require the exact two-newline separator the webhook uses between original text and appended tool-block explanations. Also add `returned_content != sent_content` to text_blocked so an unchanged passthrough is never classified as a block. Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(guardrails/rubrik): default_on=False when omitted (follow existing pattern) Remove the custom raw-dict lookup that was defaulting default_on to True when omitted from the guardrail config. Follow the standard litellm convention: omitted resolves to False (users must explicitly opt in with default_on: true). - initialize_guardrail: pass litellm_params.default_on directly - RubrikLogger.__init__: is-None guard defaults to False not True - Test updated to assert the correct False default Co-Authored-By: Claude Sonnet 4.6 (1M context) --------- Co-authored-by: Claude Sonnet 4.6 (1M context) * chore(rubrik): keep the ported guardrail within staging lint budgets Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore: credit the original author of the rubrik guardrail work Co-authored-by: Joseph Barker <156112794+seph-barker@users.noreply.github.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore: keep this mirror PR's diff limited to the rubrik files Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Joseph Barker <156112794+seph-barker@users.noreply.github.com> Co-authored-by: Claude Sonnet 4.6 (1M context) Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/rubrik.py | 1040 +++++++++++++---- .../guardrail_hooks/rubrik/__init__.py | 12 + .../test_litellm/integrations/test_rubrik.py | 956 +++++++++++++-- 3 files changed, 1691 insertions(+), 317 deletions(-) diff --git a/litellm/integrations/rubrik.py b/litellm/integrations/rubrik.py index 2e49da45ce9..4bcbe8bae37 100644 --- a/litellm/integrations/rubrik.py +++ b/litellm/integrations/rubrik.py @@ -1,12 +1,14 @@ -"""Rubrik LiteLLM Plugin for tool blocking and batch logging.""" +"""Rubrik LiteLLM Plugin for prompt/response moderation and batch logging.""" import asyncio import os import random import time -import urllib.parse import uuid from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Literal, Optional import httpx @@ -18,6 +20,10 @@ from litellm.integrations.custom_guardrail import ( ModifyResponseException, ) from litellm.litellm_core_utils.core_helpers import safe_deep_copy +from litellm.litellm_core_utils.model_param_helper import ModelParamHelper +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + convert_content_list_to_str, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -35,15 +41,16 @@ if TYPE_CHECKING: Logging as LiteLLMLoggingObj, ) -_ENDPOINT_ANTHROPIC_MESSAGES = "/v1/messages" -_WEBHOOK_PATH_TOOL_BLOCKING = "/v1/after_completion/openai/v1" +_WEBHOOK_PATH_RESPONSE_MODERATION = "/v1/after_completion/openai/v1" +_WEBHOOK_PATH_PROMPT_MODERATION = "/v1/before_prompt/openai/v1" _WEBHOOK_PATH_LOGGING_BATCH = "/v1/litellm/batch" _MAX_QUEUE_SIZE = 10_000 _DROP_WARNING_INTERVAL_SECONDS = 60.0 +_EMPTY_MAPPING: Mapping[str, Any] = MappingProxyType({}) class _MalformedToolBlockingResponseError(Exception): - """Raised when the tool blocking service returns a structurally invalid + """Raised when the response moderation service returns a structurally invalid response (e.g. empty ``choices``). Distinct from transient network/HTTP errors so callers can surface a @@ -52,11 +59,15 @@ class _MalformedToolBlockingResponseError(Exception): """ -class RubrikLogger(CustomGuardrail, CustomBatchLogger): - @classmethod - def get_supported_event_hooks(cls) -> list[GuardrailEventHooks]: - return [GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call] +@dataclass +class BlockedResponseResult: + """Returned by _extract_response_block when the response was blocked + (response text replaced, or at least one tool call removed).""" + explanation: str + + +class RubrikLogger(CustomGuardrail, CustomBatchLogger): def __init__( self, api_key: str | None = None, @@ -67,21 +78,82 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): kwargs.setdefault("guardrail_name", "rubrik") # `initialize_guardrail` always passes these kwargs explicitly, with # value `None` when the user omits `mode` / `default_on` from the - # guardrail config. Coerce None (omitted) to the desired default - # while preserving any explicit value the caller did set -- - # in particular `default_on=False` if the user wants the guardrail - # off by default. + # guardrail config. Follow the standard litellm convention: omitted + # resolves to False (off by default, user must opt in explicitly). kwargs["event_hook"] = kwargs.get("event_hook") or GuardrailEventHooks.post_call if kwargs.get("default_on") is None: - kwargs["default_on"] = True - kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks())) + kwargs["default_on"] = False super().__init__( flush_lock=self.flush_lock, + supported_event_hooks=list(self.get_supported_event_hooks()), **kwargs, ) verbose_logger.debug("initializing rubrik logger") + # Defining ``apply_guardrail`` routes streaming responses through + # litellm's ``unified_guardrail.async_post_call_streaming_iterator_hook``. + # By default that hook samples intermediate chunks + # (``streaming_sampling_rate``, default 5) and also moderates at + # end-of-stream, so a streamed response costs ~ceil(N/5)+1 Rubrik + # webhook round-trips. litellm reads this attribute via + # ``getattr(guardrail, "streaming_end_of_stream_only", False)``; when + # True it yields chunks unprocessed and only moderates the fully + # assembled response once at end of stream. + self.streaming_end_of_stream_only = True + + # ``streaming_end_of_stream_only`` is detect-only: it releases every + # chunk to the client *before* moderating, so a block can only append a + # trailing message -- the original content has already been delivered. + # ``streaming_buffer_until_moderated`` (litellm >= BerriAI/litellm#31389) + # withholds all chunks until end-of-stream moderation passes, then + # releases the original response (clean) or only the block message + # (blocked). On older litellm this attribute is ignored and we fall + # back to the detect-only behavior above. + self.streaming_buffer_until_moderated = True + + self._parse_sampling_rate() + + self.key = api_key or os.getenv("RUBRIK_API_KEY") + if not self.key: + verbose_logger.warning("Rubrik: No API key configured. Requests will be unauthenticated.") + + self._parse_batch_size() + + # Cap the in-memory retry queue so a Rubrik webhook outage cannot let + # authenticated traffic accumulate prompt/response payloads until the + # proxy runs out of memory. Once the cap is reached, oldest events are + # dropped to make room for fresh ones (drop-oldest backpressure). + self.max_queue_size = _MAX_QUEUE_SIZE + self._dropped_since_warning = 0 + self._last_drop_warning_time = 0.0 + + _webhook_url = api_base or os.getenv("RUBRIK_WEBHOOK_URL") + if not _webhook_url: + raise ValueError("Rubrik webhook URL not configured. Set RUBRIK_WEBHOOK_URL or pass api_base.") + + _webhook_url = _webhook_url.rstrip("/").removesuffix("/v1") + self._setup_clients(_webhook_url) + + self._headers: Mapping[str, str] = MappingProxyType( + {"Content-Type": "application/json", "Authorization": f"Bearer {self.key}"} + if self.key + else {"Content-Type": "application/json"} + ) + + self._periodic_flush_task: asyncio.Task[Any] | None = self._start_periodic_flush_task() + + @classmethod + def get_supported_event_hooks(cls) -> list[GuardrailEventHooks]: + """Return the guardrail event hooks this integration supports. + + Prompt moderation (``pre_call``) evaluates the user's message before + the LLM is called. Response moderation (``post_call``) evaluates the + assistant's reply and tool calls after the LLM returns. + """ + return [GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call] + + def _parse_sampling_rate(self) -> None: self.sampling_rate = 1.0 rbrk_sampling_rate = os.getenv("RUBRIK_SAMPLING_RATE") if rbrk_sampling_rate is not None: @@ -93,80 +165,54 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): except ValueError: verbose_logger.warning(f"Invalid RUBRIK_SAMPLING_RATE: {rbrk_sampling_rate!r}, using 1.0") - self.key = api_key or os.getenv("RUBRIK_API_KEY") - if not self.key: - verbose_logger.warning("Rubrik: No API key configured. Requests will be unauthenticated.") + def _parse_batch_size(self) -> None: _batch_size = os.getenv("RUBRIK_BATCH_SIZE") - if _batch_size: try: - self.batch_size = int(_batch_size) + parsed_size = int(_batch_size) + if parsed_size <= 0: + verbose_logger.warning(f"RUBRIK_BATCH_SIZE={_batch_size!r} must be > 0, using default") + else: + self.batch_size = parsed_size except ValueError: verbose_logger.warning(f"Invalid RUBRIK_BATCH_SIZE: {_batch_size!r}, using default") - # Cap the in-memory retry queue so a Rubrik webhook outage cannot let - # authenticated traffic accumulate prompt/response payloads until the - # proxy runs out of memory. Once the cap is reached, oldest events are - # dropped to make room for fresh ones (drop-oldest backpressure). - self.max_queue_size = _MAX_QUEUE_SIZE - self._dropped_since_warning = 0 - self._last_drop_warning_time = 0.0 - - _webhook_url = api_base or os.getenv("RUBRIK_WEBHOOK_URL") - - if _webhook_url is None: - raise ValueError("Rubrik webhook URL not configured. Set RUBRIK_WEBHOOK_URL or pass api_base.") - - _webhook_url = _webhook_url.rstrip("/").removesuffix("/v1") - self.tool_blocking_endpoint = f"{_webhook_url}{_WEBHOOK_PATH_TOOL_BLOCKING}" - self.logging_endpoint = f"{_webhook_url}{_WEBHOOK_PATH_LOGGING_BATCH}" + def _setup_clients(self, webhook_url: str) -> None: + self.response_moderation_endpoint = f"{webhook_url}{_WEBHOOK_PATH_RESPONSE_MODERATION}" + self.prompt_moderation_endpoint = f"{webhook_url}{_WEBHOOK_PATH_PROMPT_MODERATION}" + self.logging_endpoint = f"{webhook_url}{_WEBHOOK_PATH_LOGGING_BATCH}" self.async_httpx_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback) - self.tool_blocking_client = get_async_httpx_client( + self.moderation_client = get_async_httpx_client( llm_provider=httpxSpecialProvider.LoggingCallback, params={"timeout": httpx.Timeout(5.0, connect=2.0)}, ) - self._headers: dict[str, str] = {"Content-Type": "application/json"} - if self.key: - self._headers["Authorization"] = f"Bearer {self.key}" - - # Periodic flush is started lazily on the first log event so that - # low-traffic deployments still get their batches drained even when the - # logger is instantiated outside a running event loop (sync init). - self._flush_task: asyncio.Task[Any] | None = self._start_periodic_flush_task() - def _start_periodic_flush_task(self) -> asyncio.Task[Any] | None: """Start the periodic flush task only when an event loop is already running.""" try: loop = asyncio.get_running_loop() except RuntimeError: - verbose_logger.debug( - "Rubrik logger init: no running event loop, periodic flush will start on first log event." - ) return None return loop.create_task(self.periodic_flush()) def _ensure_periodic_flush_task(self) -> None: - # Synchronous helper: in asyncio's cooperative model there is no await - # between the check and assignment, so two callers cannot race here. - if self._flush_task is None or self._flush_task.done(): - self._flush_task = self._start_periodic_flush_task() + if self._periodic_flush_task is None or self._periodic_flush_task.done(): + self._periodic_flush_task = self._start_periodic_flush_task() async def aclose(self): - """Close the dedicated HTTP clients used by this logger.""" - # Cancel the periodic flush task before closing the HTTP clients so - # the loop doesn't wake up and try to POST via a closed client. - if self._flush_task is not None and not self._flush_task.done(): - self._flush_task.cancel() - try: - await self._flush_task - except (asyncio.CancelledError, Exception): - pass - self._flush_task = None - await self.tool_blocking_client.close() - await self.async_httpx_client.close() + """Cancel the periodic flush task. + + ``moderation_client`` and ``async_httpx_client`` are shared objects + from LiteLLM's global HTTP-client cache (``get_async_httpx_client`` + uses the same cache key for all instances with equal parameters). + Closing them here would close the shared connection pool for every + other logger instance; let LiteLLM manage their lifecycle instead. + """ + task = getattr(self, "_periodic_flush_task", None) + if task is not None: + task.cancel() # -- Guardrail hook -------------------------------------------------------- @@ -177,67 +223,104 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): input_type: Literal["request", "response"], logging_obj: Optional["LiteLLMLoggingObj"] = None, ) -> GenericGuardrailAPIInputs: - """Validate tool calls against the blocking service (fail-open).""" - if input_type != "response": - return inputs + """Moderate prompts (request) and responses (response); fail-open. - tool_calls = inputs.get("tool_calls") - if not tool_calls: - return inputs + - ``request``: evaluate the prompt via the before_prompt webhook and + block disallowed prompts before the model is called. + - ``response``: evaluate the assistant's response text and tool calls + via the after_completion webhook and block on a policy violation. + litellm's guardrail-translation layer normalizes Anthropic and OpenAI + requests/responses into ``inputs`` before this runs, so a single code + path covers both wire formats. The configured guardrail ``mode`` + selects which surface(s) run. + """ + if input_type == "request": + return await self._guarded( + self._moderate_prompt(inputs, request_data, logging_obj), + inputs, + "Prompt moderation", + ) + if input_type == "response": + return await self._guarded( + self._moderate_response(inputs, request_data, logging_obj), + inputs, + "Response moderation", + ) + return inputs + + @staticmethod + async def _guarded( + coro: Any, + inputs: GenericGuardrailAPIInputs, + label: str, + ) -> GenericGuardrailAPIInputs: + """Await a moderation coroutine fail-open: re-raise an intentional + block, log at critical for malformed service responses, and swallow + any other error returning ``inputs`` unchanged.""" try: - return await self._check_tool_calls(inputs, tool_calls, request_data, logging_obj) + return await coro except ModifyResponseException: raise except _MalformedToolBlockingResponseError as e: - # Distinct from transient errors: the service responded but the - # payload was structurally invalid, which usually indicates a - # misconfigured webhook or a breaking change in its response - # format. Log loudly so operators notice their tool-blocking - # policy is not actually being enforced. + # The service responded but the payload was structurally invalid, + # which usually indicates a misconfigured webhook or a breaking + # change in its response format. Log loudly so operators notice + # their moderation policy is not actually being enforced. verbose_logger.critical( - "Tool blocking service returned a malformed response: %s. " - "Tool calls are NOT being checked -- verify the webhook " - "configuration. Returning original response unchanged.", + "Response moderation service returned a malformed response: %s. " + "Requests are NOT being checked -- verify the webhook " + "configuration. Returning original inputs unchanged.", e, exc_info=True, ) return inputs except Exception as e: verbose_logger.error( - f"Tool blocking hook failed: {e}. Returning original response unchanged.", + f"{label} hook failed: {e}. Returning original inputs unchanged.", exc_info=True, ) return inputs - async def _check_tool_calls( + async def _moderate_response( self, inputs: GenericGuardrailAPIInputs, - tool_calls: Any, request_data: dict, logging_obj: Optional["LiteLLMLoggingObj"], ) -> GenericGuardrailAPIInputs: - """Send tool calls to blocking service, raise if any are blocked.""" - message_tool_calls = self._normalize_tool_calls(tool_calls) + """Send response text + tool calls to the after_completion webhook and + raise if either the response text or any tool call is blocked.""" + tool_calls = inputs.get("tool_calls") + texts = inputs.get("texts") + if not tool_calls and not texts: + return inputs - call_details = getattr(logging_obj, "model_call_details", {}) if logging_obj else {} - response = request_data.get("response") - request_id = getattr(response, "id", None) if response else None + message_tool_calls = self._normalize_tool_calls(tool_calls or ()) + sent_content = self._join_texts(texts) + + call_details = getattr(logging_obj, "model_call_details", _EMPTY_MAPPING) if logging_obj else _EMPTY_MAPPING if logging_obj and not call_details: verbose_logger.warning( "Rubrik: logging_obj present but model_call_details is empty -- request context will be missing" ) - response_data = self._build_tool_call_payload(message_tool_calls, request_id) - req_data = self._extract_request_data(call_details) + # The moderation payload's ``id`` becomes the tool-blocking log's + # correlation key (the S3 filename), so it must match the failure + # (response) log written for the same blocked request. Both use + # ``litellm_call_id`` -- see ``_correlation_id``. + request_id = self._correlation_id(call_details, request_data) - service_response = await self._post_to_tool_blocking_service(response_data, req_data) - blocked_explanation = self._extract_blocked_tools(service_response, message_tool_calls) + response_data = self._build_response_moderation_payload(message_tool_calls, sent_content, request_id) + req_data = self._extract_request_data(call_details, request_data) - if blocked_explanation is not None: + service_response = await self._post_to_response_moderation_endpoint(response_data, req_data) + blocked = self._extract_response_block(service_response, message_tool_calls, sent_content) + + if blocked: model = self._resolve_model(request_data, call_details) + self._stash_block_context(logging_obj, request_data) raise ModifyResponseException( - message=blocked_explanation, + message=blocked.explanation, model=model, request_data=request_data, guardrail_name=self.guardrail_name, @@ -245,43 +328,125 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): return inputs - @staticmethod - def _normalize_tool_calls(tool_calls: Any) -> list[ChatCompletionMessageToolCall]: - """Convert tool_calls from inputs to ChatCompletionMessageToolCall objects.""" - result = [] - for tc in tool_calls: - if isinstance(tc, ChatCompletionMessageToolCall): - result.append(tc) - elif isinstance(tc, dict): - func = tc.get("function", {}) - result.append( - ChatCompletionMessageToolCall( - id=tc.get("id", ""), - type=tc.get("type", "function"), - function=Function( - name=func.get("name", ""), - arguments=func.get("arguments", ""), - ), - ) - ) - elif hasattr(tc, "id") and hasattr(tc, "function"): - result.append( - ChatCompletionMessageToolCall( - id=tc.id or "", - type=getattr(tc, "type", None) or "function", - function=tc.function, - ) - ) - else: - raise TypeError(f"Cannot normalize tool_call of type {type(tc).__name__}") - return result + async def _moderate_prompt( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + logging_obj: Optional["LiteLLMLoggingObj"], + ) -> GenericGuardrailAPIInputs: + """Send the (normalized) prompt to the before_prompt webhook and raise + if the prompt is blocked.""" + messages = inputs.get("structured_messages") + if not messages: + # For non-chat request types (e.g. /v1/completions), litellm + # supplies the prompt as ``texts`` with no structured_messages. + # Synthesise a user-message so the webhook can evaluate the prompt. + texts = inputs.get("texts") + if texts: + joined = "\n".join(t for t in texts if t) + if joined: + messages = [{"role": "user", "content": joined}] + if not messages: + return inputs + + payload = self._build_prompt_moderation_payload(inputs, request_data) + service_response = await self._post_to_prompt_moderation_endpoint(payload) + refusal = self._extract_prompt_refusal(service_response) + if refusal is None: + return inputs + + model = inputs.get("model") or request_data.get("model") or "unknown" + self._stash_block_context(logging_obj, request_data) + raise ModifyResponseException( + message=refusal, + model=model, + request_data=request_data, + guardrail_name=self.guardrail_name, + ) @staticmethod - def _build_tool_call_payload( - tool_calls: list[ChatCompletionMessageToolCall], + def _stash_block_context( + logging_obj: Optional["LiteLLMLoggingObj"], + request_data: dict, + ) -> None: + """Stash signals so the deferred success-event skips this request and + ``async_post_call_failure_hook`` can build the failure payload. + + - Sets a flag on ``logging_obj.model_call_details`` so the deferred + success-event handler short-circuits. + - Stashes a reference to ``logging_obj`` on ``request_data`` under a + custom key. ``ProxyLogging.post_call_failure_hook`` pops only + ``litellm_logging_obj`` before iterating callbacks, so this key + survives. + + When ``logging_obj`` is ``None`` the success-event has no way to + observe the block (the flag has nowhere to live), so we log an error + instead of silently dropping the signal. + """ + if logging_obj is None: + verbose_logger.error( + "Rubrik: moderation block fired with logging_obj=None for " + f"litellm_call_id={request_data.get('litellm_call_id')}; " + "cannot suppress success event or attach failure payload." + ) + request_data["_rubrik_logging_obj"] = None + return + logging_obj.model_call_details["_rubrik_blocked"] = True + request_data["_rubrik_logging_obj"] = logging_obj + + @staticmethod + def _normalize_tool_calls(tool_calls: Any) -> tuple[ChatCompletionMessageToolCall, ...]: + """Convert tool_calls from inputs to ChatCompletionMessageToolCall objects.""" + return tuple(RubrikLogger._normalize_tool_call(tc) for tc in tool_calls) + + @staticmethod + def _normalize_tool_call(tc: Any) -> ChatCompletionMessageToolCall: + if isinstance(tc, ChatCompletionMessageToolCall): + return tc + if isinstance(tc, dict): + func = tc.get("function") or _EMPTY_MAPPING + return ChatCompletionMessageToolCall( + id=tc.get("id", ""), + type=tc.get("type", "function"), + function=Function( + name=func.get("name", ""), + arguments=func.get("arguments", ""), + ), + ) + if hasattr(tc, "id") and hasattr(tc, "function"): + return ChatCompletionMessageToolCall( + id=tc.id or "", + type=getattr(tc, "type", None) or "function", + function=tc.function, + ) + raise TypeError(f"Cannot normalize tool_call of type {type(tc).__name__}: {tc!r}") + + @staticmethod + def _join_texts(texts: Any) -> str: + """Join response text segments into the single content string the + webhook evaluates. Empty when there is no assistant text.""" + if not texts: + return "" + return "\n".join(t for t in texts if t) + + @staticmethod + def _build_response_moderation_payload( + tool_calls: Sequence[ChatCompletionMessageToolCall], + content: str, request_id: str | None, - ) -> dict[str, Any]: - """Build a full OpenAI ChatCompletion-format dict for the blocking service.""" + ) -> Mapping[str, Any]: + """Build an OpenAI ChatCompletion-format dict (assistant text + tool + calls) for the after_completion webhook. + + ``content`` is sent so the webhook can moderate the response text; + ``None`` when the assistant produced no text (tool-call-only response). + """ + message: dict[str, Any] = { + "role": "assistant", + "content": content or None, + } + if tool_calls: + message["tool_calls"] = tuple(tc.model_dump(exclude_none=True) for tc in tool_calls) return { "id": request_id or f"chatcmpl-{uuid.uuid4()}", "object": "chat.completion", @@ -290,42 +455,133 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): "choices": [ { "index": 0, - "message": { - "role": "assistant", - "content": None, - "tool_calls": [tc.model_dump(exclude_none=True) for tc in tool_calls], - }, - "finish_reason": "tool_calls", + "message": message, + "finish_reason": "tool_calls" if tool_calls else "stop", } ], } @staticmethod - def _extract_request_data(call_details: dict[str, Any]) -> dict[str, Any]: - """Extract original request data from model_call_details.""" - if not call_details: - return {} - litellm_params = call_details.get("litellm_params", {}) or {} + def _flatten_messages_for_moderation(messages: Any) -> tuple[Mapping[str, Any], ...]: + """Collapse each message's content to a plain string for the webhook. + + litellm normalizes Anthropic ``/v1/messages`` requests to OpenAI shape, + but a turn sent as content-parts (``[{"type": "text", ...}]``) stays a + list. The before_prompt webhook reads ``content`` as a string and drops + non-string content, so we flatten text parts here (images skipped, per + ``convert_content_list_to_str``) -- otherwise block-content prompts + would pass through unmoderated. Builds a new list; never mutates the + shared ``structured_messages``. + """ + return tuple( + { + "role": message.get("role"), + "content": "\n".join(p for p in RubrikLogger._moderation_text_parts(message) if p), + } + for message in messages or () + if isinstance(message, dict) + ) + + @staticmethod + def _moderation_text_parts(message: Mapping[str, Any]) -> tuple[str, ...]: + """Every attacker-controlled text segment of a message: its content plus + the arguments of any tool call or deprecated function call.""" + fc = message.get("function_call") + return ( + # Base text content (flattens Anthropic content-part arrays) + convert_content_list_to_str(message), # pyright: ignore[reportArgumentType] # dict[str,Any] is AllMessageValues at runtime + *( + str((tc.get("function") or _EMPTY_MAPPING).get("arguments") or "") + for tc in message.get("tool_calls") or () + if isinstance(tc, dict) + ), + str((fc.get("arguments") if isinstance(fc, dict) else None) or ""), + ) + + @staticmethod + def _build_prompt_moderation_payload( + inputs: GenericGuardrailAPIInputs, + request_data: Mapping[str, Any], + ) -> Mapping[str, Any]: + """Build the bare OpenAI request the before_prompt webhook consumes. + + Unlike the after_completion envelope, this endpoint takes a raw OpenAI + chat-completions request. ``structured_messages`` is litellm's + OpenAI-normalized view of the prompt, so this works for Anthropic + ``/v1/messages`` requests too. Optional fields are sent only when + present so the payload stays clean. + """ + payload: dict[str, Any] = { + "model": inputs.get("model") or request_data.get("model") or "", + "messages": RubrikLogger._flatten_messages_for_moderation(inputs.get("structured_messages")), + } + tools = inputs.get("tools") + if tools is not None: + payload["tools"] = tools + user = request_data.get("user") + if user: + payload["user"] = user + # Fall back to litellm_call_id, the stable cross-provider join key the + # response/tool path uses (see _correlation_id). LiteLLM does not + # populate request_data["correlation_key"]; it carries litellm_call_id. + # The before_prompt webhook skips the *_prompt_moderation.json S3 write + # when correlation_key is empty, so without this the block fires but no + # log is ever written. An explicit correlation_key still wins. + correlation_key = request_data.get("correlation_key") or request_data.get("litellm_call_id") + if correlation_key: + payload["correlation_key"] = correlation_key + return payload + + @staticmethod + def _extract_request_data( + call_details: Mapping[str, Any], + request_data: Mapping[str, Any] | None, + ) -> Mapping[str, Any]: + """Extract original request data from model_call_details for the + response moderation service envelope. + + Includes the agent's declared ``tools`` (OpenAI-format) when available + so the webhook's hallucination evaluator can compare returned tool calls + against the declared tool list. + """ + if not call_details and not request_data: + return _EMPTY_MAPPING + call_details = call_details or _EMPTY_MAPPING + request_data = request_data or _EMPTY_MAPPING + optional_params = call_details.get("optional_params") or _EMPTY_MAPPING + + # Use ``in`` rather than truthy ``or`` so an explicit empty list + # (caller declared the agent has NO tools) is forwarded as-is. + # The response moderation service uses that signal to flag tool-call + # hallucinations -- ``or`` would mask it by falling through to + # optional_params. + if "tools" in request_data: + tools = request_data["tools"] + else: + tools = optional_params.get("tools") + + # The response moderation service consumes only messages/model/tools. + # Don't forward proxy_server_request -- in litellm >=1.83 its ``body`` + # snapshot carries a UserAPIKeyAuth instance that breaks json.dumps, + # silently fail-opening the guardrail. return { "messages": call_details.get("messages"), "model": call_details.get("model"), - "proxy_server_request": RubrikLogger._sanitize_proxy_server_request( - litellm_params.get("proxy_server_request") - ), + "tools": tools, } @staticmethod def _sanitize_proxy_server_request(proxy_server_request: Any) -> Any: """Allowlist only routing fields (``url``, ``method``) when forwarding - ``proxy_server_request`` to the external Rubrik webhook, dropping - inbound ``headers`` (Authorization, Cookie, x-api-key, ...) and the raw + ``proxy_server_request`` to an external webhook, dropping inbound + ``headers`` (Authorization, Cookie, x-api-key, ...) and the raw request ``body`` so proxy credentials are not exfiltrated.""" if not isinstance(proxy_server_request, dict): return proxy_server_request return {key: proxy_server_request[key] for key in ("url", "method") if key in proxy_server_request} @staticmethod - def _resolve_model(request_data: dict[str, Any], call_details: dict[str, Any]) -> str: + def _resolve_model(request_data: Mapping[str, Any], call_details: Mapping[str, Any]) -> str: """Get the model name for the ModifyResponseException.""" response = request_data.get("response") if response and hasattr(response, "model"): @@ -334,8 +590,70 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): # -- Logging hooks --------------------------------------------------------- - async def _prepare_log_payload(self, kwargs: dict, event_type: str) -> StandardLoggingPayload | None: - """Shared logic for success and failure logging.""" + @staticmethod + def _correlation_id(call_details: Mapping[str, Any], request_data: Mapping[str, Any] | None = None) -> str | None: + """The id that joins a blocked request's two S3 logs by filename: the + moderation (``_blocking``) log and the failure (response) log. + + Always ``litellm_call_id``. It is assigned at request start and is + present identically in both the guardrail path (``model_call_details`` + / ``request_data``) and the failure-hook path. Unlike ``response.id`` + or ``standard_logging_object["id"]`` it is immune to the race where a + block fires before the response/logging object is populated, so the + two logs correlate for every provider (OpenAI and Anthropic alike). + """ + return call_details.get("litellm_call_id") or (request_data or _EMPTY_MAPPING).get("litellm_call_id") + + @classmethod + def _apply_correlation_id(cls, payload: dict[str, Any], source: Mapping[str, Any]) -> None: + """Pin ``payload["id"]`` to ``litellm_call_id`` in place so this log + shares its S3 filename id with the moderation (``_blocking``) and + failure logs for the same request -- for every provider. + + ``standard_logging_object["id"]`` is the provider response id + (``response_obj.get("id", litellm_call_id)``), a ``chatcmpl-*`` value + for OpenAI, which would not correlate. ``litellm_call_id`` is assigned + at request start and is identical across all log paths. Falls back to + the existing id when ``litellm_call_id`` is somehow absent rather than + writing a null filename key. + + ``source`` may be ``model_call_details`` directly or a ``kwargs`` dict + that aliases it -- same shape either way. + """ + correlated = cls._correlation_id(source) + if correlated: + payload["id"] = correlated + + @staticmethod + def _prepend_system_prompt(payload: dict[str, Any], source: Mapping[str, Any]) -> None: + """Prepend ``source["system"]`` onto ``payload["messages"]``. + + Builds a NEW messages list rather than mutating ``payload["messages"]`` + in place. The fallback branch of ``_prepare_block_failure_payload`` + aliases ``call_details["messages"]`` directly, so an in-place + ``list.insert(0, ...)`` would mutate the shared source dict. + + No-op if no system prompt is present. Tolerates list/dict/str + message shapes; on unexpected shape, leaves payload alone. + """ + system_prompt = source.get("system") + if not system_prompt: + return + try: + system_scaffold = {"role": "system", "content": system_prompt} + messages = payload.get("messages") + if isinstance(messages, list): + payload["messages"] = (system_scaffold, *messages) + elif isinstance(messages, (dict, str)): + payload["messages"] = (system_scaffold, messages) + except Exception as e: + verbose_logger.warning( + f"Rubrik: failed to prepend system prompt: {e}", + exc_info=True, + ) + + async def _prepare_log_payload(self, kwargs: Mapping[str, Any], event_type: str) -> StandardLoggingPayload | None: + """Shared logic for success logging (sampled).""" if random.random() > self.sampling_rate: verbose_logger.debug(f"Skipping Rubrik {event_type} logging (sampling_rate={self.sampling_rate})") return None @@ -343,59 +661,17 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): # Deep-copy so mutations don't affect other callbacks sharing this object standard_logging_payload: StandardLoggingPayload = safe_deep_copy(kwargs["standard_logging_object"]) - # For Anthropic /v1/messages requests, LiteLLM creates a separate - # ModelResponse (with a generated chatcmpl-* id) for logging, which - # differs from the original Anthropic msg-* id on the response dict. - # Normalize to litellm_call_id so that the logging and tool-blocking - # endpoints see the same request identifier. - litellm_params = kwargs.get("litellm_params", {}) or {} - proxy_request = litellm_params.get("proxy_server_request", {}) or {} - url_path = urllib.parse.urlparse(proxy_request.get("url", "")).path - if url_path.endswith(_ENDPOINT_ANTHROPIC_MESSAGES): - _litellm_call_id = kwargs.get("litellm_call_id") - if _litellm_call_id: - standard_logging_payload["id"] = _litellm_call_id # type: ignore[literal-required] - - if "system" in kwargs: - system_prompt_msg_list = kwargs["system"] - try: - if system_prompt_msg_list: - system_scaffold = { - "role": "system", - "content": system_prompt_msg_list, - } - if isinstance(standard_logging_payload["messages"], list): - standard_logging_payload["messages"].insert(0, system_scaffold) - elif isinstance(standard_logging_payload["messages"], (dict, str)): - standard_logging_payload["messages"] = [ - system_scaffold, - standard_logging_payload["messages"], - ] - except Exception as e: - verbose_logger.warning( - f"Rubrik: failed to prepend system prompt: {e}", - exc_info=True, - ) + self._apply_correlation_id(standard_logging_payload, kwargs) # pyright: ignore[reportArgumentType] # StandardLoggingPayload is dict[str,Any] at runtime + self._prepend_system_prompt(standard_logging_payload, kwargs) # pyright: ignore[reportArgumentType] # StandardLoggingPayload is dict[str,Any] at runtime return standard_logging_payload - async def _enqueue_log_event(self, kwargs: dict, event_type: str): - try: - self._ensure_periodic_flush_task() - payload = await self._prepare_log_payload(kwargs, event_type) - if payload is None: - return - - self.log_queue.append(payload) - self._enforce_max_queue_size() - - if len(self.log_queue) >= self.batch_size: - await self.flush_queue() - except Exception as e: - verbose_logger.error( - f"Rubrik {event_type} logging hook failed: {e}. Skipping logging for this event.", - exc_info=True, - ) + async def _append_and_maybe_flush(self, payload) -> None: + self._ensure_periodic_flush_task() + self.log_queue.append(payload) + self._enforce_max_queue_size() + if len(self.log_queue) >= self.batch_size: + await self.flush_queue() def _enforce_max_queue_size(self) -> None: overflow = len(self.log_queue) - self.max_queue_size @@ -415,18 +691,213 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): self._dropped_since_warning = 0 self._last_drop_warning_time = now + async def _enqueue_log_event(self, kwargs: Mapping[str, Any], event_type: str): + try: + payload = await self._prepare_log_payload(kwargs, event_type) + if payload is None: + return + await self._append_and_maybe_flush(payload) + except Exception as e: + verbose_logger.error( + f"Rubrik {event_type} logging hook failed: {e}. Skipping logging for this event.", + exc_info=True, + ) + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + # Blocked requests are logged via async_post_call_failure_hook; + # skip here to avoid double-logging the pre-block response. + if kwargs.get("_rubrik_blocked"): + verbose_logger.debug( + f"Rubrik: skipping success event for blocked request litellm_call_id={kwargs.get('litellm_call_id')}" + ) + return await self._enqueue_log_event(kwargs, "success") async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + # Log regular LLM failures (timeouts, upstream errors, etc.) to Rubrik. + # NOTE: ``ModifyResponseException`` blocks are NOT routed here; they + # bypass ``Logging.async_failure_handler`` entirely and reach + # ``async_post_call_failure_hook`` instead. So there is no risk of + # double-logging a block through this path. await self._enqueue_log_event(kwargs, "failure") + async def async_post_call_failure_hook( + self, + request_data: dict, + original_exception: Exception, + user_api_key_dict: Any, + traceback_str: str | None = None, + ) -> None: + """Log blocked requests signalled via ``ModifyResponseException`` + (prompt blocks, response/tool blocks, streaming blocks). + + Carries the stashed ``_rubrik_logging_obj``. For every other + exception we no-op; LiteLLM's standard failure plumbing handles those. + """ + if not isinstance(original_exception, ModifyResponseException): + return + + # Guard by guardrail_name so that when multiple Rubrik instances are + # registered, only the instance that raised the block handles it. + # The failure hook is called for every registered callback; without + # this check the first instance pops the stash and the originating + # instance finds None and silently skips logging. + if getattr(original_exception, "guardrail_name", None) != self.guardrail_name: + return + + logging_obj = request_data.pop("_rubrik_logging_obj", None) + if logging_obj is None: + # Legitimate when a non-Rubrik guardrail raised the block; + # problematic if Rubrik did and the stash was lost (e.g. + # ``_stash_block_context`` ran with ``logging_obj=None``). Either + # way we cannot build the payload. + verbose_logger.warning( + "Rubrik: block exception without stashed logging_obj. " + f"litellm_call_id={request_data.get('litellm_call_id')}, " + f"model={request_data.get('model')}, " + f"user_id={getattr(user_api_key_dict, 'user_id', None)}, " + f"raising_guardrail=" + f"{getattr(original_exception, 'guardrail_name', None)}" + ) + return + + call_id: str | None = None + await self._build_and_enqueue_block_event(logging_obj, original_exception, call_id) + + async def _build_and_enqueue_block_event( + self, + logging_obj: "LiteLLMLoggingObj", + exception: "ModifyResponseException", + call_id: str | None, + ) -> None: + try: + call_details = logging_obj.model_call_details + # Do NOT pop "_rubrik_blocked" here. The deferred success-handler + # task may still be iterating callbacks, and popping mid-iteration + # (between two awaited callback invocations) would cause this + # plugin's success-event callback to read the flag as absent and + # log the pre-block response -- the exact bug this hook exists to + # prevent. The flag dies with model_call_details when the request + # completes; there's nothing to clean up. + call_id = call_details.get("litellm_call_id") + payload = self._prepare_block_failure_payload(logging_obj, exception) + except (AttributeError, KeyError, TypeError) as e: + verbose_logger.error( + f"Rubrik: failed to build blocked-tool payload for " + f"litellm_call_id={call_id}: {e}. Event will NOT be logged.", + exc_info=True, + ) + return + + try: + await self._append_and_maybe_flush(payload) + except Exception as e: + verbose_logger.error( + f"Rubrik: failed to enqueue blocked-tool event for litellm_call_id={call_id}: {e}.", + exc_info=True, + ) + + def _prepare_block_failure_payload( + self, + logging_obj: "LiteLLMLoggingObj", + exception: "ModifyResponseException", + ) -> StandardLoggingPayload: + """Build a failure-style payload using the exception text as response. + + Blocked-tool events are security-relevant and **bypass sampling**: + every block is logged. + + The deferred success-handler runs as a separately-scheduled task and + races with this hook, so ``standard_logging_object`` on + ``model_call_details`` may not yet be populated. If present we reuse + it; otherwise we fall back to a best-effort payload built from the + fields available at block time. + + For prompt blocks the LLM is never called, so ``standard_logging_object`` + is never populated. The fallback therefore must carry enough fields to + pass the log processor's ``LogEntry`` schema (``BaseLogEntry`` requires + ``metadata``, ``model_id``, ``model_group``, ``model_parameters``, + ``startTime``, ``endTime``, and ``completionStartTime``). Without a + parseable payload the log processor discards the entry with a parse + error and no session is created, so prompt-moderation violations are + silently dropped even though the ``_prompt_moderation.json`` forensic + log is written correctly. + + Field sourcing for the fallback path: + - ``model`` / ``model_group``: ``call_details["model"]`` -- this is the + model-group name (e.g. "gpt-4o") set by the proxy before the guardrail + fires. The router writes ``metadata["model_group"]`` only inside + ``acompletion()``, which hasn't run yet for a prompt block. + - ``model_id``: not available before the LLM returns hidden_params; + defaults to empty string. + - ``user_api_key_hash``: ``call_details["metadata"]["user_api_key"]`` -- + the hashed token written by ``add_user_information_to_request_data`` + before ``pre_call_hook`` fires. + - time fields: ``call_details["start_time"]`` reused for all three; + end/completion times are meaningless for a prompt block. + """ + call_details = logging_obj.model_call_details + exception_text = f"{type(exception).__name__}: {exception.message}" + + base = call_details.get("standard_logging_object") + if base is not None: + payload: dict = safe_deep_copy(base) + else: + verbose_logger.debug( + "Rubrik: standard_logging_object not yet on model_call_details " + f"for litellm_call_id={call_details.get('litellm_call_id')}; " + "using best-effort fallback payload." + ) + payload = self._build_fallback_payload(call_details) + + payload["response"] = exception_text + + # Pin the correlation key to litellm_call_id so this failure log shares + # its S3 filename id with the moderation (``_blocking``) log for the + # same request. The copied ``standard_logging_object["id"]`` is + # ``response_obj.get("id", litellm_call_id)`` -- a provider ``chatcmpl-*`` + # value for OpenAI -- which would not correlate; overwrite it. + payload["id"] = self._correlation_id(call_details) or f"chatcmpl-{uuid.uuid4()}" + self._prepend_system_prompt(payload, call_details) + + return payload # type: ignore[return-value] + + @staticmethod + def _build_fallback_payload(call_details: Mapping[str, Any]) -> dict[str, Any]: + _metadata: Mapping[str, Any] = call_details.get("metadata") or _EMPTY_MAPPING + # Convert datetime to a Unix float so json.dumps can serialize it. + # httpx's json= parameter uses stdlib json.dumps with no custom encoder. + _raw_start = call_details.get("start_time") + _start = _raw_start.timestamp() if _raw_start is not None else None + return { + "id": call_details.get("litellm_call_id"), + "model": call_details.get("model") or "", + # model_group is set by the router inside acompletion(), which + # hasn't run for a prompt block; use the model name instead. + "model_group": call_details.get("model") or "", + # model_id comes from response.hidden_params -- unavailable here. + "model_id": "", + "model_parameters": ModelParamHelper.get_standard_logging_model_parameters( + call_details.get("optional_params") or _EMPTY_MAPPING # pyright: ignore[reportArgumentType] # helper only reads the mapping + ), + "startTime": _start, + "endTime": _start, + "completionStartTime": _start, + "messages": call_details.get("messages") or (), + "metadata": { + # "user_api_key" is the hashed token written by + # add_user_information_to_request_data before guardrails fire. + "user_api_key_hash": _metadata.get("user_api_key_hash") or _metadata.get("user_api_key") or "", + }, + "status": "failure", + } + # -- Batch logging --------------------------------------------------------- async def _log_batch_to_rubrik(self, data): - # NOTE: this method intentionally re-raises on failure so the parent - # CustomBatchLogger.flush_queue keeps the unsent events in the queue - # for the next flush attempt instead of silently dropping them. + # NOTE: this method intentionally re-raises on failure so flush_queue + # can preserve the unsent events for the next flush attempt instead of + # silently dropping them. try: response = await self.async_httpx_client.post( url=self.logging_endpoint, @@ -452,10 +923,8 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): if not self.log_queue: return - log_queue_snapshot = list(self.log_queue) - verbose_logger.debug("Rubrik: Flushing batch of %s events", len(log_queue_snapshot)) await self._log_batch_to_rubrik( - data=log_queue_snapshot, + data=self.log_queue, ) async def flush_queue(self): @@ -463,8 +932,8 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): Overrides the base implementation so the same snapshot drives both the HTTP send and the queue truncation. This avoids the subtle - coupling where the base class captures `len(self.log_queue)` - separately from the snapshot taken inside `async_send_batch`, + coupling where the base class captures ``len(self.log_queue)`` + separately from the snapshot taken inside ``async_send_batch``, which could otherwise drift in a future refactor and cause duplicate deliveries to Rubrik. """ @@ -485,70 +954,141 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): del self.log_queue[: len(snapshot)] self.last_flush_time = time.time() - # -- Tool blocking service ------------------------------------------------- + # -- Webhook services ------------------------------------------------------ - async def _post_to_tool_blocking_service( + async def _post_json(self, endpoint: str, payload: Mapping[str, Any], service_name: str) -> Mapping[str, Any]: + """POST ``payload`` to a Rubrik webhook and return its dict response. + + Raises: + Exception: If the service is unavailable or returns an error. + TypeError: If the response JSON is not a dict. + """ + verbose_logger.debug(f"Sending request to {service_name}: {endpoint}") + http_response = await self.moderation_client.post( + endpoint, + json=payload, + headers=self._headers, + ) + http_response.raise_for_status() + result = http_response.json() + if not isinstance(result, dict): + raise TypeError( + f"{service_name} returned non-dict JSON " + f"({type(result).__name__}); expected OpenAI chat completion " + "shape or empty object." + ) + return result + + async def _post_to_response_moderation_endpoint( self, - response_data: dict[str, Any], - request_data: dict[str, Any], - ) -> dict[str, Any]: - """Post a payload to the tool blocking service and return the response. + response_data: Mapping[str, Any], + request_data: Mapping[str, Any], + ) -> Mapping[str, Any]: + """Post the ``{request, response}`` envelope to the after_completion + webhook and return its (possibly rewritten) response. Args: response_data: The OpenAI-formatted response payload to send. request_data: Original LLM request data to include alongside the response for additional context. Empty dict if unavailable. - - Raises: - Exception: If the service is unavailable or returns an error. """ - envelope = { - "request": request_data, - "response": response_data, - } - verbose_logger.debug(f"Sending request to tool blocking service: {self.tool_blocking_endpoint}") - http_response = await self.tool_blocking_client.post( - self.tool_blocking_endpoint, - json=envelope, - headers=self._headers, + envelope = {"request": request_data, "response": response_data} + return await self._post_json( + self.response_moderation_endpoint, + envelope, + "Response moderation service", ) - http_response.raise_for_status() - result: dict[str, Any] = http_response.json() - return result + + async def _post_to_prompt_moderation_endpoint(self, payload: Mapping[str, Any]) -> Mapping[str, Any]: + """Post a bare OpenAI request to the before_prompt webhook. + + Returns ``{}`` (passthrough) or a synthetic chat.completion (block). + """ + return await self._post_json(self.prompt_moderation_endpoint, payload, "Prompt moderation service") @staticmethod - def _extract_blocked_tools( - service_response: dict[str, Any], - all_tool_calls: list[ChatCompletionMessageToolCall], - ) -> str | None: - """Return the blocking explanation if any tool calls were blocked. + def _extract_prompt_refusal(service_response: Mapping[str, Any]) -> str | None: + """Return the refusal text when the prompt was blocked, else None. - Compares the service response (which contains only allowed tools) against - the full set of tool calls. Returns ``None`` if all tools are allowed, or - the explanation string (prefixed with newlines) otherwise. + The before_prompt webhook returns ``{}`` (passthrough) or a synthetic + chat.completion whose ``choices[0].message.content`` is the refusal + explanation. + """ + choices = service_response.get("choices") + if not choices: + return None + message = choices[0].get("message") or _EMPTY_MAPPING + content = message.get("content") + return content or "Request blocked by policy." + + @staticmethod + def _extract_response_block( + service_response: Mapping[str, Any], + all_tool_calls: Sequence[ChatCompletionMessageToolCall], + sent_content: str, + ) -> BlockedResponseResult | None: + """Detect whether the webhook moderated the response text or tool calls. + + The after_completion webhook rewrites the response in place with no + explicit "blocked" flag, so we infer a block by diffing what we sent + against what came back: + + - Tool block: a tool call we sent is absent from the returned (allowed) + set. + - Text block: the returned content was REPLACED wholesale (a text + violation), as opposed to having a tool-block explanation APPENDED to + the original content. We tell them apart with ``startswith``, which + mirrors the webhook's own append-vs-replace behavior. + + Returns None when nothing was moderated. A text block supersedes a tool + block (mirroring the webhook, which drops tool calls on a text block). Expects service_response in OpenAI chat completion format: {"choices": [{"message": {"tool_calls": [...], "content": "..."}}]} """ - choices = service_response.get("choices", []) + choices = service_response.get("choices") or () if not choices: - raise _MalformedToolBlockingResponseError("Tool blocking service returned empty response") + raise _MalformedToolBlockingResponseError("Response moderation service returned empty response") - message = choices[0].get("message", {}) - returned_tool_calls = message.get("tool_calls") or [] - blocking_explanation = message.get("content", "") + message = choices[0].get("message") or _EMPTY_MAPPING + returned_tool_calls = message.get("tool_calls") or () + returned_content = message.get("content") or "" - allowed_id_counts: Counter = Counter( - tc["id"] for tc in returned_tool_calls if isinstance(tc, dict) and tc.get("id") - ) - required_id_counts: Counter = Counter(tc.id for tc in all_tool_calls if tc.id) - - all_allowed = len(returned_tool_calls) >= len(all_tool_calls) and all( - allowed_id_counts.get(tc_id, 0) >= count for tc_id, count in required_id_counts.items() + # Use Counter so duplicate IDs are handled correctly: if the model + # emits two calls with the same ID (one allowed, one prohibited) and + # the service returns only the allowed one, a set-based check would + # miss the block. Counter preserves multiplicity. + returned_id_counts: Counter[str] = Counter(tc["id"] for tc in returned_tool_calls if tc.get("id")) + required_id_counts: Counter[str] = Counter(tc.id for tc in all_tool_calls if tc.id) + # Cardinality check catches ID-less tool calls (not counted in + # required_id_counts because tc.id is falsy); Counter check catches + # duplicate-ID attacks where one occurrence is silently removed. + tools_blocked = len(returned_tool_calls) < len(all_tool_calls) or not all( + returned_id_counts.get(tc_id, 0) >= count for tc_id, count in required_id_counts.items() ) - if all_allowed: - return None + # The webhook either replaces content wholesale (text block) or appends + # a tool-block explanation to the original text. ``appended`` tells the + # two apart, and is reused below to recover just the explanation. A text + # block requires there to have been assistant text to block. + # Use the documented ``\n\n`` separator to distinguish a tool-block + # append from a text replacement that shares the original as a prefix. + # Without the separator, a replacement like "Hello, blocked." where the + # original was "Hello" would be classified as an append (not a text + # block) and silently pass through to the client. + appended = bool(sent_content) and returned_content.startswith(f"{sent_content}\n\n") + text_blocked = bool(sent_content) and returned_content != sent_content and not appended - explanation = blocking_explanation or "Tool call blocked by policy." - return f"\n\n{explanation}" + if text_blocked: + return BlockedResponseResult(explanation=returned_content or "Response blocked by policy.") + + if tools_blocked: + if appended: + # Recover just the appended explanation: drop the original text + # and the leading separator the webhook inserted before it. + explanation = returned_content[len(sent_content) :].lstrip("\n") + else: + explanation = returned_content + return BlockedResponseResult(explanation=explanation or "Tool call blocked by policy.") + + return None diff --git a/litellm/proxy/guardrails/guardrail_hooks/rubrik/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/rubrik/__init__.py index 4ad29bbeae8..2f2228ae312 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/rubrik/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/rubrik/__init__.py @@ -10,6 +10,18 @@ if TYPE_CHECKING: def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail") -> RubrikLogger: + """Create and register a RubrikLogger instance. + + The ``mode`` field in the guardrail config controls which surfaces are + moderated: + - ``pre_call`` (or a mode that includes it): prompt moderation via the + ``/v1/before_prompt/openai/v1`` webhook. + - ``post_call`` (the default when ``mode`` is omitted): response and tool + call moderation via the ``/v1/after_completion/openai/v1`` webhook. + + Both hooks are active when ``mode`` covers both ``pre_call`` and + ``post_call``. + """ import litellm rubrik_callback = RubrikLogger( diff --git a/tests/test_litellm/integrations/test_rubrik.py b/tests/test_litellm/integrations/test_rubrik.py index 922d2fe8a15..7f589dc15bf 100644 --- a/tests/test_litellm/integrations/test_rubrik.py +++ b/tests/test_litellm/integrations/test_rubrik.py @@ -1,8 +1,8 @@ """ Tests for the Rubrik LiteLLM plugin. -Covers initialization, apply_guardrail tool blocking (all allowed, all blocked, -partial blocking, fail-open), batch logging, and Anthropic format handling. +Covers initialization, apply_guardrail (prompt moderation + response/tool +blocking), batch logging, and Anthropic format handling. """ import os @@ -13,8 +13,10 @@ import httpx import pytest from litellm.integrations.custom_guardrail import ModifyResponseException -from litellm.integrations.rubrik import RubrikLogger -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.integrations.rubrik import ( + RubrikLogger, + _MalformedToolBlockingResponseError, +) from tests.test_litellm.integrations.rubrik_test_helpers import ( make_inputs_with_tools, @@ -50,19 +52,19 @@ class TestInitialization: with patch("asyncio.create_task", Mock()): handler = RubrikLogger() assert ( - handler.tool_blocking_endpoint + handler.response_moderation_endpoint == "http://localhost:8080/v1/after_completion/openai/v1" ) assert handler.logging_endpoint == "http://localhost:8080/v1/litellm/batch" assert handler.key == "test-api-key" - assert isinstance(handler.tool_blocking_client, AsyncHTTPHandler) + assert handler.moderation_client is not None def test_init_with_constructor_params(self): with patch("asyncio.create_task", Mock()): handler = RubrikLogger(api_key="ctor-key", api_base="http://ctor-host:9090") assert handler.key == "ctor-key" assert ( - handler.tool_blocking_endpoint + handler.response_moderation_endpoint == "http://ctor-host:9090/v1/after_completion/openai/v1" ) @@ -82,7 +84,7 @@ class TestInitialization: with patch.dict(os.environ, {"RUBRIK_WEBHOOK_URL": "http://localhost:8080/"}): with patch("asyncio.create_task", Mock()): assert ( - RubrikLogger().tool_blocking_endpoint + RubrikLogger().response_moderation_endpoint == "http://localhost:8080/v1/after_completion/openai/v1" ) @@ -90,13 +92,13 @@ class TestInitialization: with patch("asyncio.create_task", Mock()): with patch.dict(os.environ, {"RUBRIK_WEBHOOK_URL": "http://host/v1"}): assert ( - RubrikLogger().tool_blocking_endpoint + RubrikLogger().response_moderation_endpoint == "http://host/v1/after_completion/openai/v1" ) with patch.dict(os.environ, {"RUBRIK_WEBHOOK_URL": "http://host/v11"}): assert ( - RubrikLogger().tool_blocking_endpoint + RubrikLogger().response_moderation_endpoint == "http://host/v11/v1/after_completion/openai/v1" ) @@ -155,10 +157,10 @@ class TestInitialization: # Do NOT patch asyncio.create_task — the real call should be # guarded and fall back gracefully when there is no event loop. handler = RubrikLogger() - assert handler.tool_blocking_endpoint.startswith("http://localhost:8080") + assert handler.response_moderation_endpoint.startswith("http://localhost:8080") # Without a running loop at init, the periodic flush task should be # deferred so batches still get drained once a log event arrives. - assert handler._flush_task is None + assert handler._periodic_flush_task is None @pytest.mark.asyncio async def test_periodic_flush_task_started_lazily_on_first_log(self, mock_env): @@ -170,7 +172,7 @@ class TestInitialization: side_effect=RuntimeError("no running loop"), ): handler = RubrikLogger() - assert handler._flush_task is None + assert handler._periodic_flush_task is None kwargs = { "standard_logging_object": { @@ -183,8 +185,8 @@ class TestInitialization: with patch.object(handler, "_log_batch_to_rubrik", AsyncMock()): await handler.async_log_success_event(kwargs, None, None, None) - assert handler._flush_task is not None - handler._flush_task.cancel() + assert handler._periodic_flush_task is not None + handler._periodic_flush_task.cancel() def test_event_hook_defaults_to_post_call_when_none_passed(self, mock_env): """`initialize_guardrail` always passes ``event_hook=litellm_params.mode`` @@ -204,14 +206,13 @@ class TestInitialization: handler = RubrikLogger(event_hook=GuardrailEventHooks.pre_call) assert handler.event_hook == GuardrailEventHooks.pre_call - def test_default_on_defaults_to_true_when_none_passed(self, mock_env): - """`initialize_guardrail` always passes ``default_on=litellm_params.default_on`` - (which is ``None`` when the user omits ``default_on``). The logger must - coerce a None ``default_on`` to True, otherwise ``should_run_guardrail`` - (which checks ``self.default_on is True``) silently skips the guardrail.""" + def test_default_on_defaults_to_false_when_none_passed(self, mock_env): + """Follows the standard litellm pattern: omitted ``default_on`` resolves + to ``False`` (off by default). Users must explicitly set + ``default_on: true`` to enable the guardrail for all requests.""" with patch("asyncio.create_task", Mock()): handler = RubrikLogger(default_on=None) - assert handler.default_on is True + assert handler.default_on is False def test_explicit_default_on_false_preserved(self, mock_env): """A user explicitly setting ``default_on: false`` in their guardrail @@ -421,7 +422,7 @@ class TestBatchLogging: ) assert len(handler.log_queue) == 1 msgs = handler.log_queue[0]["messages"] - assert isinstance(msgs, list) + assert isinstance(msgs, tuple) assert msgs[0]["role"] == "system" assert msgs[1] == {"role": "user", "content": "hi"} @@ -444,7 +445,10 @@ class TestBatchLogging: ) assert handler.log_queue[0]["id"] == "litellm-call-123" - async def test_non_anthropic_id_unchanged(self, handler): + async def test_litellm_call_id_always_used_as_correlation_key(self, handler): + """The merged plugin always uses litellm_call_id as the log ID for all + providers (not just Anthropic) so that logs correlate with the + moderation (_blocking) and failure logs for the same request.""" kwargs = { "standard_logging_object": { "id": "chatcmpl-original", @@ -461,7 +465,7 @@ class TestBatchLogging: await handler.async_log_success_event( kwargs=kwargs, response_obj=None, start_time=None, end_time=None ) - assert handler.log_queue[0]["id"] == "chatcmpl-original" + assert handler.log_queue[0]["id"] == "litellm-call-123" async def test_payload_deep_copied_not_mutated(self, handler): """Verify the shared standard_logging_object is not mutated.""" @@ -536,7 +540,7 @@ class TestApplyGuardrail: tc2 = make_tool_call_dict("call_2", "get_time") inputs = make_inputs_with_tools([tc1, tc2]) - handler.tool_blocking_client = _echo_service() + handler.moderation_client = _echo_service() result = await handler.apply_guardrail( inputs=inputs, request_data={}, input_type="response" @@ -548,7 +552,7 @@ class TestApplyGuardrail: tc2 = make_tool_call_dict("call_2", "drop_database") inputs = make_inputs_with_tools([tc1, tc2]) - handler.tool_blocking_client = _mock_service_response( + handler.moderation_client = _mock_service_response( { "choices": [ { @@ -594,7 +598,7 @@ class TestApplyGuardrail: mock_client = AsyncMock() mock_client.post = mock_post - handler.tool_blocking_client = mock_client + handler.moderation_client = mock_client with pytest.raises(ModifyResponseException): await handler.apply_guardrail( @@ -607,7 +611,7 @@ class TestApplyGuardrail: mock_client = AsyncMock() mock_client.post = AsyncMock(side_effect=httpx.TimeoutException("Timeout")) - handler.tool_blocking_client = mock_client + handler.moderation_client = mock_client result = await handler.apply_guardrail( inputs=inputs, request_data={}, input_type="response" @@ -618,7 +622,7 @@ class TestApplyGuardrail: tc1 = make_tool_call_dict("call_1", "test_tool") inputs = make_inputs_with_tools([tc1]) - handler.tool_blocking_client = _mock_service_response({"choices": []}) + handler.moderation_client = _mock_service_response({"choices": []}) result = await handler.apply_guardrail( inputs=inputs, request_data={}, input_type="response" @@ -641,7 +645,7 @@ class TestApplyGuardrail: mock_client = AsyncMock() mock_client.post = mock_post - handler.tool_blocking_client = mock_client + handler.moderation_client = mock_client await handler.apply_guardrail( inputs=inputs, request_data={}, input_type="response" @@ -675,7 +679,7 @@ class TestApplyGuardrail: mock_client = AsyncMock() mock_client.post = mock_post - handler.tool_blocking_client = mock_client + handler.moderation_client = mock_client logging_obj = Mock() logging_obj.model_call_details = { @@ -697,7 +701,10 @@ class TestApplyGuardrail: assert req["model"] == "gpt-4" assert req["messages"] == [{"role": "user", "content": "hi"}] - async def test_proxy_server_request_headers_stripped(self, handler): + async def test_proxy_server_request_not_forwarded(self, handler): + """proxy_server_request is intentionally NOT included in the request + envelope: in litellm >=1.83 its ``body`` carries a UserAPIKeyAuth + instance that breaks json.dumps, silently fail-opening the guardrail.""" tc = make_tool_call_dict("call_1", "test_tool") inputs = make_inputs_with_tools([tc]) @@ -712,7 +719,7 @@ class TestApplyGuardrail: mock_client = AsyncMock() mock_client.post = mock_post - handler.tool_blocking_client = mock_client + handler.moderation_client = mock_client logging_obj = Mock() logging_obj.model_call_details = { @@ -739,8 +746,8 @@ class TestApplyGuardrail: logging_obj=logging_obj, ) - forwarded = captured_payload["request"]["proxy_server_request"] - assert forwarded == {"url": "/chat/completions", "method": "POST"} + # proxy_server_request is deliberately excluded from the forwarded envelope + assert "proxy_server_request" not in captured_payload["request"] # -- Anthropic format ---------------------------------------------------------- @@ -760,7 +767,7 @@ class TestApplyGuardrailAnthropicFormat: ) inputs = make_inputs_with_tools([tc], texts=["I'll check the weather."]) - handler.tool_blocking_client = _echo_service() + handler.moderation_client = _echo_service() result = await handler.apply_guardrail( inputs=inputs, request_data={}, input_type="response" @@ -771,7 +778,7 @@ class TestApplyGuardrailAnthropicFormat: tc = make_tool_call_dict("toolu_123", "dangerous_tool", '{"arg": "value"}') inputs = make_inputs_with_tools([tc]) - handler.tool_blocking_client = _mock_service_response( + handler.moderation_client = _mock_service_response( { "choices": [ { @@ -790,21 +797,21 @@ class TestApplyGuardrailAnthropicFormat: inputs=inputs, request_data={}, input_type="response" ) - async def test_text_only_response_no_blocking(self, handler): + async def test_text_only_response_sent_to_moderation(self, handler): + """Text-only responses (no tool calls) are sent to the response + moderation service to check the assistant's text content.""" from litellm.types.utils import GenericGuardrailAPIInputs inputs = GenericGuardrailAPIInputs(texts=["Hello! I'm Claude."]) - mock_client = AsyncMock() - mock_client.post = AsyncMock() - handler.tool_blocking_client = mock_client + # Service allows the response (returns the content unchanged) + handler.moderation_client = _echo_service() result = await handler.apply_guardrail( inputs=inputs, request_data={}, input_type="response" ) assert result is inputs - mock_client.post.assert_not_called() async def test_service_failure_preserves_tools(self, handler): tc = make_tool_call_dict("toolu_123", "get_weather", '{"location": "SF"}') @@ -812,7 +819,7 @@ class TestApplyGuardrailAnthropicFormat: mock_client = AsyncMock() mock_client.post = AsyncMock(side_effect=httpx.TimeoutException("Timeout")) - handler.tool_blocking_client = mock_client + handler.moderation_client = mock_client result = await handler.apply_guardrail( inputs=inputs, request_data={}, input_type="response" @@ -850,10 +857,13 @@ class TestNormalizeToolCalls: RubrikLogger._normalize_tool_calls(["not_a_tool_call"]) -# -- Extract blocked tools ----------------------------------------------------- +# -- Extract response block ---------------------------------------------------- -class TestExtractBlockedTools: +class TestExtractResponseBlock: + """Tests for _extract_response_block, which replaces the upstream + _extract_blocked_tools and handles both text blocks and tool blocks.""" + def test_all_allowed_returns_none(self): from litellm.types.utils import ChatCompletionMessageToolCall, Function @@ -870,7 +880,7 @@ class TestExtractBlockedTools: } ] } - result = RubrikLogger._extract_blocked_tools(service_resp, [tc]) + result = RubrikLogger._extract_response_block(service_resp, [tc], "") assert result is None def test_some_blocked_returns_explanation(self): @@ -896,13 +906,13 @@ class TestExtractBlockedTools: } ] } - result = RubrikLogger._extract_blocked_tools(service_resp, [tc1, tc2]) + result = RubrikLogger._extract_response_block(service_resp, [tc1, tc2], "") assert result is not None - assert "blocked fn2" in result + assert "blocked fn2" in result.explanation def test_empty_choices_raises(self): - with pytest.raises(Exception, match="empty response"): - RubrikLogger._extract_blocked_tools({"choices": []}, []) + with pytest.raises(_MalformedToolBlockingResponseError): + RubrikLogger._extract_response_block({"choices": []}, [], "") def test_null_tool_calls_treated_as_all_blocked(self): from litellm.types.utils import ChatCompletionMessageToolCall, Function @@ -920,36 +930,55 @@ class TestExtractBlockedTools: } ] } - result = RubrikLogger._extract_blocked_tools(service_resp, [tc]) + result = RubrikLogger._extract_response_block(service_resp, [tc], "") assert result is not None - assert "blocked everything" in result + assert "blocked everything" in result.explanation - def test_duplicate_ids_block_when_only_one_returned(self): + def test_text_block_detected(self): + """When the service replaces the response text wholesale, it's a text block.""" from litellm.types.utils import ChatCompletionMessageToolCall, Function - tc1 = ChatCompletionMessageToolCall( - id="call_dup", - type="function", - function=Function(name="fn", arguments="{}"), - ) - tc2 = ChatCompletionMessageToolCall( - id="call_dup", - type="function", - function=Function(name="fn", arguments="{}"), - ) service_resp = { "choices": [ { "message": { - "tool_calls": [{"id": "call_dup"}], - "content": "blocked duplicate", + "tool_calls": [], + "content": "This content violates policy.", } } ] } - result = RubrikLogger._extract_blocked_tools(service_resp, [tc1, tc2]) + result = RubrikLogger._extract_response_block( + service_resp, [], "Original assistant text." + ) assert result is not None - assert "blocked duplicate" in result + assert "violates policy" in result.explanation + + def test_tool_block_with_appended_explanation(self): + """When the service appends an explanation to the original text, only the + appended part is returned as the explanation.""" + from litellm.types.utils import ChatCompletionMessageToolCall, Function + + tc = ChatCompletionMessageToolCall( + id="call_1", type="function", function=Function(name="fn", arguments="{}") + ) + original_text = "Here is my response." + appended_explanation = "Tool call was blocked." + service_resp = { + "choices": [ + { + "message": { + "tool_calls": [], + "content": original_text + "\n\n" + appended_explanation, + } + } + ] + } + result = RubrikLogger._extract_response_block( + service_resp, [tc], original_text + ) + assert result is not None + assert appended_explanation in result.explanation # -- Sanitize proxy server request ------------------------------------------- @@ -1010,3 +1039,796 @@ class TestResolveModel: {"response": response}, {"model": "fallback"} ) assert result == "unknown" + + +# -- Additional Initialization edge cases ------------------------------------ + + +class TestInitializationEdgeCases: + def test_batch_size_zero_uses_default(self): + """RUBRIK_BATCH_SIZE=0 must warn and fall back to the default.""" + with patch("asyncio.create_task", Mock()): + with patch.dict( + os.environ, + {"RUBRIK_WEBHOOK_URL": "http://host", "RUBRIK_BATCH_SIZE": "0"}, + ): + h = RubrikLogger() + # Should use default, not 0 + assert h.batch_size > 0 + + def test_batch_size_negative_uses_default(self): + """RUBRIK_BATCH_SIZE=-1 must warn and fall back to the default.""" + with patch("asyncio.create_task", Mock()): + with patch.dict( + os.environ, + {"RUBRIK_WEBHOOK_URL": "http://host", "RUBRIK_BATCH_SIZE": "-5"}, + ): + h = RubrikLogger() + assert h.batch_size > 0 + + +# -- aclose() ----------------------------------------------------------------- + + +@pytest.mark.asyncio +class TestAclose: + async def test_aclose_cancels_task_does_not_close_shared_client(self, mock_env): + """aclose() cancels the periodic flush task but does NOT close the shared + moderation_client — closing a shared cached client would break other + RubrikLogger instances that share the same connection pool.""" + with patch("asyncio.create_task", Mock()): + handler = RubrikLogger() + + mock_task = Mock() + mock_task.cancel = Mock() + handler._periodic_flush_task = mock_task + + handler.moderation_client = AsyncMock() + handler.moderation_client.close = AsyncMock() + + await handler.aclose() + + mock_task.cancel.assert_called_once() + handler.moderation_client.close.assert_not_awaited() + + async def test_aclose_with_none_task_does_not_close_client(self, mock_env): + """aclose() with no flush task still does not close the shared client.""" + with patch("asyncio.create_task", Mock()): + handler = RubrikLogger() + + handler._periodic_flush_task = None + handler.moderation_client = AsyncMock() + handler.moderation_client.close = AsyncMock() + + await handler.aclose() + + handler.moderation_client.close.assert_not_awaited() + + +# -- apply_guardrail edge cases ----------------------------------------------- + + +@pytest.mark.asyncio +class TestApplyGuardrailEdgeCases: + async def test_unknown_input_type_returns_inputs_unchanged(self, handler): + """When input_type is not 'request' or 'response', inputs are returned as-is.""" + inputs = make_inputs_with_tools([make_tool_call_dict("call_1", "tool")]) + result = await handler.apply_guardrail( + inputs=inputs, request_data={}, input_type="unknown" + ) + assert result is inputs + + async def test_response_with_no_texts_and_no_tool_calls_returns_inputs(self, handler): + """_moderate_response early-returns when both texts and tool_calls are empty.""" + from litellm.types.utils import GenericGuardrailAPIInputs + + inputs = GenericGuardrailAPIInputs() + result = await handler.apply_guardrail( + inputs=inputs, request_data={}, input_type="response" + ) + assert result is inputs + + async def test_moderate_response_empty_call_details_emits_warning(self, handler): + """When logging_obj is present but model_call_details is empty, a warning is + logged and moderation proceeds (fail-open on HTTP error).""" + tc = make_tool_call_dict("call_1", "test_tool") + inputs = make_inputs_with_tools([tc]) + + logging_obj = Mock() + logging_obj.model_call_details = {} + + handler.moderation_client = _echo_service() + + result = await handler.apply_guardrail( + inputs=inputs, + request_data={}, + input_type="response", + logging_obj=logging_obj, + ) + assert result is inputs + + +# -- Prompt moderation -------------------------------------------------------- + + +@pytest.mark.asyncio +class TestPromptModeration: + async def test_prompt_moderation_passthrough(self, handler): + """Webhook returns {} (empty dict) → inputs returned unchanged.""" + inputs = {"structured_messages": [{"role": "user", "content": "Hello"}]} + + handler.moderation_client = _mock_service_response({}) + + result = await handler.apply_guardrail( + inputs=inputs, request_data={}, input_type="request" + ) + assert result is inputs + + async def test_prompt_moderation_blocked_raises(self, handler): + """Webhook returns synthetic chat.completion → raises ModifyResponseException.""" + inputs = { + "structured_messages": [{"role": "user", "content": "Harmful prompt"}], + "model": "gpt-4", + } + + handler.moderation_client = _mock_service_response( + { + "choices": [ + { + "message": { + "role": "assistant", + "content": "This request violates our policy.", + } + } + ] + } + ) + + with pytest.raises(ModifyResponseException) as exc_info: + await handler.apply_guardrail( + inputs=inputs, request_data={"model": "gpt-4"}, input_type="request" + ) + assert "violates our policy" in exc_info.value.message + + async def test_prompt_moderation_no_messages_skips_moderation(self, handler): + """When structured_messages is absent/empty, moderation is skipped.""" + inputs = {"model": "gpt-4"} + + result = await handler.apply_guardrail( + inputs=inputs, request_data={}, input_type="request" + ) + assert result is inputs + + async def test_prompt_moderation_stashes_logging_obj_on_block(self, handler): + """On a prompt block, _stash_block_context must set the blocked flag.""" + inputs = { + "structured_messages": [{"role": "user", "content": "bad prompt"}], + } + + handler.moderation_client = _mock_service_response( + { + "choices": [ + {"message": {"role": "assistant", "content": "Blocked."}} + ] + } + ) + + logging_obj = Mock() + logging_obj.model_call_details = {} + request_data: dict = {} + + with pytest.raises(ModifyResponseException): + await handler.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="request", + logging_obj=logging_obj, + ) + + assert logging_obj.model_call_details.get("_rubrik_blocked") is True + assert request_data.get("_rubrik_logging_obj") is logging_obj + + +# -- _stash_block_context ----------------------------------------------------- + + +class TestStashBlockContext: + def test_with_non_none_logging_obj_sets_flag_and_stashes(self): + """Sets _rubrik_blocked flag and stores logging_obj on request_data.""" + logging_obj = Mock() + logging_obj.model_call_details = {} + request_data: dict = {} + + RubrikLogger._stash_block_context(logging_obj, request_data) + + assert logging_obj.model_call_details["_rubrik_blocked"] is True + assert request_data["_rubrik_logging_obj"] is logging_obj + + def test_with_none_logging_obj_stores_none_on_request_data(self): + """When logging_obj is None, stores None on request_data (logged as error).""" + request_data: dict = {"litellm_call_id": "test-id"} + + RubrikLogger._stash_block_context(None, request_data) + + assert request_data["_rubrik_logging_obj"] is None + + +# -- _normalize_tool_calls duck-typed ----------------------------------------- + + +class TestNormalizeToolCallsDuckTyped: + def test_duck_typed_object_with_id_and_function_attrs(self): + """Objects that have .id and .function attrs but are not + ChatCompletionMessageToolCall are handled by the third branch.""" + from litellm.types.utils import Function + + tc = Mock() + tc.id = "call_duck" + tc.type = "function" + tc.function = Function(name="duck_tool", arguments='{"x": 1}') + # Make isinstance(..., ChatCompletionMessageToolCall) return False + # by using a plain Mock (not a ChatCompletionMessageToolCall subclass) + + result = RubrikLogger._normalize_tool_calls([tc]) + assert len(result) == 1 + assert result[0].id == "call_duck" + assert result[0].function.name == "duck_tool" + + def test_duck_typed_without_type_defaults_to_function(self): + """getattr(tc, "type", None) falls back to "function" when absent.""" + from litellm.types.utils import Function + + tc = Mock(spec=["id", "function"]) # no .type attr + tc.id = "call_no_type" + tc.function = Function(name="fn", arguments="{}") + + result = RubrikLogger._normalize_tool_calls([tc]) + assert result[0].type == "function" + + +# -- _flatten_messages_for_moderation ----------------------------------------- + + +class TestFlattenMessagesForModeration: + def test_plain_string_content_preserved(self): + messages = [{"role": "user", "content": "Hello world"}] + result = RubrikLogger._flatten_messages_for_moderation(messages) + assert len(result) == 1 + assert result[0]["role"] == "user" + assert result[0]["content"] == "Hello world" + + def test_content_list_flattened_to_string(self): + """Content as a list of parts (e.g. Anthropic multi-part) is flattened.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello from parts"}, + ], + } + ] + result = RubrikLogger._flatten_messages_for_moderation(messages) + assert len(result) == 1 + assert result[0]["role"] == "user" + assert "Hello from parts" in result[0]["content"] + + def test_non_dict_messages_skipped(self): + """Non-dict entries in the messages list are silently skipped.""" + messages = [ + "raw string message", + {"role": "user", "content": "valid"}, + ] + result = RubrikLogger._flatten_messages_for_moderation(messages) + assert len(result) == 1 + assert result[0]["content"] == "valid" + + def test_none_messages_returns_empty(self): + result = RubrikLogger._flatten_messages_for_moderation(None) + assert result == () + + def test_multiple_messages_preserved_in_order(self): + messages = [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Question?"}, + ] + result = RubrikLogger._flatten_messages_for_moderation(messages) + assert len(result) == 2 + assert result[0]["role"] == "system" + assert result[1]["role"] == "user" + + +# -- _build_prompt_moderation_payload ----------------------------------------- + + +class TestBuildPromptModerationPayload: + def test_payload_includes_tools_when_present(self): + inputs = { + "model": "gpt-4", + "structured_messages": [{"role": "user", "content": "hi"}], + "tools": [{"type": "function", "function": {"name": "fn"}}], + } + payload = RubrikLogger._build_prompt_moderation_payload(inputs, {}) + assert payload["tools"] == [{"type": "function", "function": {"name": "fn"}}] + + def test_payload_includes_user_when_present(self): + inputs = { + "structured_messages": [{"role": "user", "content": "hi"}], + } + request_data = {"user": "alice"} + payload = RubrikLogger._build_prompt_moderation_payload(inputs, request_data) + assert payload["user"] == "alice" + + def test_payload_uses_explicit_correlation_key(self): + inputs = {"structured_messages": [{"role": "user", "content": "hi"}]} + request_data = { + "correlation_key": "corr-123", + "litellm_call_id": "litellm-456", + } + payload = RubrikLogger._build_prompt_moderation_payload(inputs, request_data) + assert payload["correlation_key"] == "corr-123" + + def test_payload_falls_back_to_litellm_call_id(self): + """When correlation_key is absent, litellm_call_id is used.""" + inputs = {"structured_messages": [{"role": "user", "content": "hi"}]} + request_data = {"litellm_call_id": "litellm-789"} + payload = RubrikLogger._build_prompt_moderation_payload(inputs, request_data) + assert payload["correlation_key"] == "litellm-789" + + def test_payload_omits_optional_fields_when_absent(self): + inputs = {"structured_messages": [{"role": "user", "content": "hi"}]} + payload = RubrikLogger._build_prompt_moderation_payload(inputs, {}) + assert "tools" not in payload + assert "user" not in payload + assert "correlation_key" not in payload + + +# -- _extract_request_data tools preference ----------------------------------- + + +class TestExtractRequestDataToolsPreference: + def test_prefers_tools_from_request_data_over_optional_params(self): + """When 'tools' key exists in request_data, it wins over optional_params.""" + call_details = { + "messages": [{"role": "user", "content": "hi"}], + "model": "gpt-4", + "optional_params": { + "tools": [{"type": "function", "function": {"name": "from_optional"}}] + }, + } + request_data = { + "tools": [{"type": "function", "function": {"name": "from_request"}}] + } + result = RubrikLogger._extract_request_data(call_details, request_data) + assert result["tools"] == [ + {"type": "function", "function": {"name": "from_request"}} + ] + + def test_falls_back_to_optional_params_when_not_in_request_data(self): + call_details = { + "optional_params": { + "tools": [{"type": "function", "function": {"name": "from_optional"}}] + } + } + result = RubrikLogger._extract_request_data(call_details, {}) + assert result["tools"] == [ + {"type": "function", "function": {"name": "from_optional"}} + ] + + def test_explicit_empty_list_in_request_data_is_forwarded(self): + """An explicit empty tools list signals 'no tools' to the moderation service.""" + call_details = { + "optional_params": { + "tools": [{"type": "function", "function": {"name": "from_optional"}}] + } + } + request_data = {"tools": []} + result = RubrikLogger._extract_request_data(call_details, request_data) + assert result["tools"] == [] + + +# -- _extract_prompt_refusal -------------------------------------------------- + + +class TestExtractPromptRefusal: + def test_passthrough_response_returns_none(self): + """Empty dict (passthrough) → None.""" + assert RubrikLogger._extract_prompt_refusal({}) is None + + def test_no_choices_returns_none(self): + assert RubrikLogger._extract_prompt_refusal({"choices": []}) is None + + def test_block_response_returns_content(self): + service_response = { + "choices": [{"message": {"content": "Request blocked by Rubrik."}}] + } + result = RubrikLogger._extract_prompt_refusal(service_response) + assert result == "Request blocked by Rubrik." + + def test_empty_content_falls_back_to_default_message(self): + """When content is empty string or falsy, falls back to default refusal.""" + service_response = {"choices": [{"message": {"content": ""}}]} + result = RubrikLogger._extract_prompt_refusal(service_response) + assert result == "Request blocked by policy." + + def test_none_content_falls_back_to_default_message(self): + service_response = {"choices": [{"message": {"content": None}}]} + result = RubrikLogger._extract_prompt_refusal(service_response) + assert result == "Request blocked by policy." + + +# -- _prepend_system_prompt exception path ------------------------------------ + + +class TestPrependSystemPromptException: + def test_exception_during_unpack_is_caught_and_logged(self): + """When an exception is raised inside _prepend_system_prompt, it is swallowed.""" + + class ExplodingList(list): + def __iter__(self): + raise RuntimeError("iteration error!") + + payload = {"messages": ExplodingList()} + source = {"system": "You are an assistant."} + + # Must not raise + RubrikLogger._prepend_system_prompt(payload, source) + + +# -- _append_and_maybe_flush batch trigger ------------------------------------ + + +@pytest.mark.asyncio +class TestAppendAndMaybeFlush: + async def test_flush_triggered_when_queue_reaches_batch_size(self, handler): + """flush_queue is called when the queue length reaches batch_size.""" + handler.batch_size = 2 + handler.flush_queue = AsyncMock() + + await handler._append_and_maybe_flush({"msg": "a"}) + handler.flush_queue.assert_not_called() + + await handler._append_and_maybe_flush({"msg": "b"}) + handler.flush_queue.assert_called_once() + + async def test_no_flush_before_batch_size(self, handler): + handler.batch_size = 5 + handler.flush_queue = AsyncMock() + + for i in range(4): + await handler._append_and_maybe_flush({"msg": str(i)}) + + handler.flush_queue.assert_not_called() + + +# -- _enqueue_log_event exception handling ------------------------------------ + + +@pytest.mark.asyncio +class TestEnqueueLogEventExceptions: + async def test_exception_from_prepare_log_payload_is_caught(self, handler): + """Exceptions raised by _prepare_log_payload are caught and logged.""" + handler._prepare_log_payload = AsyncMock( + side_effect=RuntimeError("payload error") + ) + + # Must not raise + await handler._enqueue_log_event( + {"standard_logging_object": {"messages": [], "response": ""}}, "test" + ) + assert len(handler.log_queue) == 0 + + +# -- async_log_success_event skip when _rubrik_blocked ------------------------ + + +@pytest.mark.asyncio +class TestSuccessEventBlockedSkip: + async def test_skips_enqueue_when_rubrik_blocked_flag_set(self, handler): + """When kwargs['_rubrik_blocked'] is True, the event is not enqueued.""" + kwargs = { + "_rubrik_blocked": True, + "litellm_call_id": "blocked-call-123", + "standard_logging_object": { + "messages": [{"role": "user", "content": "hi"}], + "response": "hello", + }, + } + await handler.async_log_success_event( + kwargs=kwargs, response_obj=None, start_time=None, end_time=None + ) + assert len(handler.log_queue) == 0 + + +# -- async_post_call_failure_hook --------------------------------------------- + + +@pytest.mark.asyncio +class TestPostCallFailureHook: + async def test_non_modify_exception_returns_immediately(self, handler): + """Non-ModifyResponseException causes a no-op.""" + await handler.async_post_call_failure_hook( + request_data={"litellm_call_id": "test"}, + original_exception=ValueError("unrelated error"), + user_api_key_dict=None, + ) + assert len(handler.log_queue) == 0 + + async def test_modify_exception_without_stashed_logging_obj_emits_warning( + self, handler + ): + """ModifyResponseException with no _rubrik_logging_obj → warning, no enqueue.""" + request_data = {"litellm_call_id": "test-123", "model": "gpt-4"} + exc = ModifyResponseException( + message="blocked", + model="gpt-4", + request_data=request_data, + guardrail_name="rubrik", + ) + + await handler.async_post_call_failure_hook( + request_data=request_data, + original_exception=exc, + user_api_key_dict=None, + ) + assert len(handler.log_queue) == 0 + + async def test_modify_exception_with_valid_logging_obj_enqueues_payload( + self, handler + ): + """ModifyResponseException + stashed logging_obj → builds and enqueues.""" + logging_obj = Mock() + logging_obj.model_call_details = { + "litellm_call_id": "call-abc", + "model": "gpt-4", + "messages": [{"role": "user", "content": "hi"}], + "standard_logging_object": { + "id": "chatcmpl-original", + "model": "gpt-4", + "response": "original", + "messages": [{"role": "user", "content": "hi"}], + }, + "metadata": {}, + } + + request_data = {"_rubrik_logging_obj": logging_obj} + exc = ModifyResponseException( + message="blocked by policy", + model="gpt-4", + request_data=request_data, + guardrail_name="rubrik", + ) + + handler.batch_size = 10**6 # disable auto-flush + await handler.async_post_call_failure_hook( + request_data=request_data, + original_exception=exc, + user_api_key_dict=None, + ) + assert len(handler.log_queue) == 1 + assert "ModifyResponseException" in handler.log_queue[0]["response"] + + async def test_logging_obj_popped_from_request_data(self, handler): + """_rubrik_logging_obj must be popped from request_data so it is not + forwarded downstream.""" + logging_obj = Mock() + logging_obj.model_call_details = { + "litellm_call_id": "call-pop", + "model": "gpt-4", + "messages": [], + "standard_logging_object": { + "id": "chatcmpl-pop", + "model": "gpt-4", + "response": "text", + "messages": [], + }, + "metadata": {}, + } + + request_data = {"_rubrik_logging_obj": logging_obj} + exc = ModifyResponseException( + message="popped", + model="gpt-4", + request_data=request_data, + guardrail_name="rubrik", + ) + + handler.batch_size = 10**6 + await handler.async_post_call_failure_hook( + request_data=request_data, + original_exception=exc, + user_api_key_dict=None, + ) + assert "_rubrik_logging_obj" not in request_data + + async def test_build_and_enqueue_swallows_attribute_error_from_prepare_payload( + self, handler + ): + """When _prepare_block_failure_payload raises AttributeError/KeyError/TypeError, + the error is logged and the event is silently dropped (lines 806-812).""" + logging_obj = Mock() + # Make model_call_details.get() raise TypeError + logging_obj.model_call_details = None # .get() will raise AttributeError + + exc = ModifyResponseException( + message="blocked", + model="gpt-4", + request_data={}, + guardrail_name="rubrik", + ) + + # Must not raise + await handler._build_and_enqueue_block_event(logging_obj, exc, None) + assert len(handler.log_queue) == 0 + + async def test_build_and_enqueue_swallows_flush_exception(self, handler): + """When _append_and_maybe_flush raises, the error is logged (lines 816-817).""" + logging_obj = Mock() + logging_obj.model_call_details = { + "litellm_call_id": "call-flush-err", + "model": "gpt-4", + "messages": [], + "standard_logging_object": { + "id": "id-flush-err", + "model": "gpt-4", + "response": "text", + "messages": [], + }, + "metadata": {}, + } + + exc = ModifyResponseException( + message="blocked", + model="gpt-4", + request_data={}, + guardrail_name="rubrik", + ) + + handler._append_and_maybe_flush = AsyncMock( + side_effect=RuntimeError("flush failed") + ) + + # Must not raise + await handler._build_and_enqueue_block_event(logging_obj, exc, None) + + +# -- _prepare_block_failure_payload and _build_fallback_payload --------------- + + +class TestPrepareBlockFailurePayload: + def test_uses_standard_logging_object_when_present(self, handler): + """When standard_logging_object is on model_call_details, it is used as base.""" + logging_obj = Mock() + logging_obj.model_call_details = { + "litellm_call_id": "call-slo", + "model": "gpt-4", + "standard_logging_object": { + "id": "chatcmpl-original", + "model": "gpt-4", + "response": "original response", + "messages": [{"role": "user", "content": "hi"}], + }, + "metadata": {}, + } + exc = ModifyResponseException( + message="blocked", + model="gpt-4", + request_data={}, + guardrail_name="rubrik", + ) + + payload = handler._prepare_block_failure_payload(logging_obj, exc) + + assert "ModifyResponseException: blocked" in payload["response"] + assert payload["id"] == "call-slo" + + def test_uses_fallback_when_standard_logging_object_absent(self, handler): + """When standard_logging_object is absent, _build_fallback_payload is used.""" + from datetime import datetime + + logging_obj = Mock() + logging_obj.model_call_details = { + "litellm_call_id": "call-fallback", + "model": "claude-3", + "messages": [{"role": "user", "content": "question"}], + "optional_params": {"temperature": 0.5}, + "metadata": {"user_api_key_hash": "hash-abc"}, + "start_time": datetime(2024, 6, 1), + } + exc = ModifyResponseException( + message="prompt blocked", + model="claude-3", + request_data={}, + guardrail_name="rubrik", + ) + + payload = handler._prepare_block_failure_payload(logging_obj, exc) + + assert payload["id"] == "call-fallback" + assert payload["model"] == "claude-3" + assert payload["model_group"] == "claude-3" + assert "ModifyResponseException: prompt blocked" in payload["response"] + assert payload["metadata"]["user_api_key_hash"] == "hash-abc" + assert payload["status"] == "failure" + + def test_fallback_payload_without_start_time(self, handler): + """_build_fallback_payload handles missing start_time gracefully.""" + logging_obj = Mock() + logging_obj.model_call_details = { + "litellm_call_id": "call-notime", + "model": "gpt-4", + "messages": [], + "optional_params": {}, + "metadata": {}, + } + exc = ModifyResponseException( + message="blocked", + model="gpt-4", + request_data={}, + guardrail_name="rubrik", + ) + + payload = handler._prepare_block_failure_payload(logging_obj, exc) + assert payload["startTime"] is None + + +# -- async_send_batch empty queue and flush_queue edge cases ------------------ + + +@pytest.mark.asyncio +class TestQueueEdgeCases: + async def test_async_send_batch_returns_early_on_empty_queue(self, handler): + """async_send_batch is a no-op when the queue is empty.""" + handler.async_httpx_client = AsyncMock() + await handler.async_send_batch() + handler.async_httpx_client.post.assert_not_called() + + async def test_flush_queue_returns_early_when_flush_lock_is_none(self, handler): + """flush_queue is a no-op when flush_lock is None.""" + handler.flush_lock = None + handler.log_queue = [{"msg": "a"}] + handler.async_httpx_client = AsyncMock() + + await handler.flush_queue() + handler.async_httpx_client.post.assert_not_called() + + async def test_flush_queue_returns_early_when_queue_empty_inside_lock(self, handler): + """flush_queue acquires the lock then no-ops when the queue is empty.""" + handler.log_queue = [] + handler.async_httpx_client = AsyncMock() + + await handler.flush_queue() + handler.async_httpx_client.post.assert_not_called() + + +# -- _post_json non-dict response --------------------------------------------- + + +@pytest.mark.asyncio +class TestPostJson: + async def test_raises_type_error_for_list_response(self, handler): + """When the service returns a JSON array instead of a dict, TypeError is raised.""" + mock_client = AsyncMock() + mock_resp = Mock() + mock_resp.json.return_value = ["not", "a", "dict"] + mock_resp.raise_for_status = Mock() + mock_client.post = AsyncMock(return_value=mock_resp) + handler.moderation_client = mock_client + + with pytest.raises(TypeError, match="non-dict JSON"): + await handler._post_json( + handler.prompt_moderation_endpoint, {}, "Test service" + ) + + async def test_raises_type_error_for_string_response(self, handler): + """A bare string response also raises TypeError.""" + mock_client = AsyncMock() + mock_resp = Mock() + mock_resp.json.return_value = "blocked" + mock_resp.raise_for_status = Mock() + mock_client.post = AsyncMock(return_value=mock_resp) + handler.moderation_client = mock_client + + with pytest.raises(TypeError, match="non-dict JSON"): + await handler._post_json( + handler.response_moderation_endpoint, {}, "Test service" + ) From d4d0bf0acc078f081e7c0f7628bae3696a48ae10 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 3 Aug 2026 18:09:49 -0700 Subject: [PATCH 105/123] fix(ui): hide guardrail review buttons from non-admin users (#27535) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ui): hide guardrail review buttons from non-admin users The team guardrail submissions list rendered Approve/Reject buttons for non-admin users even though the backend correctly rejected the calls. Thread userRole from the page through GuardrailsPanel into TeamGuardrailsTab and gate the row-card and detail-panel review buttons on isAdmin so the UI matches the backend authorization. Defense in depth only — the backend remains the source of truth and is double-gated at both the route admin check and the explicit endpoint role check. Refs LIT-2494 * refactor(ui): read userRole from useAuthorized hook instead of prop drilling Drop the userRole prop chain through GuardrailsPage → GuardrailsPanel → TeamGuardrailsTab. Each component reads userRole directly from the useAuthorized hook, matching the pattern used elsewhere in the dashboard. Tests now mock useAuthorized per case (the same pattern as top_key_view.test.tsx) instead of passing userRole as a prop. Refs LIT-2494 * fix(ui): drop userRole prop on GuardrailsPanel call site in src/app/page.tsx Missed in the earlier refactor — GuardrailsPanel no longer accepts userRole as a prop (reads from useAuthorized hook), so callers must not pass it. The build was failing in production type-check. Refs LIT-2494 * fix(ui): gate guardrail forward-key toggle and header editors on proxy admin * refactor(ui): remove dead app_admin case from user role formatting --- .../_components/TeamGuardrailsTab.test.tsx | 142 +++++++++++ .../_components/TeamGuardrailsTab.tsx | 221 ++++++++++-------- .../(dashboard)/hooks/useAuthorized.test.ts | 6 +- .../src/components/user_dashboard.tsx | 2 - ui/litellm-dashboard/src/utils/roles.ts | 2 - 5 files changed, 269 insertions(+), 104 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.test.tsx new file mode 100644 index 00000000000..603cddb7b89 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.test.tsx @@ -0,0 +1,142 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { renderWithProviders } from "@/../tests/test-utils"; +import { screen, fireEvent } from "@testing-library/react"; +import { TeamGuardrailsTab } from "./TeamGuardrailsTab"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; + +vi.mock("@/components/networking", () => ({ + listGuardrailSubmissions: vi.fn(), + approveGuardrailSubmission: vi.fn(), + rejectGuardrailSubmission: vi.fn(), + updateGuardrailCall: vi.fn(), +})); + +vi.mock("@/app/(dashboard)/hooks/guardrails/useRegisterGuardrail", () => ({ + useRegisterGuardrail: () => ({ + mutateAsync: vi.fn(), + isPending: false, + }), +})); + +vi.mock("@/components/common_components/team_dropdown", () => ({ + default: () => null, +})); + +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: vi.fn(), +})); + +import { listGuardrailSubmissions } from "@/components/networking"; + +const pendingSubmission = { + guardrail_id: "guard-1", + guardrail_name: "test-pending-guardrail", + status: "pending_review", + team_id: "team-1", + team_guardrail: true, + litellm_params: { + guardrail: "generic_guardrail_api", + mode: "pre_call", + api_base: "https://example.com/guard", + headers: { "X-API-Key": "secret" }, + extra_headers: ["x-request-id"], + }, + guardrail_info: {}, + submitted_at: "2026-05-09T00:00:00Z", +}; + +const baseAuth = { + token: "test-token", + accessToken: "test-token", + userId: "user-1", + userEmail: "user@example.com", + premiumUser: false, + disabledPersonalKeyCreation: null, + showSSOBanner: false, +}; + +describe("TeamGuardrailsTab — approve/reject role gate", () => { + const mockUseAuthorized = vi.mocked(useAuthorized); + + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(listGuardrailSubmissions).mockResolvedValue({ + submissions: [pendingSubmission], + summary: { total: 1, pending_review: 1, active: 0, rejected: 0 }, + }); + }); + + it("hides Approve and Reject buttons for an internal user on a pending submission", async () => { + mockUseAuthorized.mockReturnValue({ ...baseAuth, userRole: "Internal User" }); + renderWithProviders(); + + await screen.findByText("test-pending-guardrail"); + + expect(screen.queryByRole("button", { name: /approve/i })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /reject/i })).not.toBeInTheDocument(); + }); + + it("hides Approve and Reject buttons for an Admin Viewer, whom the backend rejects with 403", async () => { + mockUseAuthorized.mockReturnValue({ ...baseAuth, userRole: "Admin Viewer" }); + renderWithProviders(); + + await screen.findByText("test-pending-guardrail"); + + expect(screen.queryByRole("button", { name: /approve/i })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /reject/i })).not.toBeInTheDocument(); + }); + + it("shows Approve and Reject buttons for an admin on a pending submission", async () => { + mockUseAuthorized.mockReturnValue({ ...baseAuth, userRole: "Admin" }); + renderWithProviders(); + + await screen.findByText("test-pending-guardrail"); + + expect(screen.getByRole("button", { name: /approve/i })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /reject/i })).toBeInTheDocument(); + }); + + it("hides Approve and Reject buttons when userRole is undefined (defaults to non-admin)", async () => { + mockUseAuthorized.mockReturnValue({ ...baseAuth, userRole: undefined }); + renderWithProviders(); + + await screen.findByText("test-pending-guardrail"); + + expect(screen.queryByRole("button", { name: /approve/i })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /reject/i })).not.toBeInTheDocument(); + }); + + it("disables all admin-only write controls for a non-admin, including the detail panel", async () => { + mockUseAuthorized.mockReturnValue({ ...baseAuth, userRole: "Internal User" }); + renderWithProviders(); + + await screen.findByText("test-pending-guardrail"); + expect(screen.getByRole("switch")).toBeDisabled(); + + fireEvent.click(screen.getByRole("button", { name: "Review" })); + await screen.findByText("Forward LiteLLM API Key"); + + expect(screen.queryByRole("button", { name: /approve/i })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /reject/i })).not.toBeInTheDocument(); + screen.getAllByRole("switch").forEach((toggle) => expect(toggle).toBeDisabled()); + expect(screen.queryByRole("button", { name: "Add" })).not.toBeInTheDocument(); + expect(screen.queryByLabelText(/^Remove/)).not.toBeInTheDocument(); + expect(screen.queryByPlaceholderText("e.g. x-request-id")).not.toBeInTheDocument(); + }); + + it("keeps all write controls enabled for an admin in the detail panel", async () => { + mockUseAuthorized.mockReturnValue({ ...baseAuth, userRole: "Admin" }); + renderWithProviders(); + + await screen.findByText("test-pending-guardrail"); + + fireEvent.click(screen.getByRole("button", { name: "Review" })); + await screen.findByText("Forward LiteLLM API Key"); + + expect(screen.getAllByRole("button", { name: /approve/i }).length).toBeGreaterThanOrEqual(2); + screen.getAllByRole("switch").forEach((toggle) => expect(toggle).toBeEnabled()); + expect(screen.getAllByRole("button", { name: "Add" })).toHaveLength(2); + expect(screen.getByLabelText("Remove X-API-Key")).toBeInTheDocument(); + expect(screen.getByLabelText("Remove x-request-id")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx index 4217a765732..496e1129371 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx @@ -27,6 +27,8 @@ import { import NotificationsManager from "@/components/molecules/notifications_manager"; import TeamDropdown from "@/components/common_components/team_dropdown"; import { useRegisterGuardrail } from "@/app/(dashboard)/hooks/guardrails/useRegisterGuardrail"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { isProxyAdminRole } from "@/utils/roles"; type GuardrailStatus = "active" | "pending" | "rejected"; @@ -188,16 +190,25 @@ function StatCard({ label, value, color }: { label: string; value: number; color ); } -function Toggle({ enabled, onToggle }: { enabled: boolean; onToggle: () => void }) { +function Toggle({ + enabled, + onToggle, + disabled = false, +}: { + enabled: boolean; + onToggle: () => void; + disabled?: boolean; +}) { return ( - {g.status === "pending" && ( + {isAdmin && g.status === "pending" && ( <>
- +

When enabled, the caller's LiteLLM API key is forwarded as an{" "} @@ -456,28 +471,63 @@ function DetailPanel({ {h.key}: {h.value} - + {isAdmin && ( + + )} ))} )} -

- setNewStaticHeaderKey(e.target.value)} - placeholder="Header name (e.g. X-API-Key)" - className="flex-1 min-w-0 text-xs font-mono border border-gray-200 rounded-sm px-2 py-1.5 text-gray-700 placeholder-gray-400 focus:outline-hidden focus:ring-1 focus:ring-blue-500" - onKeyDown={(e) => { - if (e.key === "Enter") { - e.preventDefault(); + {isAdmin && ( +
+ setNewStaticHeaderKey(e.target.value)} + placeholder="Header name (e.g. X-API-Key)" + className="flex-1 min-w-0 text-xs font-mono border border-gray-200 rounded-sm px-2 py-1.5 text-gray-700 placeholder-gray-400 focus:outline-hidden focus:ring-1 focus:ring-blue-500" + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + const key = newStaticHeaderKey.trim(); + const value = newStaticHeaderValue.trim(); + if (key && !g.customHeaders.some((h) => h.key.toLowerCase() === key.toLowerCase())) { + onUpdateCustomHeaders([...g.customHeaders, { key, value }]); + setNewStaticHeaderKey(""); + setNewStaticHeaderValue(""); + } + } + }} + /> + setNewStaticHeaderValue(e.target.value)} + placeholder="Value" + className="flex-1 min-w-0 text-xs font-mono border border-gray-200 rounded-sm px-2 py-1.5 text-gray-700 placeholder-gray-400 focus:outline-hidden focus:ring-1 focus:ring-blue-500" + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + const key = newStaticHeaderKey.trim(); + const value = newStaticHeaderValue.trim(); + if (key && !g.customHeaders.some((h) => h.key.toLowerCase() === key.toLowerCase())) { + onUpdateCustomHeaders([...g.customHeaders, { key, value }]); + setNewStaticHeaderKey(""); + setNewStaticHeaderValue(""); + } + } + }} + /> + -
+ }} + className="text-xs font-medium text-blue-600 hover:text-blue-700 border border-blue-200 bg-blue-50 hover:bg-blue-100 px-2 py-1.5 rounded-sm transition-colors shrink-0" + > + Add + +
+ )}
@@ -546,50 +565,54 @@ function DetailPanel({ className="flex items-center justify-between gap-2 text-xs font-mono bg-gray-50 border border-gray-200 rounded-sm px-2 py-1.5" > {name} - + {isAdmin && ( + + )} ))} )} -
- setNewExtraHeader(e.target.value)} - placeholder="e.g. x-request-id" - className="flex-1 min-w-0 text-xs font-mono border border-gray-200 rounded-sm px-2 py-1.5 text-gray-700 placeholder-gray-400 focus:outline-hidden focus:ring-1 focus:ring-blue-500" - onKeyDown={(e) => { - if (e.key === "Enter") { - e.preventDefault(); + {isAdmin && ( +
+ setNewExtraHeader(e.target.value)} + placeholder="e.g. x-request-id" + className="flex-1 min-w-0 text-xs font-mono border border-gray-200 rounded-sm px-2 py-1.5 text-gray-700 placeholder-gray-400 focus:outline-hidden focus:ring-1 focus:ring-blue-500" + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + const name = newExtraHeader.trim().toLowerCase(); + if (name && !g.extraHeaders.map((h) => h.toLowerCase()).includes(name)) { + onUpdateExtraHeaders([...g.extraHeaders, name]); + setNewExtraHeader(""); + } + } + }} + /> + -
+ }} + className="text-xs font-medium text-blue-600 hover:text-blue-700 border border-blue-200 bg-blue-50 hover:bg-blue-100 px-2 py-1.5 rounded-sm transition-colors" + > + Add + +
+ )}
- {g.status === "pending" && ( + {isAdmin && g.status === "pending" && (
+ + ); +}; + +describe("MetadataKeyValueFields", () => { + it("renders one row per existing pair", () => { + render( + , + ); + + const keyInputs = screen.getAllByPlaceholderText("Key"); + const valueInputs = screen.getAllByPlaceholderText("Value"); + expect(keyInputs.map((input) => (input as HTMLInputElement).value)).toEqual(["department", "tier"]); + expect(valueInputs.map((input) => (input as HTMLInputElement).value)).toEqual(["research", "3"]); + }); + + it("adds a row and submits the entered pair", async () => { + const user = userEvent.setup(); + const onFinish = vi.fn(); + render(); + + await user.click(screen.getByRole("button", { name: /add key-value pair/i })); + await user.type(screen.getByPlaceholderText("Key"), "cost_center"); + await user.type(screen.getByPlaceholderText("Value"), "eng-1"); + await user.click(screen.getByRole("button", { name: "Save" })); + + await waitFor(() => { + expect(onFinish).toHaveBeenCalledWith({ metadata: [{ key: "cost_center", value: "eng-1" }] }); + }); + }); + + it("removes a row when its remove icon is clicked", async () => { + const user = userEvent.setup(); + const onFinish = vi.fn(); + render( + , + ); + + await user.click(screen.getAllByLabelText("Remove key-value pair")[0]); + await user.click(screen.getByRole("button", { name: "Save" })); + + await waitFor(() => { + expect(onFinish).toHaveBeenCalledWith({ metadata: [{ key: "tier", value: "3" }] }); + }); + }); + + it("blocks submission on duplicate keys", async () => { + const user = userEvent.setup(); + const onFinish = vi.fn(); + render( + , + ); + + await user.click(screen.getByRole("button", { name: "Save" })); + + await waitFor(() => { + expect(screen.getAllByText("Duplicate key").length).toBeGreaterThan(0); + }); + expect(onFinish).not.toHaveBeenCalled(); + }); + + it("blocks submission when a row is missing its key", async () => { + const user = userEvent.setup(); + const onFinish = vi.fn(); + render(); + + await user.click(screen.getByRole("button", { name: /add key-value pair/i })); + await user.type(screen.getByPlaceholderText("Value"), "orphan"); + await user.click(screen.getByRole("button", { name: "Save" })); + + await waitFor(() => { + expect(screen.getByText("Missing key")).toBeInTheDocument(); + }); + expect(onFinish).not.toHaveBeenCalled(); + }); +}); + +describe("MetadataKeyValueFields with a declared schema", () => { + const schema: TeamMetadataField[] = [ + { key: "cost_center", label: "Cost Center" }, + { key: "app_name", label: "Application Name" }, + ]; + + it("should prepopulate one ordinary editable pair row per declared key", async () => { + render(); + + await waitFor(() => { + expect(screen.getAllByPlaceholderText("Key").map((input) => (input as HTMLInputElement).value)).toEqual([ + "cost_center", + "app_name", + ]); + }); + screen.getAllByPlaceholderText("Key").forEach((input) => expect(input).toBeEnabled()); + expect(screen.getAllByLabelText("Remove key-value pair")).toHaveLength(2); + }); + + it("should submit a prepopulated key with its typed value", async () => { + const user = userEvent.setup(); + const onFinish = vi.fn(); + render(); + + await user.type(await screen.findByPlaceholderText("Value"), "CC-1001"); + await user.click(screen.getByRole("button", { name: "Save" })); + + await waitFor(() => { + expect(onFinish).toHaveBeenCalledWith({ metadata: [{ key: "cost_center", value: "CC-1001" }] }); + }); + }); + + it("should not add a second row for keys already present in the form", async () => { + render( + , + ); + + await waitFor(() => { + expect(screen.getAllByPlaceholderText("Key").map((input) => (input as HTMLInputElement).value)).toEqual([ + "cost_center", + "app_name", + ]); + }); + expect(screen.getAllByPlaceholderText("Value").map((input) => (input as HTMLInputElement).value)).toEqual([ + "CC-1001", + "", + ]); + }); + + it("should let the user remove a prepopulated row", async () => { + const user = userEvent.setup(); + render(); + + await screen.findAllByPlaceholderText("Key"); + await user.click(screen.getAllByLabelText("Remove key-value pair")[0]); + + await waitFor(() => { + expect(screen.getAllByPlaceholderText("Key").map((input) => (input as HTMLInputElement).value)).toEqual([ + "app_name", + ]); + }); + }); + + it("should show a skeleton instead of the editor while the schema is loading", () => { + render(); + + expect(screen.getByTestId("metadata-schema-skeleton")).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /add key-value pair/i })).not.toBeInTheDocument(); + }); + + it("should seed rows when the schema arrives after an initial loading state", async () => { + const onFinish = vi.fn(); + const { rerender } = render(); + + rerender(); + + await waitFor(() => { + expect(screen.getAllByPlaceholderText("Key").map((input) => (input as HTMLInputElement).value)).toEqual([ + "cost_center", + "app_name", + ]); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/common_components/MetadataKeyValueFields.tsx b/ui/litellm-dashboard/src/components/common_components/MetadataKeyValueFields.tsx new file mode 100644 index 00000000000..da085f95ad8 --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/MetadataKeyValueFields.tsx @@ -0,0 +1,135 @@ +import { MinusCircleOutlined, PlusOutlined } from "@ant-design/icons"; +import { Button, Form, FormInstance, Input, Skeleton, Space } from "antd"; +import React, { useEffect, useRef } from "react"; + +import { TeamMetadataField } from "@/app/(dashboard)/hooks/teams/useTeamMetadataSchema"; + +export interface MetadataPair { + key: string; + value: string; +} + +function formatMetadataValue(value: unknown): string { + if (typeof value !== "string") { + return JSON.stringify(value) ?? ""; + } + try { + JSON.parse(value); + return JSON.stringify(value); + } catch { + return value; + } +} + +function parseMetadataValue(raw: string): unknown { + try { + return JSON.parse(raw); + } catch { + return raw; + } +} + +export function metadataObjectToPairs( + metadata: Record | null | undefined, + excludedKeys: ReadonlySet = new Set(), +): MetadataPair[] { + return Object.entries(metadata ?? {}) + .filter(([key]) => !excludedKeys.has(key)) + .map(([key, value]) => ({ key, value: formatMetadataValue(value) })); +} + +export function metadataPairsToObject( + pairs: readonly (Partial | undefined)[] | undefined, +): Record { + return Object.fromEntries( + (pairs ?? []) + .filter((pair): pair is Partial & { key: string } => Boolean(pair?.key)) + .map((pair) => [pair.key, parseMetadataValue(pair.value ?? "")]), + ); +} + +interface MetadataKeyValueFieldsProps { + form: FormInstance; + name?: string; + schemaFields?: readonly TeamMetadataField[]; + schemaLoading?: boolean; +} + +const MetadataKeyValueFields: React.FC = ({ + form, + name = "metadata", + schemaFields = [], + schemaLoading = false, +}) => { + const seededRef = useRef(false); + + useEffect(() => { + if (seededRef.current || schemaLoading || schemaFields.length === 0) return; + seededRef.current = true; + const pairs: (Partial | undefined)[] = form.getFieldValue(name) ?? []; + if (!Array.isArray(pairs)) return; + const existingKeys = new Set(pairs.map((pair) => pair?.key).filter(Boolean)); + const seeded = schemaFields + .filter((field) => !existingKeys.has(field.key)) + .map((field) => ({ key: field.key, value: "" })); + if (seeded.length > 0) { + form.setFieldValue(name, [...pairs, ...seeded]); + } + }, [form, name, schemaFields, schemaLoading]); + + if (schemaLoading) { + return ( +
+ +
+ ); + } + + return ( + + {(fields, { add, remove }) => ( + <> + {fields.map(({ key, name: fieldName, ...restField }) => ( + + { + if (!value) return Promise.resolve(); + const all: (Partial | undefined)[] = form.getFieldValue(name) ?? []; + const dupes = all.filter((entry) => entry?.key === value); + if (dupes.length > 1) { + return Promise.reject(new Error("Duplicate key")); + } + return Promise.resolve(); + }, + }, + ]} + > + + + + + + remove(fieldName)} + style={{ color: "#ef4444" }} + /> + + ))} + + + + + )} + + ); +}; + +export default MetadataKeyValueFields; diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 03cf0e9583c..5a2d33ee4bd 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -38,7 +38,7 @@ import type { CoordinationRedisTestResponse, } from "@/app/(dashboard)/caching/_components/coordination_redis_settings/types"; import { MCP_TOOLS_PREVIEW_FORBIDDEN_MESSAGE } from "./mcp_tools/constants"; -import { createApiClient, deriveErrorMessage } from "@/lib/http/client"; +import { createApiClient, deriveErrorMessage, unwrapProxyErrorMessage } from "@/lib/http/client"; import { resolveApiBase } from "@/lib/http/resolveApiBase"; import { registerAuthHeaderNameGetter, @@ -2643,7 +2643,7 @@ export const teamUpdateCall = async ( const errorData = await response.text(); handleError(errorData); console.error("Error response from the server:", errorData); - NotificationsManager.fromBackend("Failed to update team settings: " + errorData); + NotificationsManager.fromBackend("Failed to update team settings: " + unwrapProxyErrorMessage(errorData)); throw new Error(errorData); } const data = (await response.json()) as { data: Team; team_id: string }; diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx index 712cff80649..513719a2ad9 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx @@ -1,3 +1,4 @@ +import { useTeamMetadataSchema } from "@/app/(dashboard)/hooks/teams/useTeamMetadataSchema"; import * as networking from "@/components/networking"; import { screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; @@ -26,6 +27,10 @@ vi.mock("@/components/utils/dataUtils", () => ({ formatNumberWithCommas: vi.fn((value: number) => value.toLocaleString()), })); +vi.mock("@/app/(dashboard)/hooks/teams/useTeamMetadataSchema", () => ({ + useTeamMetadataSchema: vi.fn(() => ({ data: [], isLoading: false })), +})); + vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({ useAllProxyModels: vi.fn(), })); @@ -220,6 +225,7 @@ describe("TeamInfoView", () => { isFetching: false, refetch: vi.fn(), } as any); + vi.mocked(useTeamMetadataSchema).mockReturnValue({ data: [], isLoading: false } as any); vi.mocked(networking.getGuardrailsList).mockResolvedValue({ guardrails: [] }); vi.mocked(networking.getPoliciesList).mockResolvedValue({ policies: [] }); @@ -893,6 +899,137 @@ describe("TeamInfoView", () => { }); }); + describe("metadata key-value editing", () => { + const openSettingsEditor = async (user: ReturnType) => { + await waitFor(() => { + const teamNameElements = screen.queryAllByText("Test Team"); + expect(teamNameElements.length).toBeGreaterThan(0); + }); + + await user.click(screen.getByRole("tab", { name: "Settings" })); + + await waitFor(() => { + expect(screen.getByRole("button", { name: /edit settings/i })).toBeInTheDocument(); + }); + + await user.click(screen.getByRole("button", { name: /edit settings/i })); + + await waitFor(() => { + expect(screen.getByLabelText("Team Name")).toBeInTheDocument(); + }); + }; + + it("prefills pairs from team metadata, hides UI-managed keys, and round-trips typed values on save", async () => { + const user = userEvent.setup({ delay: null }); + vi.mocked(networking.teamInfoCall).mockResolvedValue( + createMockTeamData({ + metadata: { + department: "research", + tier: 3, + beta: true, + config: { region: "us" }, + logging: [{ callback_name: "langfuse", callback_type: "success", callback_vars: {} }], + guardrails: ["g1"], + disable_global_guardrails: false, + model_tpm_limit: { "gpt-4": 100 }, + }, + models: ["gpt-4"], + }), + ); + vi.mocked(networking.teamUpdateCall).mockResolvedValue({ data: {}, team_id: "123" } as any); + + renderWithProviders(); + await openSettingsEditor(user); + + const keyValues = screen.getAllByPlaceholderText("Key").map((input) => (input as HTMLInputElement).value); + expect(keyValues).toEqual(["department", "tier", "beta", "config"]); + const valueValues = screen.getAllByPlaceholderText("Value").map((input) => (input as HTMLInputElement).value); + expect(valueValues).toEqual(["research", "3", "true", '{"region":"us"}']); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => { + expect(networking.teamUpdateCall).toHaveBeenCalled(); + }); + + const updateArg = vi.mocked(networking.teamUpdateCall).mock.calls[0][1]; + expect(updateArg.metadata).toMatchObject({ + department: "research", + tier: 3, + beta: true, + config: { region: "us" }, + logging: [{ callback_name: "langfuse", callback_type: "success", callback_vars: {} }], + }); + expect(updateArg.metadata).not.toHaveProperty("model_tpm_limit"); + expect(updateArg.model_tpm_limit).toEqual({ "gpt-4": 100 }); + }); + + it("includes a newly added pair in the team update", async () => { + const user = userEvent.setup({ delay: null }); + vi.mocked(networking.teamInfoCall).mockResolvedValue(createMockTeamData({ models: ["gpt-4"] })); + vi.mocked(networking.teamUpdateCall).mockResolvedValue({ data: {}, team_id: "123" } as any); + + renderWithProviders(); + await openSettingsEditor(user); + + await user.click(screen.getByRole("button", { name: /add key-value pair/i })); + await user.type(screen.getByPlaceholderText("Key"), "cost_center"); + await user.type(screen.getByPlaceholderText("Value"), "eng-1"); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => { + expect(networking.teamUpdateCall).toHaveBeenCalled(); + }); + + expect(vi.mocked(networking.teamUpdateCall).mock.calls[0][1].metadata).toMatchObject({ cost_center: "eng-1" }); + }); + + it("should keep declared keys as ordinary prefilled rows and submit the edited value", async () => { + const user = userEvent.setup({ delay: null }); + vi.mocked(useTeamMetadataSchema).mockReturnValue({ + data: [ + { key: "cost_center", label: "Cost Center" }, + { key: "app_name", label: "Application Name" }, + ], + isLoading: false, + } as any); + vi.mocked(networking.teamInfoCall).mockResolvedValue( + createMockTeamData({ + metadata: { cost_center: "CC-OLD", department: "research" }, + models: ["gpt-4"], + }), + ); + vi.mocked(networking.teamUpdateCall).mockResolvedValue({ data: {}, team_id: "123" } as any); + + renderWithProviders(); + await openSettingsEditor(user); + + await waitFor(() => { + expect(screen.getAllByPlaceholderText("Key").map((input) => (input as HTMLInputElement).value)).toEqual([ + "cost_center", + "department", + "app_name", + ]); + }); + expect(screen.getAllByPlaceholderText("Value")[0]).toHaveValue("CC-OLD"); + + await user.clear(screen.getAllByPlaceholderText("Value")[0]); + await user.type(screen.getAllByPlaceholderText("Value")[0], "CC-NEW"); + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => { + expect(networking.teamUpdateCall).toHaveBeenCalled(); + }); + + expect(vi.mocked(networking.teamUpdateCall).mock.calls[0][1].metadata).toMatchObject({ + cost_center: "CC-NEW", + department: "research", + app_name: "", + }); + }); + }); + describe("model aliases", () => { const openSettingsEditor = async (user: ReturnType) => { await waitFor(() => { diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index 34570043f52..bbe5dc05a88 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -35,6 +35,11 @@ import { CheckIcon, CopyIcon } from "lucide-react"; import React, { useEffect, useMemo, useState } from "react"; import { copyToClipboard as utilCopyToClipboard } from "../../utils/dataUtils"; import AccessGroupSelector from "../common_components/AccessGroupSelector"; +import MetadataKeyValueFields, { + metadataObjectToPairs, + metadataPairsToObject, +} from "../common_components/MetadataKeyValueFields"; +import { useTeamMetadataSchema } from "@/app/(dashboard)/hooks/teams/useTeamMetadataSchema"; import ModelAliasManager from "../common_components/ModelAliasManager"; import AgentSelector from "../agent_management/AgentSelector"; import DeleteResourceModal from "../common_components/DeleteResourceModal"; @@ -66,6 +71,18 @@ import { import TeamMembersComponent from "./TeamMemberTab"; import { TeamVirtualKeysTable } from "./TeamVirtualKeysTable"; +const UI_MANAGED_METADATA_KEYS: ReadonlySet = new Set([ + "logging", + "secret_manager_settings", + "soft_budget_alerting_emails", + "model_tpm_limit", + "model_rpm_limit", + "allowed_passthrough_routes", + "guardrails", + "opted_out_global_guardrails", + "disable_global_guardrails", +]); + export interface TeamMembership { user_id: string; team_id: string; @@ -203,6 +220,7 @@ const TeamInfoView: React.FC = ({ const [organization, setOrganization] = useState(null); const { userRole, userId } = useAuthorized(); const { data: userOrganizations = [] } = useOrganizations(); + const { data: teamMetadataSchemaFields = [], isLoading: isTeamMetadataSchemaLoading } = useTeamMetadataSchema(); const queryClient = useQueryClient(); // Check if user is org admin for this team's organization @@ -461,16 +479,7 @@ const TeamInfoView: React.FC = ({ if (!accessToken) return; setIsTeamSaving(true); - let parsedMetadata = {}; - try { - const rawMetadata = values.metadata ? JSON.parse(values.metadata) : {}; - // Exclude soft_budget_alerting_emails from parsed metadata since it's handled separately - const { soft_budget_alerting_emails, ...rest } = rawMetadata; - parsedMetadata = rest; - } catch (e) { - NotificationsManager.fromBackend("Invalid JSON in metadata field"); - return; - } + const parsedMetadata = metadataPairsToObject(values.metadata); let secretManagerSettings: Record | undefined; if (typeof values.secret_manager_settings === "string") { @@ -980,21 +989,7 @@ const TeamInfoView: React.FC = ({ soft_budget_alerting_emails: Array.isArray(info.metadata?.soft_budget_alerting_emails) ? info.metadata.soft_budget_alerting_emails.join(", ") : "", - metadata: info.metadata - ? JSON.stringify( - (({ - logging, - secret_manager_settings, - soft_budget_alerting_emails, - model_tpm_limit, - model_rpm_limit, - allowed_passthrough_routes, - ...rest - }) => rest)(info.metadata), - null, - 2, - ) - : "", + metadata: metadataObjectToPairs(info.metadata, UI_MANAGED_METADATA_KEYS), logging_settings: info.metadata?.logging || [], secret_manager_settings: info.metadata?.secret_manager_settings ? JSON.stringify(info.metadata.secret_manager_settings, null, 2) @@ -1170,6 +1165,17 @@ const TeamInfoView: React.FC = ({ + + + + = ({ /> - - - -
diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index f7cc9a1deae..8879844c24a 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -1,4 +1,4 @@ -import { renderWithProviders, screen, waitFor } from "../../../tests/test-utils"; +import { renderWithProviders, screen, waitFor, within } from "../../../tests/test-utils"; import userEvent from "@testing-library/user-event"; import { vi } from "vitest"; import AddAutoRouterTab from "./add_auto_router_tab"; @@ -52,14 +52,22 @@ describe("AddAutoRouterTab", () => { vi.clearAllMocks(); }); - it("flags every mandatory field when Add Auto Router is clicked with nothing filled", async () => { + // Nothing is filled in, so there is nothing to submit. The button reports that itself instead of + // accepting a click and answering with a toast. + it("offers no submit at all until every tier has a model", async () => { + renderWithProviders(); + + expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled(); + }); + + it("still flags the router name once the config no longer blocks the submit", async () => { const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); renderWithProviders(); await user.click(screen.getByRole("button", { name: /add auto router/i })); expect(await screen.findByText("Auto router name is required")).toBeInTheDocument(); - expect(screen.getAllByText("This tier is required")).toHaveLength(4); expect(NotificationManager.fromBackend).toHaveBeenCalledWith("Please enter an Auto Router Name"); }); @@ -97,6 +105,83 @@ describe("AddAutoRouterTab", () => { expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0]).toMatchObject({ team_id: "team-1" }); }); + // LIT-5133: "Add keyword rule" seeds a row with no keywords, and the semantic toggle that used + // to be the only thing checking them is off by default. The row was dropped on the way to the + // payload, so the create succeeded and the caller's rule was gone with nothing said about it. + it("takes the submit away while a keyword rule is left empty", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router"); + await user.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + await user.click(screen.getByRole("button", { name: /add keyword rule/i })); + + expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled(); + // The row says so on its own; there is no failed submit left to surface it. + expect(await screen.findByText("At least one keyword is required")).toBeInTheDocument(); + expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled(); + }); + + it("gives the submit back once that keyword rule is filled", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router"); + await user.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + await user.click(screen.getByRole("button", { name: /add keyword rule/i })); + expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled(); + + await user.type( + within(screen.getByText("Keywords 1").closest("div") as HTMLElement).getByRole("combobox"), + "invoice{enter}", + ); + + expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled(); + expect(screen.queryByText("At least one keyword is required")).not.toBeInTheDocument(); + }); + + it("marks only the offending keyword row, leaving a filled one alone", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router"); + await user.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + await user.click(screen.getByRole("button", { name: /add keyword rule/i })); + await user.type( + within(screen.getByText("Keywords 1").closest("div") as HTMLElement).getByRole("combobox"), + "invoice{enter}", + ); + await user.click(screen.getByRole("button", { name: /add keyword rule/i })); + + expect(await screen.findAllByText("At least one keyword is required")).toHaveLength(1); + expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled(); + }); + + it("creates the router once that keyword rule is filled in", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router"); + await user.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + await user.click(screen.getByRole("button", { name: /add keyword rule/i })); + const keywordsField = screen.getByText("Keywords 1").closest("div") as HTMLElement; + await user.type(within(keywordsField).getByRole("combobox"), "invoice{enter}"); + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0]).toMatchObject({ + complexity_router_config: { keyword_tier_rules: [{ keywords: ["invoice"], tier: "COMPLEX" }] }, + }); + }); + it("blocks the submit when a team admin has not picked a team", async () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index ea75bd8e283..ae90d42ba8a 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -18,6 +18,7 @@ import { DEFAULT_ESCALATION_KEYWORDS } from "./EscalationKeywords"; import { DEFAULT_MATCH_THRESHOLD } from "./SemanticKeywordMatching"; import { buildComplexityRouterConfig, + getKeywordTierRulesError, getMissingTiersError, getSemanticConfigError, } from "./build_complexity_router_config"; @@ -95,6 +96,11 @@ const AddAutoRouterTab: React.FC = ({ label: model_group, })); + // Why the submit is unavailable, or null when it is available. The button reads this to disable + // itself and to say what is missing, so the two can never give different answers. + const submitBlockedReason = + getMissingTiersError(complexityRouterConfig.tiers) ?? getKeywordTierRulesError(keywordTierRules); + const submitRecommendedRouter = (name: string) => { const { tiers, @@ -124,6 +130,13 @@ const AddAutoRouterTab: React.FC = ({ return; } + const keywordRulesError = getKeywordTierRulesError(keywordTierRules); + if (keywordRulesError) { + setShowValidationErrors(true); + NotificationManager.fromBackend(keywordRulesError); + return; + } + const semanticError = getSemanticConfigError({ semanticMatchingEnabled, embeddingModel, keywordTierRules }); if (semanticError) { setShowValidationErrors(true); @@ -310,14 +323,17 @@ const AddAutoRouterTab: React.FC = ({ Test Connection } - + + +
diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 9d784b57903..4cbe54ad4a6 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -1,5 +1,6 @@ import { buildComplexityRouterConfig, + getKeywordTierRulesError, getMissingTiersError, getSemanticConfigError, BuildComplexityRouterConfigParams, @@ -183,7 +184,7 @@ describe("buildComplexityRouterConfig", () => { expect(config.keyword_tier_rules).toBeUndefined(); }); - it("trims keywords and drops rules left empty, so unfilled rows never 400 the backend", () => { + it("trims keywords but keeps rules left empty, so a dropped row can never pass for a saved one", () => { const params: BuildComplexityRouterConfigParams = { ...baseParams, keywordTierRules: [ @@ -193,17 +194,13 @@ describe("buildComplexityRouterConfig", () => { ], }; const config = buildComplexityRouterConfig(params); - // r1 keeps only its real keyword (trimmed); r2 and r3 are dropped entirely. - expect(config.keyword_tier_rules).toEqual([{ keywords: ["deploy to k8s"], tier: "REASONING" }]); - }); - - it("omits keyword_tier_rules entirely when every rule is empty", () => { - const params: BuildComplexityRouterConfigParams = { - ...baseParams, - keywordTierRules: [{ id: "r1", keywords: ["", " "], tier: "COMPLEX" }], - }; - const config = buildComplexityRouterConfig(params); - expect(config.keyword_tier_rules).toBeUndefined(); + // getKeywordTierRulesError blocks this submit; r2 and r3 survive here so the backend rejects + // them loudly rather than the caller's rows vanishing on a successful save. + expect(config.keyword_tier_rules).toEqual([ + { keywords: ["deploy to k8s"], tier: "REASONING" }, + { keywords: [], tier: "COMPLEX" }, + { keywords: [], tier: "SIMPLE" }, + ]); }); it("omits adaptive fields when adaptive is disabled even if weights linger in state", () => { @@ -318,17 +315,6 @@ describe("getSemanticConfigError", () => { ).toMatch(/keyword tier rule/i); }); - it("errors when a rule has no non-empty keywords", () => { - const emptyRule = { id: "r2", keywords: ["", " "], tier: "SIMPLE" as const }; - expect( - getSemanticConfigError({ - semanticMatchingEnabled: true, - embeddingModel: "voyage-3-5", - keywordTierRules: [emptyRule], - }), - ).toMatch(/at least one keyword/i); - }); - it("returns null when enabled with both an embedding model and rules", () => { expect( getSemanticConfigError({ semanticMatchingEnabled: true, embeddingModel: "voyage-3-5", keywordTierRules: [rule] }), @@ -336,6 +322,52 @@ describe("getSemanticConfigError", () => { }); }); +describe("getKeywordTierRulesError", () => { + it("returns null when every rule carries a keyword", () => { + expect( + getKeywordTierRulesError([ + { id: "r1", keywords: ["invoice"], tier: "MEDIUM" }, + { id: "r2", keywords: ["deploy to k8s"], tier: "REASONING" }, + ]), + ).toBeNull(); + }); + + it("returns null when there are no rules at all, since the section is optional", () => { + expect(getKeywordTierRulesError([])).toBeNull(); + }); + + // The whole point of the ticket: the semantic toggle is off by default, and an unfilled row + // used to be discarded silently on an otherwise successful create. + it("rejects a row left empty while semantic matching is off", () => { + expect(getKeywordTierRulesError([{ id: "r1", keywords: [], tier: "COMPLEX" }])).toBe( + "Add at least one keyword to keyword rule(s): 1", + ); + }); + + it.each([ + ["whitespace only", [" "]], + ["blank strings, as an unfilled row between filled ones leaves behind", ["", " ", ""]], + ])("treats %s as empty rather than as a keyword", (_label, keywords) => { + expect(getKeywordTierRulesError([{ id: "r1", keywords, tier: "SIMPLE" }])).toMatch(/keyword rule\(s\): 1/); + }); + + // Row numbers have to survive rules that are fine, or the message points at the wrong input. + it("names each offending row by its position among all rules", () => { + expect( + getKeywordTierRulesError([ + { id: "r1", keywords: ["invoice"], tier: "MEDIUM" }, + { id: "r2", keywords: [], tier: "COMPLEX" }, + { id: "r3", keywords: ["billing"], tier: "SIMPLE" }, + { id: "r4", keywords: [" "], tier: "REASONING" }, + ]), + ).toBe("Add at least one keyword to keyword rule(s): 2, 4"); + }); + + it("keeps a keyword whose surrounding whitespace is the only thing trimmed", () => { + expect(getKeywordTierRulesError([{ id: "r1", keywords: [" invoice "], tier: "MEDIUM" }])).toBeNull(); + }); +}); + describe("buildComplexityRouterConfig assistant turns", () => { const llmParams: BuildComplexityRouterConfigParams = { ...baseParams, diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index cd6c697b377..dcec58479a6 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -1,5 +1,5 @@ import { KeywordTierRule } from "./KeywordTierRules"; -import { serializeKeywordTierRules } from "./complexity_router_keywords"; +import { emptyKeywordTierRuleIndexes, serializeKeywordTierRules } from "./complexity_router_keywords"; import { AdaptiveEligible, AdaptiveRouterWeights, @@ -58,6 +58,12 @@ export const getMissingTiersError = (tiers: ComplexityTiers): string | null => { return `Select a model for the following tier(s): ${missing.join(", ")}`; }; +export const getKeywordTierRulesError = (keywordTierRules: KeywordTierRule[]): string | null => { + const emptyRows = emptyKeywordTierRuleIndexes(keywordTierRules); + if (emptyRows.length === 0) return null; + return `Add at least one keyword to keyword rule(s): ${emptyRows.map((index) => index + 1).join(", ")}`; +}; + export const getSemanticConfigError = ({ semanticMatchingEnabled, embeddingModel, @@ -68,8 +74,6 @@ export const getSemanticConfigError = ({ if (!semanticMatchingEnabled) return null; if (!embeddingModel) return "Select an embedding model to use semantic keyword matching"; if (keywordTierRules.length === 0) return "Add at least one keyword tier rule to use semantic keyword matching"; - if (keywordTierRules.some((rule) => !rule.keywords.some((keyword) => keyword.trim()))) - return "Every keyword tier rule needs at least one keyword"; return null; }; @@ -94,7 +98,6 @@ export const buildComplexityRouterConfig = ({ returnRawModelName, }: BuildComplexityRouterConfigParams): ComplexityRouterConfigPayload => { const cleanedEscalationKeywords = escalationKeywords.map((keyword) => keyword.trim()).filter(Boolean); - // Trim keywords and drop empty ones; drop any rule left with no keywords. Clicking const cleanedKeywordTierRules = serializeKeywordTierRules(keywordTierRules); return { diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_keywords.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_keywords.ts index 9cfdaed4e23..6fe93cddae3 100644 --- a/ui/litellm-dashboard/src/components/add_model/complexity_router_keywords.ts +++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_keywords.ts @@ -19,13 +19,19 @@ const asKeywords = (value: unknown): string[] => : []; /** - * Drop the React-only id, trim keywords, and discard rules left empty. "Add keyword rule" - * seeds a row with no keywords, and the backend validator rejects those with a 400. + * Drop the React-only id and trim keywords, leaving one entry per rule. A rule left empty stays + * empty rather than disappearing, so getKeywordTierRulesError can name the row it came from. */ export const serializeKeywordTierRules = (rules: KeywordTierRule[]): StoredKeywordTierRule[] => - rules - .map((rule) => ({ keywords: asKeywords(rule.keywords).filter(Boolean), tier: rule.tier })) - .filter((rule) => rule.keywords.length > 0); + rules.map((rule) => ({ keywords: asKeywords(rule.keywords).filter(Boolean), tier: rule.tier })); + +/** + * Positions of the rules left without a keyword, as indexes into the caller's own array. The + * submit-time message and the inline error on the row both read this, so the row the message + * names is always the row that lights up. + */ +export const emptyKeywordTierRuleIndexes = (rules: KeywordTierRule[]): number[] => + serializeKeywordTierRules(rules).flatMap((rule, index) => (rule.keywords.length === 0 ? [index] : [])); export const hydrateKeywordTierRules = (value: unknown): KeywordTierRule[] => { if (!Array.isArray(value)) return []; diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index 971c833a0de..818dcd1f648 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -54,13 +54,16 @@ describe("buildUpdatedComplexityRouterConfig keyword matching", () => { expect(result.keyword_tier_rules).toEqual([{ keywords: ["chargeback"], tier: "COMPLEX" }]); }); - it("drops a rule left empty rather than shipping one the backend 400s on", () => { + // getKeywordTierRulesError blocks this save, so the builder never runs on a real edit. Keeping + // the rule here means that if a caller ever reaches it anyway, the stored rules are replaced by + // something the backend rejects out loud rather than by silence that reads as a clean save. + it("keeps a rule left empty rather than quietly dropping the caller's row", () => { const result = buildUpdatedComplexityRouterConfig(STORED, FORM_VALUE, undefined, { ...hydratedState, keywordTierRules: [{ id: "new-1", keywords: [" "], tier: "SIMPLE" }], }); - expect(result.keyword_tier_rules).toBeUndefined(); + expect(result.keyword_tier_rules).toEqual([{ keywords: [], tier: "SIMPLE" }]); }); it("removes the semantic trio when the toggle is turned off", () => { diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx index c0806befa52..3976e4c1381 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx @@ -118,6 +118,80 @@ describe("EditAutoRouterModal keyword matching", () => { await waitFor(() => expect(NotificationsManager.fromBackend).toHaveBeenCalled()); expect(modelPatchUpdateCall).not.toHaveBeenCalled(); }); + + // LIT-5133, edit side. Semantic matching is off here on purpose: it used to be the only thing + // that checked a rule for keywords, so with it on this save was already blocked and the test + // would pass without the fix. Off, the unfilled row was dropped and the save reported success. + it("blocks a save that adds a keyword rule and leaves it empty", async () => { + const user = userEvent.setup(); + renderWithProviders( + , + ); + + await screen.findByText(/Escalation Keywords/i); + fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + await user.click(screen.getByRole("button", { name: /add keyword rule/i })); + + // The modal renders the same controls as the create form, so it owes the same treatment: + // the row says what is missing and the save is not offered while it is. + expect(await screen.findByText("At least one keyword is required")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /save changes/i })).toBeDisabled(); + expect(modelPatchUpdateCall).not.toHaveBeenCalled(); + }); + + it("gives the save back once the added keyword rule is filled", async () => { + const user = userEvent.setup(); + renderWithProviders( + , + ); + + await screen.findByText(/Escalation Keywords/i); + fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + await user.click(screen.getByRole("button", { name: /add keyword rule/i })); + expect(screen.getByRole("button", { name: /save changes/i })).toBeDisabled(); + + await user.type( + within(screen.getByText("Keywords 2").closest("div") as HTMLElement).getByRole("combobox"), + "chargeback{enter}", + ); + + expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled(); + expect(screen.queryByText("At least one keyword is required")).not.toBeInTheDocument(); + }); }); describe("EditAutoRouterModal classifier context window", () => { diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index a70fc31d6fe..2c58cd70cb9 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -1,12 +1,12 @@ import React, { useEffect, useState } from "react"; -import { Modal, Form, Button, Select as AntdSelect } from "antd"; +import { Modal, Form, Button, Select as AntdSelect, Tooltip } from "antd"; import { Text, TextInput } from "@tremor/react"; import { modelAvailableCall, modelPatchUpdateCall } from "../networking"; import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; import RouterConfigBuilder from "../add_model/RouterConfigBuilder"; import { normalizeTierModels } from "../add_model/complexity_router_tiers"; import { isComplexityRouter } from "../add_model/auto_router_strategies"; -import { getSemanticConfigError } from "../add_model/build_complexity_router_config"; +import { getKeywordTierRulesError, getSemanticConfigError } from "../add_model/build_complexity_router_config"; import { KeywordTierRule } from "../add_model/KeywordTierRules"; import { DEFAULT_MATCH_THRESHOLD } from "../add_model/SemanticKeywordMatching"; import { hydrateKeywordTierRules, serializeKeywordTierRules } from "../add_model/complexity_router_keywords"; @@ -118,8 +118,8 @@ export const buildUpdatedComplexityRouterConfig = ( }), ...(value.return_raw_model_name && { return_raw_model_name: true }), ...(keywordMatching && { - // Mirrors buildComplexityRouterConfig: rules only when non-empty (the backend rejects - // an empty rule with a 400), escalation keywords always, semantic trio only when on. + // Mirrors buildComplexityRouterConfig: the key only when there is a rule to write, + // escalation keywords always, semantic trio only when on. ...(storedKeywordRules.length > 0 && { keyword_tier_rules: storedKeywordRules }), escalation_keywords: keywordMatching.escalationKeywords.map((k) => k.trim()).filter(Boolean), ...(keywordMatching.semanticMatchingEnabled && { @@ -145,6 +145,7 @@ const EditAutoRouterModal: React.FC = ({ const [modelInfo, setModelInfo] = useState([]); const [showCustomDefaultModel, setShowCustomDefaultModel] = useState(false); const [showCustomEmbeddingModel, setShowCustomEmbeddingModel] = useState(false); + const [showValidationErrors, setShowValidationErrors] = useState(false); const [routerConfig, setRouterConfig] = useState(null); const [customTechnicalKeywords, setCustomTechnicalKeywords] = useState([]); const [keywordTierRules, setKeywordTierRules] = useState([]); @@ -158,6 +159,15 @@ const EditAutoRouterModal: React.FC = ({ }); const isComplexityRouterModel = isComplexityRouter(modelData?.litellm_params); + // Mirrors the create form: the button says why it is unavailable and disables on the same + // answer. Tiers use this modal's own rule, which allows a partly filled router, so an edit that + // is legal today stays legal. + const submitBlockedReason = !isComplexityRouterModel + ? null + : (Object.values(complexityRouterConfig.tiers).every((models) => models.length === 0) + ? "Please select at least one model for a complexity tier" + : null) ?? getKeywordTierRulesError(keywordTierRules); + useEffect(() => { if (isVisible && modelData) { initializeForm(); @@ -295,24 +305,29 @@ const EditAutoRouterModal: React.FC = ({ if (isComplexityRouterModel) { const { tiers, classifier_type, classifier_llm_config } = complexityRouterConfig; if (Object.values(tiers).every((models) => models.length === 0)) { + setShowValidationErrors(true); NotificationsManager.fromBackend("Please select at least one model for a complexity tier"); return; } if (classifier_type === "llm" && !classifier_llm_config?.model) { + setShowValidationErrors(true); NotificationsManager.fromBackend("Please select a classifier model, or switch back to Heuristic"); return; } - // Same guard the create form applies (add_auto_router_tab.tsx). The backend rejects - // semantic_keyword_matching without an embedding model or keyword rules - // (complexity_router/config.py), so without this a save fails as a raw 400 instead of - // an inline message. + // Same guards the create form applies (add_auto_router_tab.tsx). The backend rejects a + // keyword rule with no keyword, and semantic_keyword_matching without an embedding model + // or keyword rules (complexity_router/config.py), so without these a save fails as a raw + // 400 instead of an inline message. + const keywordRulesError = getKeywordTierRulesError(keywordTierRules); + if (keywordRulesError) { + setShowValidationErrors(true); + NotificationsManager.fromBackend(keywordRulesError); + return; + } - // Same guard the create form applies (add_auto_router_tab.tsx). The backend rejects - // semantic_keyword_matching without an embedding model or keyword rules - // (complexity_router/config.py), so without this a save fails as a raw 400 instead of - // an inline message. const semanticError = getSemanticConfigError({ semanticMatchingEnabled, embeddingModel, keywordTierRules }); if (semanticError) { + setShowValidationErrors(true); NotificationsManager.fromBackend(semanticError); return; } @@ -410,9 +425,11 @@ const EditAutoRouterModal: React.FC = ({ , - , + + + , ]} width={1000} destroyOnHidden @@ -436,6 +453,7 @@ const EditAutoRouterModal: React.FC = ({ /* Complexity Router Configuration */
{ From 2d1f650e9a5a79a3e67b0e9bbe1d4f1717de3aee Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 3 Aug 2026 19:56:24 -0700 Subject: [PATCH 110/123] fix(guardrails/rubrik): attribute blocked requests to the caller that made them (#35734) The block event Rubrik receives sourced caller identity from model_call_details[metadata], where the enriched litellm metadata never lives; it sits under litellm_params. Every block therefore reported user_api_key_hash as an empty string, so a security block could not be traced to a key, user, or team. Read identity off the authenticated UserAPIKeyAuth the failure hook is already handed, via the same mapper the success path and the proxy spend logger use, so a block log and a success log describe their caller with an identical key set. --- basedpyright-code-budget.json | 4 +- litellm/integrations/rubrik.py | 68 +++++--- ruff-strict-budget.json | 2 +- .../test_litellm/integrations/test_rubrik.py | 157 +++++++++++++++--- type-discipline-budget.json | 2 +- 5 files changed, 188 insertions(+), 45 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index f6dd90077b1..90e0a283c63 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 29813 + "limit": 29811 }, "reportArgumentType": { "limit": 2645 @@ -24,7 +24,7 @@ "limit": 42 }, "reportExplicitAny": { - "limit": 9473 + "limit": 9471 }, "reportFunctionMemberAccess": { "limit": 11 diff --git a/litellm/integrations/rubrik.py b/litellm/integrations/rubrik.py index 4bcbe8bae37..9942776bc00 100644 --- a/litellm/integrations/rubrik.py +++ b/litellm/integrations/rubrik.py @@ -34,12 +34,14 @@ from litellm.types.utils import ( Function, GenericGuardrailAPIInputs, StandardLoggingPayload, + StandardLoggingUserAPIKeyMetadata, ) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import ( Logging as LiteLLMLoggingObj, ) + from litellm.proxy._types import UserAPIKeyAuth _WEBHOOK_PATH_RESPONSE_MODERATION = "/v1/after_completion/openai/v1" _WEBHOOK_PATH_PROMPT_MODERATION = "/v1/before_prompt/openai/v1" @@ -725,7 +727,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): self, request_data: dict, original_exception: Exception, - user_api_key_dict: Any, + user_api_key_dict: "UserAPIKeyAuth", traceback_str: str | None = None, ) -> None: """Log blocked requests signalled via ``ModifyResponseException`` @@ -755,20 +757,21 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): "Rubrik: block exception without stashed logging_obj. " f"litellm_call_id={request_data.get('litellm_call_id')}, " f"model={request_data.get('model')}, " - f"user_id={getattr(user_api_key_dict, 'user_id', None)}, " + f"user_id={user_api_key_dict.user_id}, " f"raising_guardrail=" f"{getattr(original_exception, 'guardrail_name', None)}" ) return call_id: str | None = None - await self._build_and_enqueue_block_event(logging_obj, original_exception, call_id) + await self._build_and_enqueue_block_event(logging_obj, original_exception, call_id, user_api_key_dict) async def _build_and_enqueue_block_event( self, logging_obj: "LiteLLMLoggingObj", exception: "ModifyResponseException", call_id: str | None, + user_api_key_dict: "UserAPIKeyAuth", ) -> None: try: call_details = logging_obj.model_call_details @@ -780,8 +783,8 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): # prevent. The flag dies with model_call_details when the request # completes; there's nothing to clean up. call_id = call_details.get("litellm_call_id") - payload = self._prepare_block_failure_payload(logging_obj, exception) - except (AttributeError, KeyError, TypeError) as e: + payload = self._prepare_block_failure_payload(logging_obj, exception, user_api_key_dict) + except (AttributeError, ImportError, KeyError, TypeError) as e: verbose_logger.error( f"Rubrik: failed to build blocked-tool payload for " f"litellm_call_id={call_id}: {e}. Event will NOT be logged.", @@ -801,17 +804,20 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): self, logging_obj: "LiteLLMLoggingObj", exception: "ModifyResponseException", + user_api_key_dict: "UserAPIKeyAuth", ) -> StandardLoggingPayload: """Build a failure-style payload using the exception text as response. Blocked-tool events are security-relevant and **bypass sampling**: every block is logged. - The deferred success-handler runs as a separately-scheduled task and - races with this hook, so ``standard_logging_object`` on - ``model_call_details`` may not yet be populated. If present we reuse - it; otherwise we fall back to a best-effort payload built from the - fields available at block time. + A non-streaming block always takes the fallback, and not because of a + race: registering a post_call guardrail sets ``_defer_async_logging``, + which parks the success handler that would have written + ``standard_logging_object``, and ``_flush_deferred_async_logging`` + returns early once an exception was raised. Streaming requests never set + that flag, so a streamed block can arrive with the object already + populated; the branch below covers it and wins over the fallback. For prompt blocks the LLM is never called, so ``standard_logging_object`` is never populated. The fallback therefore must carry enough fields to @@ -830,9 +836,11 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): ``acompletion()``, which hasn't run yet for a prompt block. - ``model_id``: not available before the LLM returns hidden_params; defaults to empty string. - - ``user_api_key_hash``: ``call_details["metadata"]["user_api_key"]`` -- - the hashed token written by ``add_user_information_to_request_data`` - before ``pre_call_hook`` fires. + - caller identity: ``_caller_metadata`` off the ``user_api_key_dict`` + the failure hook is handed. The enriched litellm metadata lives under + ``call_details["litellm_params"]["metadata"]``, never at the top + level, so the previous top-level read resolved to an empty string for + every block. - time fields: ``call_details["start_time"]`` reused for all three; end/completion times are meaningless for a prompt block. """ @@ -848,7 +856,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): f"for litellm_call_id={call_details.get('litellm_call_id')}; " "using best-effort fallback payload." ) - payload = self._build_fallback_payload(call_details) + payload = self._build_fallback_payload(call_details, user_api_key_dict) payload["response"] = exception_text @@ -863,8 +871,30 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): return payload # type: ignore[return-value] @staticmethod - def _build_fallback_payload(call_details: Mapping[str, Any]) -> dict[str, Any]: - _metadata: Mapping[str, Any] = call_details.get("metadata") or _EMPTY_MAPPING + def _caller_metadata(user_api_key_dict: "UserAPIKeyAuth") -> StandardLoggingUserAPIKeyMetadata: + """Identify the caller whose request was blocked. + + Uses the same mapper the success path and the proxy spend logger use, so + a block log and a success log agree on the caller key set. + + The import is deferred because ``litellm/integrations/`` is SDK-side + while the mapper lives under ``proxy/``: ``rubrik.py`` is imported during + guardrail discovery and must not pull proxy-only dependencies into its + import chain. It is unguarded because the only dispatcher of this hook, + ``ProxyLogging.post_call_failure_hook``, already imports fastapi at + module scope, so there is no path where this hook runs and the mapper is + missing. + """ + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + return LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict) + + @classmethod + def _build_fallback_payload( + cls, + call_details: Mapping[str, Any], + user_api_key_dict: "UserAPIKeyAuth", + ) -> dict[str, Any]: # Convert datetime to a Unix float so json.dumps can serialize it. # httpx's json= parameter uses stdlib json.dumps with no custom encoder. _raw_start = call_details.get("start_time") @@ -884,11 +914,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): "endTime": _start, "completionStartTime": _start, "messages": call_details.get("messages") or (), - "metadata": { - # "user_api_key" is the hashed token written by - # add_user_information_to_request_data before guardrails fire. - "user_api_key_hash": _metadata.get("user_api_key_hash") or _metadata.get("user_api_key") or "", - }, + "metadata": cls._caller_metadata(user_api_key_dict), "status": "failure", } diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index d3ef01940bb..003e09de0c2 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -24,7 +24,7 @@ "limit": 130 }, "ANN401": { - "limit": 1851 + "limit": 1850 }, "ASYNC230": { "limit": 14 diff --git a/tests/test_litellm/integrations/test_rubrik.py b/tests/test_litellm/integrations/test_rubrik.py index 7f589dc15bf..4a2ee487c65 100644 --- a/tests/test_litellm/integrations/test_rubrik.py +++ b/tests/test_litellm/integrations/test_rubrik.py @@ -17,6 +17,7 @@ from litellm.integrations.rubrik import ( RubrikLogger, _MalformedToolBlockingResponseError, ) +from litellm.proxy._types import UserAPIKeyAuth from tests.test_litellm.integrations.rubrik_test_helpers import ( make_inputs_with_tools, @@ -44,6 +45,18 @@ def handler(mock_env): return RubrikLogger() +@pytest.fixture +def user_api_key_dict(): + """The authenticated caller the proxy hands to async_post_call_failure_hook.""" + return UserAPIKeyAuth( + api_key="sk-block-attribution-test", + key_alias="rubrik-probe-key", + user_id="probe-user-1", + team_id="probe-team-1", + org_id="probe-org-1", + ) + + # -- Initialization ----------------------------------------------------------- @@ -1544,17 +1557,17 @@ class TestSuccessEventBlockedSkip: @pytest.mark.asyncio class TestPostCallFailureHook: - async def test_non_modify_exception_returns_immediately(self, handler): + async def test_non_modify_exception_returns_immediately(self, handler, user_api_key_dict): """Non-ModifyResponseException causes a no-op.""" await handler.async_post_call_failure_hook( request_data={"litellm_call_id": "test"}, original_exception=ValueError("unrelated error"), - user_api_key_dict=None, + user_api_key_dict=user_api_key_dict, ) assert len(handler.log_queue) == 0 async def test_modify_exception_without_stashed_logging_obj_emits_warning( - self, handler + self, handler, user_api_key_dict ): """ModifyResponseException with no _rubrik_logging_obj → warning, no enqueue.""" request_data = {"litellm_call_id": "test-123", "model": "gpt-4"} @@ -1568,12 +1581,12 @@ class TestPostCallFailureHook: await handler.async_post_call_failure_hook( request_data=request_data, original_exception=exc, - user_api_key_dict=None, + user_api_key_dict=user_api_key_dict, ) assert len(handler.log_queue) == 0 async def test_modify_exception_with_valid_logging_obj_enqueues_payload( - self, handler + self, handler, user_api_key_dict ): """ModifyResponseException + stashed logging_obj → builds and enqueues.""" logging_obj = Mock() @@ -1602,12 +1615,12 @@ class TestPostCallFailureHook: await handler.async_post_call_failure_hook( request_data=request_data, original_exception=exc, - user_api_key_dict=None, + user_api_key_dict=user_api_key_dict, ) assert len(handler.log_queue) == 1 assert "ModifyResponseException" in handler.log_queue[0]["response"] - async def test_logging_obj_popped_from_request_data(self, handler): + async def test_logging_obj_popped_from_request_data(self, handler, user_api_key_dict): """_rubrik_logging_obj must be popped from request_data so it is not forwarded downstream.""" logging_obj = Mock() @@ -1636,12 +1649,12 @@ class TestPostCallFailureHook: await handler.async_post_call_failure_hook( request_data=request_data, original_exception=exc, - user_api_key_dict=None, + user_api_key_dict=user_api_key_dict, ) assert "_rubrik_logging_obj" not in request_data async def test_build_and_enqueue_swallows_attribute_error_from_prepare_payload( - self, handler + self, handler, user_api_key_dict ): """When _prepare_block_failure_payload raises AttributeError/KeyError/TypeError, the error is logged and the event is silently dropped (lines 806-812).""" @@ -1657,10 +1670,10 @@ class TestPostCallFailureHook: ) # Must not raise - await handler._build_and_enqueue_block_event(logging_obj, exc, None) + await handler._build_and_enqueue_block_event(logging_obj, exc, None, user_api_key_dict) assert len(handler.log_queue) == 0 - async def test_build_and_enqueue_swallows_flush_exception(self, handler): + async def test_build_and_enqueue_swallows_flush_exception(self, handler, user_api_key_dict): """When _append_and_maybe_flush raises, the error is logged (lines 816-817).""" logging_obj = Mock() logging_obj.model_call_details = { @@ -1688,14 +1701,14 @@ class TestPostCallFailureHook: ) # Must not raise - await handler._build_and_enqueue_block_event(logging_obj, exc, None) + await handler._build_and_enqueue_block_event(logging_obj, exc, None, user_api_key_dict) # -- _prepare_block_failure_payload and _build_fallback_payload --------------- class TestPrepareBlockFailurePayload: - def test_uses_standard_logging_object_when_present(self, handler): + def test_uses_standard_logging_object_when_present(self, handler, user_api_key_dict): """When standard_logging_object is on model_call_details, it is used as base.""" logging_obj = Mock() logging_obj.model_call_details = { @@ -1716,12 +1729,37 @@ class TestPrepareBlockFailurePayload: guardrail_name="rubrik", ) - payload = handler._prepare_block_failure_payload(logging_obj, exc) + payload = handler._prepare_block_failure_payload(logging_obj, exc, user_api_key_dict) assert "ModifyResponseException: blocked" in payload["response"] assert payload["id"] == "call-slo" - def test_uses_fallback_when_standard_logging_object_absent(self, handler): + def test_standard_logging_object_identity_is_not_overwritten(self, handler, user_api_key_dict): + """A streamed block can arrive with the object populated; it keeps its own identity.""" + logging_obj = Mock() + logging_obj.model_call_details = { + "litellm_call_id": "call-slo-identity", + "model": "gpt-4", + "standard_logging_object": { + "id": "chatcmpl-original", + "model": "gpt-4", + "response": "original response", + "messages": [], + "metadata": {"user_api_key_hash": "hash-from-standard-logging-object"}, + }, + } + exc = ModifyResponseException( + message="blocked", + model="gpt-4", + request_data={}, + guardrail_name="rubrik", + ) + + payload = handler._prepare_block_failure_payload(logging_obj, exc, user_api_key_dict) + + assert payload["metadata"]["user_api_key_hash"] == "hash-from-standard-logging-object" + + def test_uses_fallback_when_standard_logging_object_absent(self, handler, user_api_key_dict): """When standard_logging_object is absent, _build_fallback_payload is used.""" from datetime import datetime @@ -1731,7 +1769,7 @@ class TestPrepareBlockFailurePayload: "model": "claude-3", "messages": [{"role": "user", "content": "question"}], "optional_params": {"temperature": 0.5}, - "metadata": {"user_api_key_hash": "hash-abc"}, + "metadata": {"headers": {"host": "127.0.0.1:4000", "user-agent": "curl/8.7.1"}}, "start_time": datetime(2024, 6, 1), } exc = ModifyResponseException( @@ -1741,16 +1779,15 @@ class TestPrepareBlockFailurePayload: guardrail_name="rubrik", ) - payload = handler._prepare_block_failure_payload(logging_obj, exc) + payload = handler._prepare_block_failure_payload(logging_obj, exc, user_api_key_dict) assert payload["id"] == "call-fallback" assert payload["model"] == "claude-3" assert payload["model_group"] == "claude-3" assert "ModifyResponseException: prompt blocked" in payload["response"] - assert payload["metadata"]["user_api_key_hash"] == "hash-abc" assert payload["status"] == "failure" - def test_fallback_payload_without_start_time(self, handler): + def test_fallback_payload_without_start_time(self, handler, user_api_key_dict): """_build_fallback_payload handles missing start_time gracefully.""" logging_obj = Mock() logging_obj.model_call_details = { @@ -1767,10 +1804,90 @@ class TestPrepareBlockFailurePayload: guardrail_name="rubrik", ) - payload = handler._prepare_block_failure_payload(logging_obj, exc) + payload = handler._prepare_block_failure_payload(logging_obj, exc, user_api_key_dict) assert payload["startTime"] is None +class TestBlockPayloadCallerAttribution: + """A block log must identify the caller that triggered it. + + The enriched litellm metadata lives under + ``model_call_details["litellm_params"]["metadata"]``, never at the top + level, so sourcing identity from ``call_details["metadata"]`` yielded an + empty string for every block. Identity comes from the authenticated + ``user_api_key_dict`` the failure hook is handed. + """ + + def _blocked_call_details(self): + return { + "litellm_call_id": "call-attr", + "model": "claude-3", + "messages": [{"role": "user", "content": "question"}], + "optional_params": {}, + "metadata": {"headers": {"host": "127.0.0.1:4000", "user-agent": "curl/8.7.1"}}, + } + + def test_fallback_payload_identifies_the_caller(self, handler, user_api_key_dict): + payload = handler._build_fallback_payload(self._blocked_call_details(), user_api_key_dict) + + metadata = payload["metadata"] + assert metadata["user_api_key_hash"] == user_api_key_dict.api_key + assert metadata["user_api_key_alias"] == "rubrik-probe-key" + assert metadata["user_api_key_user_id"] == "probe-user-1" + assert metadata["user_api_key_team_id"] == "probe-team-1" + assert metadata["user_api_key_org_id"] == "probe-org-1" + + def test_metadata_covers_the_full_caller_key_set(self, handler, user_api_key_dict): + """A block log and a success log agree on the caller key set.""" + from litellm.types.utils import StandardLoggingUserAPIKeyMetadata + + payload = handler._build_fallback_payload(self._blocked_call_details(), user_api_key_dict) + + expected = StandardLoggingUserAPIKeyMetadata.__required_keys__ | StandardLoggingUserAPIKeyMetadata.__optional_keys__ + assert set(payload["metadata"]) == set(expected) + + def test_virtual_key_is_logged_hashed(self, handler): + """A virtual key reaches the webhook as its hash, never as the raw token.""" + raw = "sk-block-attribution-test" + payload = handler._build_fallback_payload( + self._blocked_call_details(), UserAPIKeyAuth(api_key=raw) + ) + + assert payload["metadata"]["user_api_key_hash"] not in (raw, "") + + def test_request_header_metadata_is_not_used_as_identity(self, handler, user_api_key_dict): + """The pre-fix source is present and misleading; it must not win.""" + call_details = self._blocked_call_details() + call_details["metadata"]["user_api_key_hash"] = "stale-hash-from-request-metadata" + + payload = handler._build_fallback_payload(call_details, user_api_key_dict) + + assert payload["metadata"]["user_api_key_hash"] == user_api_key_dict.api_key + + async def test_enqueued_block_event_carries_attribution(self, handler, user_api_key_dict): + """End of the real hook chain: what actually lands on the Rubrik queue.""" + logging_obj = Mock() + logging_obj.model_call_details = self._blocked_call_details() + request_data = {"litellm_call_id": "call-attr", "_rubrik_logging_obj": logging_obj} + exc = ModifyResponseException( + message="prompt blocked", + model="claude-3", + request_data=request_data, + guardrail_name="rubrik", + ) + + await handler.async_post_call_failure_hook( + request_data=request_data, + original_exception=exc, + user_api_key_dict=user_api_key_dict, + ) + + assert len(handler.log_queue) == 1 + metadata = handler.log_queue[0]["metadata"] + assert metadata["user_api_key_hash"] == user_api_key_dict.api_key + assert metadata["user_api_key_user_id"] == "probe-user-1" + + # -- async_send_batch empty queue and flush_queue edge cases ------------------ diff --git a/type-discipline-budget.json b/type-discipline-budget.json index f071c381916..bf0b30967e7 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -3,7 +3,7 @@ "limit": 23350 }, "LIT002": { - "limit": 27256 + "limit": 27255 }, "LIT003": { "limit": 292 From 9e3a8df6c045bf00d30dcfcf92a7343d8b5d9aa9 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 3 Aug 2026 20:10:37 -0700 Subject: [PATCH 111/123] feat(spend): add net auto-router savings to the cost-optimization dashboard (#35521) * feat(spend): add net auto-router savings to the cost-optimization dashboard The dashboard credited compression and prompt caching but said nothing about the optimization that picks the model, so the driver with the largest lever on a bill was the one an operator could not see. Savings are the counterfactual: without a router a deployment runs one model, and it has to be one that can carry the hardest request, so the baseline is the priciest model in the router's hardest configured tier. A cheap tier is a choice the router made, not a ceiling it was bounded by. `auto_router_savings_baseline_model` overrides it for operators who would genuinely have run something else. Both are provider-qualified before pricing, because a bare name can resolve to a different vendor's rates or to nothing at all, and a deployment is priced by its `base_model` where it has one, which is how Azure deployments are priced everywhere else. Both arms price the request's real usage through `generic_cost_per_token` rather than re-deriving per-token arithmetic, so tiered rates, ephemeral cache-write tiers and regional uplifts stay consistent with what was actually billed. `prompt_tokens` already includes the cache buckets, so charging them again at the input rate would price the same tokens twice. Cache state is what makes this hard. The baseline serves every turn, so whether it had the prompt cached is whether the conversation was already underway. On a continuing conversation it wrote the prompt earlier and would only read it now, so this request's write is what switching cost and counts against the saving. On a first turn nothing was cached for any model, the baseline would have written the same prompt, and both arms carry the write at their own rates. Charging the write to both cases understates a first turn to a few percent of its value, and because the write premium is fixed by prompt size while the saving grows with completion length, it can render a profitable route as a loss. That shape is read off the conversation rather than remembered: a second human ask means an earlier turn was served. No cache, no session id, and no dependence on a caller sending a session header. It cannot see a switch on a turn the router did not classify, and it reads a few-shot prompt's synthetic turns as prior conversation; both err toward charging the write, which under-claims. The baseline and the shape ride on the existing `routing_decision` record, which is already carried from the router to the spend log, already classified for redaction, and already written-or-cleared per attempt. A fallback that re-enters the hook therefore cannot leave either fact behind to be attributed to a deployment that never routed, and no new metadata key crosses the trust boundary. The result is signed. Whether a switch pays off is a race between the rate gap and the cache-write cost, and a narrow gap loses; flooring at zero would hide exactly the routing behaviour an operator needs to see. The donut plots only drivers that saved, while the card and range total keep the sign. Savings accrue into a new `autorouter_savings_spend` column on the six daily rollup tables, declared `NotRequired` because rows queued by a pod on the previous release carry no such key. It is summed by the rollup merge the cross-pod Redis drain also runs, and carried through the aggregation query, the per-row accumulation and the response model, so the dashboard reads a value the API actually sends. Tests enumerate the drivers from the response model itself and assert each is summed, accumulated, carried and totalled, so one added later cannot be half-wired. * fix(spend): let the baseline pay for a continuing turn's own growth `_baseline_usage` moved every cache-creation token into the baseline's read bucket whenever the conversation was underway. That is right for a switch, where the baseline never left the model it was on and really would only read, but wrong for a turn that stayed put: the prompt grew, and the tokens written are that growth. They are new to every model, so the baseline would have paid to write them too. Forgiving it that write made the counterfactual cheaper than it was and shrank the reported saving on ordinary steady-state traffic, by about 2% per turn. The selected arm was never involved; it has always been priced on the real usage. The error sat entirely on the baseline. The condition is that the request read more than it wrote, not that it read anything. A switch onto a model already holding a small prefix of this prompt still writes most of it, and that write is the switch's own cost; keying off a nonzero read would have handed such a request the full rate gap, turning +$0.0056 into +$0.1177. Comparing the two buckets separates a warm continuation, which reads far more than it writes, from a cold arrival, which does the reverse, and it leaves the existing invariant intact: a request reading 0 and one reading 1 both still land in the same place. * fix(spend): price each arm under the key litellm billed it, and see agent turns Two ways the savings number read the wrong thing, both from identifying a model by its name when the name is not what it costs. The counterfactual was ranked and priced on the public rate for the model a deployment names. A deployment may not be charged that rate: the router registers its configured prices under the deployment's own id and deliberately keeps them off the shared model-name key so deployments sharing a backend model do not pollute each other. So a hardest-tier deployment configured above its public rate lost the ranking to a cheaper candidate, and once chosen was priced at a rate nobody pays. Which key prices a deployment is now `_select_model_name_for_cost_calc`'s decision, the resolver the real request is billed through, rather than a second rule here that would have to re-learn that per-second and tiered overrides count, that a partial override still counts, and that a deployment configured at zero is priced at zero rather than treated as unpriced. The arm being subtracted had the same fault and a sharper edge. It priced the spend log's `model`, which on Azure is the deployment name, absent from the cost map, so the whole driver silently read zero for that traffic. It no longer re-derives anything: `model_map_information.model_map_key` is what litellm actually billed the request under, recorded at request time by that same resolver with `base_model` and custom pricing already applied. Separately, the conversation-shape discriminator counted human asks, and an agent loop can run twenty turns on one of them. Its tool traffic rides `tool_result` blocks on user turns that flatten to empty text, and `tool` roles that are never read, so a long agentic conversation looked like its own first turn and was handed the arithmetic that leaves the cache write on both arms. That is the one direction this must never fail in, because it inflates. An assistant turn is the direct evidence that something answered earlier, and it is blind to how the tool plumbing is spelled on either surface. * fix(spend): give the cost-key resolver both inputs the selected arm needs The served model was resolved through one input at a time, and each choice broke the half the other fixed. `model_map_key` is the served model already resolved through `base_model`, which is the only way an Azure deployment name reaches the cost map at all; without it the selected arm priced a name absent from the map, returned nothing, and the whole driver silently read zero for that traffic. But it is built without `router_model_id`, so it never carries a deployment's own price overrides, and a custom-priced deployment was compared at its public rate while the baseline used the real override. On a deployment configured well above its public rate that inverted the answer outright: a route that lost $21.88 reported saving $0.10. `_select_model_name_for_cost_calc` takes both, so it gets both. Which key prices a deployment stays its decision rather than a rule restated here. * fix(spend): same model is only the same cost when it is the same deployment The short-circuit compared resolved model identity, so two deployments of one model collapsed to "no switch" and reported zero. They are not the same cost: a deployment can carry a negotiated rate, and routing from the dear one to the list-price one is a real saving the dashboard reported as $0.00 against a true $21.93. Both arms now carry the key litellm prices them under, so the comparison is between deployments rather than between names. * refactor(spend): price from resolved rates, not from a name we keep re-resolving Four review rounds landed on one mechanism: which identifier prices a deployment. base_model, then the deployment id, then cache-only overrides. Each round added a clause to a resolution rule that should not exist, and a wrong primitive fails once per input shape, so each shape arrived as its own finding. `Router.get_deployment_model_info` already owns this. It merges a deployment's configured prices over the built-in map, folds in `base_model` defaults for deployments whose name is not a model, and falls back to the model name when nothing is overridden. Every shape hand-rolled here (cache-only, partial, per-second, Azure) was that function re-implemented badly. `generic_cost_per_token` now accepts already-resolved rates instead of demanding a name it looks up itself, which is what forced the name-bending in the first place. Both arms resolve through the owner and pass what they got: the counterfactual by the deployment the router would have used, the served request by the deployment that served it. The invented cost-key resolver is gone, and `Baseline` carries a deployment id rather than a key we chose on litellm's behalf. Net 64 insertions against 79 deletions. * test(spend): follow _most_expensive onto the router that prices its candidates Ranking moved through `Router.get_deployment_model_info`, since what a deployment costs is the router's answer to give; these four cases were still calling the old free-function signature. * fix(spend): rank baseline candidates by what a request costs, not by two rates "Most expensive" was decided by comparing output rate then input rate. That is a property of a rate, not of a request: a deployment dearer per output token can be cheaper per cached token, so the comparison ordered cache-heavy traffic backwards and recorded the wrong counterfactual. Candidates are now costed on one reference request through the same engine the savings themselves use, which leaves cache read and write rates, tiered tables and every other billing dimension to that engine rather than to another rule restated here. The reference request is cache-heavy because auto-routed traffic is. * fix(spend): pick the baseline against the request that ran, not a stand-in for one Ranking happened in the pre-routing hook, where the request has not executed yet, so candidates were costed against a hard-coded reference workload: 20k prompt, 19k of it cached, 1k out. Which candidate is dearest depends on that mix, so a pooled hardest tier holding a deployment with non-proportional configured rates could be ranked for a request nothing like the one served. The mix is known on the spend path, so the ranking belongs there. The routing decision now carries the tier's candidates rather than a winner already chosen, and the baseline is resolved against the usage that actually happened. The reference workload is gone; nothing here assumes a traffic shape any more. The router is passed in rather than imported from `proxy_server` inside the computation, so the savings stay a pure function of their arguments and the caller owns where the router comes from. That also makes the spend path testable without a running proxy, which the previous shape was not. * refactor(spend): measure savings against one configured model, not a derived one The counterfactual was derived per request: enumerate the hardest tier's deployments, resolve each one's effective pricing, price them all, take the dearest. That machinery produced a review finding per input shape it had not anticipated, and every answer it gave was one an operator could have stated in a line of config. So they state it. `litellm_settings.autorouter_savings_baseline_model` names the model the traffic would have run on without a router, for every auto-router on the proxy, and unset means the driver is off rather than a model nobody named being guessed at. `savings_baseline.py` and its tests are deleted outright, along with the tier enumeration, the candidate list on the routing decision, and the per-deployment override that shadowed it. Cache-state handling is untouched: the baseline is still priced on this request's own read and write split, so a switch still pays for re-warming the cache and a first turn still charges the write to both arms. 45 insertions against 482 deletions. * refactor(router): compute the conversation shape once and pass it down `_classify_and_route` re-derived it from the messages the hook had already resolved, so an ordinary routed request walked the turn list twice for one boolean. The hook computes it and hands it over, which is also where the affinity-hit path already got it from. Also moves `_get_llm_router` below the imports it sat among. * fix(router): drop the dead conversation_continuing parameter off the hook It was added to `async_pre_routing_hook` by mistake and immediately overwritten by the value the hook computes, so it never did anything. It also widened a signature every pre-routing strategy shares with the protocol in `types/router.py`, leaving this one router diverged from `AutoRouter` and the interface for no reason. Also records why an unreadable request counts as continuing: no messages is no evidence a turn was served, so it pays the cache write and under-claims rather than being handed a first turn's larger saving on nothing. * fix(spend): charge a baseline its input rate for cache buckets it cannot price A model with no cache_creation_input_token_cost, which is every OpenAI, Azure and Gemini entry, resolved that rate to 0.0 and carried the whole written prompt for free, so a first turn routed onto a cheaper model reported a loss. Same hole on cache reads. Those tokens are plain input on such a model, so they move into the text bucket. * refactor(spend): build the daily upsert payloads in one shot `common_data` and `update_data` were constructed and then appended to: `request_id` conditionally for tag rows, `endpoint` unconditionally a few lines later. A dict that grows after its literal cannot be reasoned about by reading the literal, which is the whole point of building it at once. The conditional key resolves to a spreadable value before either payload, so both are single expressions and the tag branch appears once instead of twice. Not wrapped in MappingProxyType, though it was suggested: these go straight to prisma, whose query builder branches on `isinstance(value, dict)` to tell a nested node from a scalar. A mappingproxy is a Mapping but not a dict, so it falls through to the serializer and raises `TypeError: Type not serializable` inside the batch upsert, where the surrounding except would log it and leave the rollups silently unwritten. * fix(spend): keep the one-shot upsert payloads under the type-discipline budget Building both payloads as single literals traded a mutation for two dict literals, and LIT002 counts construction rather than mutation, so the change the review asked for is the one the gate charges for. The empty branch is the avoidable half: it is the same value every time, so it moves to a module constant built once instead of a literal per transaction, and it is a read-only mapping so none of the call sites that spread it can fill it in later. --- .../migration.sql | 17 + .../litellm_proxy_extras/schema.prisma | 6 + litellm/__init__.py | 1 + .../litellm_core_utils/llm_cost_calc/utils.py | 8 +- litellm/proxy/_types.py | 7 +- litellm/proxy/db/db_spend_update_writer.py | 121 +++-- .../daily_spend_update_queue.py | 4 + .../common_daily_activity.py | 9 + litellm/proxy/schema.prisma | 6 + litellm/proxy/spend_tracking/savings.py | 270 +++++++++- .../complexity_router/complexity_router.py | 49 +- .../common_daily_activity.py | 3 + litellm/types/utils.py | 2 + schema.prisma | 6 + .../test_daily_spend_update_queue.py | 60 +++ .../test_common_daily_activity.py | 63 ++- .../proxy/spend_tracking/test_savings.py | 460 +++++++++++++++++- .../router_strategy/test_complexity_router.py | 156 ++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 10 + 19 files changed, 1197 insertions(+), 61 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql new file mode 100644 index 00000000000..a7efd444fdb --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql @@ -0,0 +1,17 @@ +-- AlterTable +ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 37ea55f8c13..0d7fa8692c8 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/litellm/__init__.py b/litellm/__init__.py index a9a78846fa1..62c41c2959b 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -264,6 +264,7 @@ databricks_key: Optional[str] = None openai_like_key: Optional[str] = None azure_key: Optional[str] = None anthropic_key: Optional[str] = None +autorouter_savings_baseline_model: Optional[str] = None replicate_key: Optional[str] = None bytez_key: Optional[str] = None gdc_key: Optional[str] = None diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index face1d1b49f..2eaac7cb1ca 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -683,6 +683,7 @@ def generic_cost_per_token( custom_llm_provider: str, service_tier: str | None = None, data_residency: str | None = None, + model_info: ModelInfo | None = None, ) -> tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -700,7 +701,12 @@ def generic_cost_per_token( """ ## GET MODEL INFO - model_info = get_model_info(model=model, custom_llm_provider=custom_llm_provider) + # A caller that already resolved the deployment's effective rates passes them in + # rather than handing back a name for this to re-resolve. A name cannot express a + # per-deployment override: those are registered under the deployment id and kept off + # the shared model-name key, so resolving from the name here reads the public rate. + if model_info is None: + model_info = get_model_info(model=model, custom_llm_provider=custom_llm_provider) ## CALCULATE INPUT COST ### Cost of processing (non-cache hit + cache hit) + Cost of cache-writing (cache writing) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 268b08d0f2e..5063bc790b5 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -14,7 +14,7 @@ from pydantic import ( field_validator, model_validator, ) -from typing_extensions import Required, TypedDict +from typing_extensions import NotRequired, Required, TypedDict from litellm._uuid import uuid from litellm.constants import MCP_STDIO_ALLOWED_COMMANDS @@ -4566,6 +4566,11 @@ class BaseDailySpendTransaction(TypedDict): # cost-savings metrics (dollars, priced per request before aggregation) compression_savings_spend: float prompt_caching_savings_spend: float + # Not required: rows queued by a pod running the previous release, or replayed from + # the Redis buffer across an upgrade, carry no such key. Every reader coalesces a + # missing value to zero, so requiring it here would describe a shape the aggregation + # is explicitly tested against. + autorouter_savings_spend: NotRequired[float] # request level metrics spend: float diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 17410698aed..7c9ee96e809 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -12,7 +12,9 @@ import os import random import time import traceback +from collections.abc import Mapping from datetime import datetime, timedelta, timezone +from types import MappingProxyType from typing import ( TYPE_CHECKING, Any, @@ -68,6 +70,26 @@ else: ProxyLogging = Any +# Only tag rows carry a request_id, so the other entity types spread nothing. Built +# once here rather than as an empty literal per transaction, and read-only so it cannot +# be filled in by accident from one of the call sites that spreads it. +_NO_TAG_REQUEST_ID: Mapping[str, Any] = MappingProxyType({}) + + +def _get_llm_router(): + """The proxy's router, or None outside a running proxy. + + Injected rather than imported where it is used, so the savings computation stays + a pure function of its arguments and the caller owns where the router comes from. + """ + try: + from litellm.proxy.proxy_server import llm_router + + return llm_router + except Exception: # noqa: BLE001 # no proxy in scope; savings degrade to zero + return None + + def _extract_cache_read_tokens(usage_obj: dict) -> int: """ Anthropic: top-level cache_read_input_tokens field. @@ -1545,6 +1567,40 @@ class DBSpendUpdateWriter: # Get the table dynamically table = getattr(batcher, table_name) + # Additive metrics that older queued rows may omit; one + # enumeration feeds both the create and the increment below + optional_metrics = { + field: value + for field, value in ( + ("cache_read_input_tokens", transaction.get("cache_read_input_tokens")), + ( + "cache_creation_input_tokens", + transaction.get("cache_creation_input_tokens"), + ), + ("compression_saved_tokens", transaction.get("compression_saved_tokens")), + ( + "compression_savings_spend", + transaction.get("compression_savings_spend"), + ), + ( + "prompt_caching_savings_spend", + transaction.get("prompt_caching_savings_spend"), + ), + ("autorouter_savings_spend", transaction.get("autorouter_savings_spend")), + ) + if value is not None + } + + # Only tag rows carry a request_id. Resolved to a spreadable + # value here so both payloads are built in one shot: a dict + # appended to after construction is one nobody can reason about + # by reading its literal. + tag_request_id: Mapping[str, Any] = ( + MappingProxyType({"request_id": transaction["request_id"]}) + if entity_type == "tag" and "request_id" in transaction + else _NO_TAG_REQUEST_ID + ) + # Common data structure for both create and update common_data = { entity_id_field: entity_id, @@ -1561,34 +1617,10 @@ class DBSpendUpdateWriter: "api_requests": transaction["api_requests"], "successful_requests": transaction["successful_requests"], "failed_requests": transaction["failed_requests"], + **optional_metrics, + **tag_request_id, } - # Add cache-related fields if they exist - if "cache_read_input_tokens" in transaction: - common_data["cache_read_input_tokens"] = transaction.get( - "cache_read_input_tokens", 0 - ) - if "cache_creation_input_tokens" in transaction: - common_data["cache_creation_input_tokens"] = transaction.get( - "cache_creation_input_tokens", 0 - ) - if "compression_saved_tokens" in transaction: - common_data["compression_saved_tokens"] = transaction.get( - "compression_saved_tokens", 0 - ) - if "compression_savings_spend" in transaction: - common_data["compression_savings_spend"] = transaction.get( - "compression_savings_spend", 0 - ) - if "prompt_caching_savings_spend" in transaction: - common_data["prompt_caching_savings_spend"] = transaction.get( - "prompt_caching_savings_spend", 0 - ) - - if entity_type == "tag" and "request_id" in transaction: - common_data["request_id"] = transaction.get("request_id") - - # Create update data structure update_data = { "prompt_tokens": {"increment": transaction["prompt_tokens"]}, "completion_tokens": {"increment": transaction["completion_tokens"]}, @@ -1596,36 +1628,12 @@ class DBSpendUpdateWriter: "api_requests": {"increment": transaction["api_requests"]}, "successful_requests": {"increment": transaction["successful_requests"]}, "failed_requests": {"increment": transaction["failed_requests"]}, + **{field: {"increment": value} for field, value in optional_metrics.items()}, + # An existing row predating the endpoint column gets it filled in here + "endpoint": transaction.get("endpoint") or "", + **tag_request_id, } - # Add cache-related fields to update if they exist - if "cache_read_input_tokens" in transaction: - update_data["cache_read_input_tokens"] = { - "increment": transaction.get("cache_read_input_tokens", 0) - } - if "cache_creation_input_tokens" in transaction: - update_data["cache_creation_input_tokens"] = { - "increment": transaction.get("cache_creation_input_tokens", 0) - } - if "compression_saved_tokens" in transaction: - update_data["compression_saved_tokens"] = { - "increment": transaction.get("compression_saved_tokens", 0) - } - if "compression_savings_spend" in transaction: - update_data["compression_savings_spend"] = { - "increment": transaction.get("compression_savings_spend", 0) - } - if "prompt_caching_savings_spend" in transaction: - update_data["prompt_caching_savings_spend"] = { - "increment": transaction.get("prompt_caching_savings_spend", 0) - } - - if entity_type == "tag" and "request_id" in transaction: - update_data["request_id"] = transaction.get("request_id") - - # Add endpoint to update_data so existing rows get their endpoint field updated - update_data["endpoint"] = transaction.get("endpoint") or "" - table.upsert( where=where_clause, data={ @@ -1875,6 +1883,10 @@ class DBSpendUpdateWriter: custom_llm_provider=payload.get("custom_llm_provider", None), compression_saved_tokens=compression_saved_tokens, cache_read_input_tokens=cache_read_input_tokens, + routing_decision=_metadata.get("routing_decision"), + model_id=payload.get("model_id"), + llm_router=_get_llm_router(), + usage_object=usage_obj, ) daily_transaction = BaseDailySpendTransaction( @@ -1896,6 +1908,7 @@ class DBSpendUpdateWriter: compression_saved_tokens=compression_saved_tokens, compression_savings_spend=savings_spend.compression, prompt_caching_savings_spend=savings_spend.prompt_caching, + autorouter_savings_spend=savings_spend.autorouter, ) return daily_transaction except Exception as e: diff --git a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py index 18df25093f1..a61db128382 100644 --- a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py @@ -133,6 +133,10 @@ class DailySpendUpdateQueue(BaseUpdateQueue): payload.get("prompt_caching_savings_spend", 0) or 0 ) + daily_transaction.get("prompt_caching_savings_spend", 0) + daily_transaction["autorouter_savings_spend"] = ( + payload.get("autorouter_savings_spend", 0) or 0 + ) + daily_transaction.get("autorouter_savings_spend", 0) + else: aggregated_daily_spend_update_transactions[_key] = deepcopy(payload) return aggregated_daily_spend_update_transactions diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 0dc85f98786..132a8409e06 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -95,6 +95,9 @@ class DailySpendRecord(Protocol): @property def prompt_caching_savings_spend(self) -> float: ... + @property + def autorouter_savings_spend(self) -> float: ... + @property def api_requests(self) -> int: ... @@ -135,6 +138,7 @@ class _GroupingSetsRow(SimpleNamespace): compression_saved_tokens: int | None compression_savings_spend: float | None prompt_caching_savings_spend: float | None + autorouter_savings_spend: float | None api_requests: int | None successful_requests: int | None failed_requests: int | None @@ -158,6 +162,7 @@ def update_metrics(existing_metrics: SpendMetrics, record: DailySpendRecord) -> existing_metrics.compression_saved_tokens += record.compression_saved_tokens or 0 existing_metrics.compression_savings_spend += record.compression_savings_spend or 0 existing_metrics.prompt_caching_savings_spend += record.prompt_caching_savings_spend or 0 + existing_metrics.autorouter_savings_spend += record.autorouter_savings_spend or 0 existing_metrics.api_requests += record.api_requests or 0 existing_metrics.successful_requests += record.successful_requests or 0 existing_metrics.failed_requests += record.failed_requests or 0 @@ -590,6 +595,7 @@ def _build_aggregated_sql_query( SUM(compression_saved_tokens)::bigint AS compression_saved_tokens, SUM(compression_savings_spend)::float AS compression_savings_spend, SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend, + SUM(autorouter_savings_spend)::float AS autorouter_savings_spend, SUM(api_requests)::bigint AS api_requests, SUM(successful_requests)::bigint AS successful_requests, SUM(failed_requests)::bigint AS failed_requests @@ -732,6 +738,7 @@ def _record_to_spend_metrics(record: _GroupingSetsRow) -> SpendMetrics: compression_saved_tokens=record.compression_saved_tokens or 0, compression_savings_spend=record.compression_savings_spend or 0, prompt_caching_savings_spend=record.prompt_caching_savings_spend or 0, + autorouter_savings_spend=record.autorouter_savings_spend or 0, api_requests=record.api_requests or 0, successful_requests=record.successful_requests or 0, failed_requests=record.failed_requests or 0, @@ -986,6 +993,7 @@ async def get_daily_activity( total_compression_saved_tokens=metadata_metrics.compression_saved_tokens, total_compression_savings_spend=metadata_metrics.compression_savings_spend, total_prompt_caching_savings_spend=metadata_metrics.prompt_caching_savings_spend, + total_autorouter_savings_spend=metadata_metrics.autorouter_savings_spend, page=page, total_pages=-(-total_count // page_size), # Ceiling division has_more=(page * page_size) < total_count, @@ -1075,6 +1083,7 @@ async def get_daily_activity_aggregated( total_compression_saved_tokens=aggregated["totals"].compression_saved_tokens, total_compression_savings_spend=aggregated["totals"].compression_savings_spend, total_prompt_caching_savings_spend=aggregated["totals"].prompt_caching_savings_spend, + total_autorouter_savings_spend=aggregated["totals"].autorouter_savings_spend, page=1, total_pages=1, has_more=False, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 37ea55f8c13..0d7fa8692c8 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index ad9d02052bb..1a1c813d784 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -8,15 +8,22 @@ are known) and summed into the daily tables; tokens cannot be priced after they have been aggregated across models. """ -from typing import NamedTuple +from collections.abc import Mapping +from typing import TYPE_CHECKING, NamedTuple import litellm from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token + +if TYPE_CHECKING: + from litellm.router import Router +from litellm.types.utils import ModelInfo, PromptTokensDetailsWrapper, Usage class SavingsSpend(NamedTuple): compression: float prompt_caching: float + autorouter: float = 0.0 def _input_and_cache_read_cost(model: str | None, custom_llm_provider: str | None) -> tuple[float, float]: @@ -44,11 +51,243 @@ def _input_and_cache_read_cost(model: str | None, custom_llm_provider: str | Non return input_cost, float(cache_read_cost) +class _ModelIdentity(NamedTuple): + model: str + provider: str + + +def _resolve_model(model: str | None, custom_llm_provider: str | None) -> _ModelIdentity | None: + """Canonical ``(model, provider)``, or ``None`` when the model cannot be resolved. + + The two sides of the comparison arrive spelled differently: the spend log records a + normalized model name alongside its provider, while the baseline arrives as the + operator wrote it in config, with the provider prefixed, implied, or absent. Raw + string equality therefore reads `anthropic/claude-opus-5` as a switch away from + `claude-opus-5`, and pricing a bare name with no provider can resolve it to a + different vendor's rates than the deployment it names. + """ + if not model: + return None + try: + resolved_model, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + except Exception as e: # noqa: BLE001 # get_llm_provider raises for unroutable names; degrade to zero savings + verbose_proxy_logger.debug( + "savings: cannot resolve provider for model=%s custom_llm_provider=%s (%s)", model, custom_llm_provider, e + ) + return None + return _ModelIdentity(model=resolved_model, provider=provider) + + +def _effective_model_info(router: "Router | None", deployment_id: str | None, model: str) -> ModelInfo | None: + """What a deployment is actually charged, or ``None`` to price by name. + + `Router.get_deployment_model_info` owns this: it merges a deployment's configured + prices over the built-in map, folds in `base_model` defaults for deployments whose + name is not a model, and falls back to the model name when nothing is overridden. + Resolving a name here instead reads the public rate, which a deployment with a + negotiated price does not pay, and an Azure deployment name prices to nothing at all. + """ + if router is None or deployment_id is None: + return None + try: + return router.get_deployment_model_info(deployment_id, model) + except Exception as e: # noqa: BLE001 # a dashboard metric must not fail the spend write + verbose_proxy_logger.debug("savings: no deployment pricing for %s (%s)", model, e) + return None + + +def _model_info(model: _ModelIdentity) -> ModelInfo | None: + """The public rates for ``model``, or ``None`` when it has none.""" + try: + return litellm.get_model_info(model=model.model, custom_llm_provider=model.provider) + except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models + verbose_proxy_logger.debug("savings: no pricing for provider=%s model=%s (%s)", model.provider, model.model, e) + return None + + +def _cost_of_usage(model: _ModelIdentity, usage: Usage, model_info: ModelInfo | None = None) -> float | None: + """What ``usage`` costs on ``model``, or ``None`` when the model has no pricing.""" + try: + prompt_cost, completion_cost = generic_cost_per_token( + model=model.model, usage=usage, custom_llm_provider=model.provider, model_info=model_info + ) + except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings + verbose_proxy_logger.debug( + "savings: cannot price usage for provider=%s model=%s (%s)", model.provider, model.model, e + ) + return None + return prompt_cost + completion_cost + + +def _cache_token_split(usage: Usage) -> tuple[int, int]: + """``(cache_read_tokens, cache_creation_tokens)`` for a request.""" + details = usage.prompt_tokens_details + if details is None: + return 0, 0 + read = getattr(details, "cached_tokens", 0) or 0 + created = (getattr(details, "cache_creation_tokens", 0) or 0) or (getattr(details, "cache_write_tokens", 0) or 0) + return int(read), int(created) + + +_CACHE_SPLIT_FIELDS = frozenset( + ("cached_tokens", "cache_creation_tokens", "cache_write_tokens", "cache_creation_token_details", "text_tokens") +) + + +def _baseline_cache_rate_keys(baseline_info: ModelInfo | None) -> tuple[bool, bool]: + """Whether the baseline model has a ``(cache read, cache write)`` rate of its own. + + A missing rate is not a free bucket. `_get_token_base_cost` resolves an absent + `cache_read_input_token_cost` or `cache_creation_input_token_cost` to 0.0, so a + baseline whose provider prices caching implicitly, which is every OpenAI, Azure and + Gemini entry for cache writes, would carry the whole prompt for nothing and turn a + profitable route into a reported loss. Such a model pays its plain input rate for + those tokens, so the buckets it cannot price become ordinary input below. + """ + if baseline_info is None: + return True, True + return bool(baseline_info.get("cache_read_input_token_cost")), bool( + baseline_info.get("cache_creation_input_token_cost") + ) + + +def _baseline_usage(usage: Usage, conversation_continuing: bool, baseline_info: ModelInfo | None = None) -> Usage: + """The same request as a single-model baseline would have met it. + + The baseline is one model serving every turn, so whether it had this prompt cached + is simply whether the conversation was already underway. On a continuing + conversation it wrote the prompt on an earlier turn and would only read it now, so + the cache tokens move into the read bucket and whatever this request paid to write + counts against the saving; that write is what switching models costs. + + On a conversation's first turn nothing was cached anywhere, for any model. The + baseline would have written the same prompt, so the cache buckets stay where they are + and both arms carry the write at their own rates, unless the baseline has no rate for + a bucket, in which case those tokens are its plain input. Charging the write to this case + too, which is all a single rollup row can support, understates a first turn to a + few percent of its value and can render a profitable route as a loss. + + A continuing turn that mostly read from cache is the third case: the selected model + was already warm, so it is the one that has been serving this conversation and the + baseline's cache holds exactly what its does. The tokens written are the turn's own + growth, new to every model, and the baseline would have paid to write them too. + Moving them would forgive the baseline a write it really owes and shrink the + reported saving. "Mostly read" rather than "read anything" on purpose: a switch onto + a model holding a small prefix of this prompt still writes most of it, and must keep + counting that write against the saving. + + Only the cache buckets move. Every other field the request was priced on travels + through untouched, audio and image and video counts among them, because the baseline + is this same request served by a model that happened to be warm; naming the fields to + keep instead would price the baseline on a request that never ran, and would go stale + the next time a priced field is added. + """ + cache_read, cache_creation = _cache_token_split(usage) + details = usage.prompt_tokens_details + if details is None or (cache_read <= 0 and cache_creation <= 0): + return usage + + # The tokens this request paid to write move into the cached count and the creation + # charge is dropped: on one model that cache was already warm, so the baseline would + # have read them rather than paying to create them. The 5m/1h breakdown goes with + # them; left behind it re-charges the write. + warm = conversation_continuing and cache_creation > 0 and cache_read <= cache_creation + reads = cache_read + cache_creation if warm else cache_read + writes = 0 if warm else cache_creation + + prices_reads, prices_writes = _baseline_cache_rate_keys(baseline_info) + reads = reads if prices_reads else 0 + writes = writes if prices_writes else 0 + if (reads, writes) == (cache_read, cache_creation): + return usage + + other_modalities = sum( + (getattr(details, field, 0) or 0) for field in ("audio_tokens", "image_tokens", "video_tokens") + ) + return Usage( + prompt_tokens=usage.prompt_tokens, + completion_tokens=usage.completion_tokens, + total_tokens=usage.total_tokens, + completion_tokens_details=usage.completion_tokens_details, + prompt_tokens_details=PromptTokensDetailsWrapper( + **details.model_dump(exclude=_CACHE_SPLIT_FIELDS), + cached_tokens=reads, + cache_creation_tokens=writes, + cache_write_tokens=writes, + cache_creation_token_details=details.cache_creation_token_details if writes else None, + # Whatever no longer sits in a cache bucket is plain input on the baseline. + text_tokens=max(usage.prompt_tokens - reads - writes - other_modalities, 0), + ), + ) + + +def compute_autorouter_savings( + baseline_model: str | None, + selected_model: str | None, + selected_provider: str | None, + usage: Usage, + conversation_continuing: bool = True, + selected_info: ModelInfo | None = None, +) -> float: + """Net dollars the router saved, or cost, by serving this request on ``selected_model``. + + Signed on purpose. Switching models leaves the new one with a cold cache, so the + request pays a cache-creation charge that staying on one model would not have + incurred; when that charge outweighs the cheaper rates, routing lost money and the + dashboard has to be able to say so. Zero when both sides resolve to the same + deployment, or when either cannot be resolved or priced. + + ``conversation_continuing`` says whether the baseline would already have had this + prompt cached. It defaults to True because that is the conservative reading: a + request whose shape the router could not determine is charged the write and + under-claims rather than inflating a savings figure. + """ + # No provider argument for the baseline on purpose: it arrives from the routing + # metadata as a single self-describing string, already qualified by the auto-router, + # so there is no second field that could disagree with it. + baseline = _resolve_model(baseline_model, None) + selected = _resolve_model(selected_model, selected_provider) + if baseline is None or selected is None: + return 0.0 + # Same model is only the same cost when it is also the same deployment. Two + # deployments of one model can carry different negotiated rates, and routing from + # the dear one to the cheap one is a real saving that short-circuiting on the model + # name alone reports as zero. + if baseline == selected: + return 0.0 + baseline_info = _model_info(baseline) + baseline_cost = _cost_of_usage( + baseline, _baseline_usage(usage, conversation_continuing, baseline_info), baseline_info + ) + selected_cost = _cost_of_usage(selected, usage, selected_info) + if baseline_cost is None or selected_cost is None: + return 0.0 + return baseline_cost - selected_cost + + +def _usage_from_spend_log(usage_object: Mapping[str, object] | None) -> Usage | None: + """Rebuild the request's ``Usage`` from the copy the spend log recorded.""" + if not usage_object: + return None + try: + return Usage(**usage_object) + except Exception as e: # noqa: BLE001 # a malformed usage_object must not fail the daily spend write + # Warning, not debug: this silently zeroes the auto-router driver for every + # affected row, and a shape change in Usage would otherwise show up only as a + # dashboard that quietly reads $0.00. + verbose_proxy_logger.warning("savings: unusable usage_object, auto-router savings will read zero (%s)", e) + return None + + def compute_savings_spend( model: str | None, custom_llm_provider: str | None, compression_saved_tokens: int, cache_read_input_tokens: int, + routing_decision: Mapping[str, object] | None = None, + usage_object: Mapping[str, object] | None = None, + model_id: str | None = None, + llm_router: "Router | None" = None, ) -> SavingsSpend: """ Dollar savings for one request, split by optimization driver. @@ -56,8 +295,35 @@ def compute_savings_spend( Compression savings price the tokens compression removed at the model's input rate. Prompt-caching savings price the cache-read tokens at the difference between the input rate and the discounted cache-read rate. + Auto-router savings compare the served ``model`` against the counterfactual + baseline the router recorded on its ``routing_decision``, and are zero unless the + two differ. That record also says whether the conversation was already underway, + which is what tells a mid-conversation switch from a first turn. """ input_cost, cache_read_cost = _input_and_cache_read_cost(model, custom_llm_provider) compression = max(compression_saved_tokens, 0) * input_cost prompt_caching = max(cache_read_input_tokens, 0) * max(input_cost - cache_read_cost, 0.0) - return SavingsSpend(compression=compression, prompt_caching=prompt_caching) + + usage = _usage_from_spend_log(usage_object) + if usage is None or not model: + return SavingsSpend(compression=compression, prompt_caching=prompt_caching) + + # The counterfactual is one model an operator would have run instead of the router, + # configured once for the proxy rather than derived per request. Unset means the + # driver is off; a routing decision is what says this request was auto-routed at all. + decision = routing_decision if isinstance(routing_decision, Mapping) else {} + autorouter = ( + compute_autorouter_savings( + baseline_model=litellm.autorouter_savings_baseline_model, + selected_model=model, + selected_provider=custom_llm_provider, + usage=usage, + # Absent means the router never recorded a shape, which is the conservative + # reading: charge the cache write rather than claim a first turn's saving. + conversation_continuing=decision.get("conversation_continuing") is not False, + selected_info=_effective_model_info(llm_router, model_id, model or ""), + ) + if decision + else 0.0 + ) + return SavingsSpend(compression=compression, prompt_caching=prompt_caching, autorouter=autorouter) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 6f2bf61834b..ca2e7fcc038 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -224,6 +224,40 @@ def _iter_human_asks_newest_first(messages: Sequence[Mapping[str, object]]) -> I ) +def _conversation_is_continuing(messages: Sequence[Mapping[str, object]] | None) -> bool: + """Whether this request continues a conversation that was already underway. + + The counterfactual the savings driver prices against is one model serving every + turn, so whether that model had this prompt cached is just whether an earlier turn + exists. An assistant turn in the history is the direct evidence of one: something + answered before, so a single-model deployment wrote the prompt then and would only + read it now, and the write this request paid is what switching models cost. A + conversation's first turn has no assistant turn, nothing was cached for any model, + and the baseline would have paid the same write. + + Assistant turns rather than human asks, because an agent loop can run twenty turns + on one human ask: its tool traffic rides `tool_result` blocks on user turns that + flatten to empty text, and on `tool` roles, so counting asks reads a long + conversation as its own first turn and hands it the untouched-write arithmetic. That + is the one direction this must never fail in, since it inflates. + + Reading the conversation rather than remembering it keeps this free of a cache, a + session id and their failure modes, and it works for callers that send no session + header at all. A few-shot prompt's synthetic assistant turns read as prior + conversation, which charges the write and under-claims; that is the safe side. + + So is an unreadable request. No messages says nothing about whether a turn was + served, and a surface that carries its turns somewhere this cannot see, or a + genuinely single-turn call arriving with none, is treated as continuing: it pays the + cache write and under-claims rather than being handed a first turn's larger saving + on no evidence. That direction is deliberate in both cases and is the only one that + cannot inflate. + """ + if not messages: + return True + return any(message.get("role") == "assistant" for message in messages) + + def _newest_turn_ask(messages: Sequence[Mapping[str, object]]) -> str | None: """The human ask on the newest user turn, or None when that turn carries only plumbing. @@ -647,6 +681,7 @@ class ComplexityRouter(CustomLogger): escalation_keyword: str | None = None, escalated: bool = False, classifier_model: str | None = None, + conversation_continuing: bool = True, ) -> StandardLoggingRoutingDecision: """Assemble the per-request provenance record for this router's decision. @@ -660,6 +695,7 @@ class ComplexityRouter(CustomLogger): router_type="complexity", routed_model=routed_model, cause=cause, + conversation_continuing=conversation_continuing, ) if tier is not None: decision["tier"] = tier.value @@ -1392,6 +1428,8 @@ class ComplexityRouter(CustomLogger): if isinstance(metadata, dict): metadata[RETURN_RAW_MODEL_NAME_METADATA_KEY] = True + conversation_continuing = _conversation_is_continuing(self._resolve_messages(messages, request_kwargs)) + use_session_affinity = self.config.session_affinity and not self.config.plugins session_id = self._get_session_id_from_request_kwargs(request_kwargs) if use_session_affinity else None cache_key = self._get_session_affinity_cache_key(session_id, request_kwargs) if session_id is not None else None @@ -1438,6 +1476,7 @@ class ComplexityRouter(CustomLogger): cause=cause, escalation_keyword=pin_escalation_keyword, escalated=escalated, + conversation_continuing=conversation_continuing, ), ) @@ -1447,6 +1486,7 @@ class ComplexityRouter(CustomLogger): messages=messages, input=input, specific_deployment=specific_deployment, + conversation_continuing=conversation_continuing, ) if cache_key is not None and response is not None: await self.litellm_router_instance.cache.async_set_cache( @@ -1463,6 +1503,7 @@ class ComplexityRouter(CustomLogger): messages: list[dict[str, Any]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, + conversation_continuing: bool = True, ) -> PreRoutingHookResponse | None: """ Classifies the request by complexity and returns the appropriate model. @@ -1509,7 +1550,11 @@ class ComplexityRouter(CustomLogger): return PreRoutingHookResponse( model=routed_model, messages=messages if has_original_messages else None, - routing_decision=self._build_routing_decision(routed_model=routed_model, cause="default_fallback"), + routing_decision=self._build_routing_decision( + routed_model=routed_model, + cause="default_fallback", + conversation_continuing=conversation_continuing, + ), ) newest_ask = _newest_turn_ask(resolved_messages) @@ -1532,6 +1577,7 @@ class ComplexityRouter(CustomLogger): messages=messages if has_original_messages else None, routing_decision=self._build_routing_decision( routed_model=routed_model, + conversation_continuing=conversation_continuing, cause=keyword_cause, tier=routed_tier, matched_keyword=override.matched_keyword, @@ -1579,6 +1625,7 @@ class ComplexityRouter(CustomLogger): messages=messages if has_original_messages else None, routing_decision=self._build_routing_decision( routed_model=routed_model, + conversation_continuing=conversation_continuing, cause=outcome.cause, tier=tier, score=score, diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 00f67d0f39c..21b7ffca3f2 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -25,6 +25,7 @@ class SpendMetrics(BaseModel): compression_saved_tokens: int = Field(default=0) compression_savings_spend: float = Field(default=0.0) prompt_caching_savings_spend: float = Field(default=0.0) + autorouter_savings_spend: float = Field(default=0.0) total_tokens: int = Field(default=0) successful_requests: int = Field(default=0) failed_requests: int = Field(default=0) @@ -85,6 +86,7 @@ class DailySpendMetadata(BaseModel): total_compression_saved_tokens: int = Field(default=0) total_compression_savings_spend: float = Field(default=0.0) total_prompt_caching_savings_spend: float = Field(default=0.0) + total_autorouter_savings_spend: float = Field(default=0.0) page: int = Field(default=1) total_pages: int = Field(default=1) has_more: bool = Field(default=False) @@ -111,6 +113,7 @@ class LiteLLM_DailyUserSpend(BaseModel): compression_saved_tokens: int = 0 compression_savings_spend: float = 0.0 prompt_caching_savings_spend: float = 0.0 + autorouter_savings_spend: float = 0.0 spend: float = 0.0 api_requests: int = 0 successful_requests: int = 0 diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 3539ac0f27a..21d69d9925a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2734,6 +2734,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): classifier_model: str escalated: bool tier_boundaries: StandardLoggingRoutingDecisionTierBoundaries + conversation_continuing: bool # Fields whose values quote the caller's prompt. Dropped when an operator turns message @@ -2753,6 +2754,7 @@ DERIVED_ROUTING_DECISION_FIELDS: FrozenSet[str] = frozenset( "classifier_model", "escalated", "tier_boundaries", + "conversation_continuing", } ) diff --git a/schema.prisma b/schema.prisma index 37ea55f8c13..0d7fa8692c8 100644 --- a/schema.prisma +++ b/schema.prisma @@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py index c7e7ef5d469..a55d4f0dcfd 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py @@ -16,6 +16,9 @@ from litellm.proxy._types import ( Litellm_EntityType, SpendUpdateQueueItem, ) +from typing import get_args + +from litellm.proxy._types import BaseDailySpendTransaction from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import ( DailySpendUpdateQueue, ) @@ -209,6 +212,7 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key(): "compression_saved_tokens": 0, "compression_savings_spend": 0, "prompt_caching_savings_spend": 0, + "autorouter_savings_spend": 0, } updates = [{test_key: test_transaction1}, {test_key: test_transaction2}] @@ -259,6 +263,7 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions( "compression_saved_tokens": 0, "compression_savings_spend": 0, "prompt_caching_savings_spend": 0, + "autorouter_savings_spend": 0, } # Add updates to queue @@ -527,3 +532,58 @@ async def test_compression_saved_tokens_aggregation(daily_spend_update_queue): assert agg["cache_creation_input_tokens"] == 7 assert agg["compression_savings_spend"] == pytest.approx(0.0076) assert agg["prompt_caching_savings_spend"] == pytest.approx(0.0108) + + +@pytest.mark.asyncio +async def test_every_optional_daily_metric_aggregates(daily_spend_update_queue): + """Every additive metric must survive the merge, not just the ones wired by hand. + + Two requests landing on one rollup key before a flush is the common case under + load, and this same merge runs again on every cross-pod Redis drain. A metric + persisted by the database write but skipped here is silently dropped on both + paths, so the driver reads as zero on the dashboard however much it saved. + """ + test_key = "user1_2023-01-01_key123_claude-haiku-4-5_anthropic" + def _numeric(annotation): + # additive metrics may be declared NotRequired[float] for rows queued by a pod + # running the previous release, so unwrap before matching + args = get_args(annotation) + return (args[0] if args else annotation) in (int, float) + + numeric_fields = [ + name for name, annotation in BaseDailySpendTransaction.__annotations__.items() if _numeric(annotation) + ] + assert "autorouter_savings_spend" in numeric_fields + increments = {field: index + 1 for index, field in enumerate(numeric_fields)} + + await daily_spend_update_queue.add_update({test_key: dict(increments)}) + await daily_spend_update_queue.add_update({test_key: dict(increments)}) + await daily_spend_update_queue.aggregate_queue_updates() + updates = await daily_spend_update_queue.flush_all_updates_from_in_memory_queue() + + agg = updates[0][test_key] + for field, value in increments.items(): + assert agg[field] == pytest.approx(value * 2), f"{field} did not accumulate" + + +@pytest.mark.asyncio +async def test_optional_metric_missing_from_an_older_payload_still_aggregates( + daily_spend_update_queue, +): + """A queued row written before a metric existed must not zero it out.""" + test_key = "user1_2023-01-01_key123_claude-haiku-4-5_anthropic" + base = { + "spend": 1.0, + "prompt_tokens": 10, + "completion_tokens": 5, + "api_requests": 1, + "successful_requests": 1, + "failed_requests": 0, + } + + await daily_spend_update_queue.add_update({test_key: dict(base)}) + await daily_spend_update_queue.add_update({test_key: {**base, "autorouter_savings_spend": 0.25}}) + await daily_spend_update_queue.aggregate_queue_updates() + updates = await daily_spend_update_queue.flush_all_updates_from_in_memory_queue() + + assert updates[0][test_key]["autorouter_savings_spend"] == pytest.approx(0.25) diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index c2a0d34a915..f2749be5d6e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -19,7 +19,10 @@ from litellm.proxy.management_endpoints.common_daily_activity import ( get_daily_activity_aggregated, update_metrics, ) -from litellm.types.proxy.management_endpoints.common_daily_activity import SpendMetrics +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + DailySpendMetadata, + SpendMetrics, +) @pytest.mark.asyncio @@ -153,6 +156,7 @@ async def test_get_daily_activity_aggregated_with_endpoint_breakdown(): "compression_saved_tokens": 0, "compression_savings_spend": 0.0, "prompt_caching_savings_spend": 0.0, + "autorouter_savings_spend": 0.0, "failed_requests": 0, } mock_rows = [ @@ -498,6 +502,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_1.compression_saved_tokens = 0 mock_record_1.compression_savings_spend = 0.0 mock_record_1.prompt_caching_savings_spend = 0.0 + mock_record_1.autorouter_savings_spend = 0.0 mock_record_1.api_requests = 10 mock_record_1.successful_requests = 9 mock_record_1.failed_requests = 1 @@ -520,6 +525,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_2.compression_saved_tokens = 0 mock_record_2.compression_savings_spend = 0.0 mock_record_2.prompt_caching_savings_spend = 0.0 + mock_record_2.autorouter_savings_spend = 0.0 mock_record_2.api_requests = 5 mock_record_2.successful_requests = 5 mock_record_2.failed_requests = 0 @@ -582,6 +588,7 @@ async def test_aggregated_activity_preserves_metadata_for_deleted_keys(): "compression_saved_tokens": 0, "compression_savings_spend": 0.0, "prompt_caching_savings_spend": 0.0, + "autorouter_savings_spend": 0.0, "failed_requests": 0, } mock_rows = [ @@ -669,6 +676,7 @@ def _daily_user_spend_record(*, user_id, api_key, spend, model="gpt-4", model_gr compression_saved_tokens=0, compression_savings_spend=0.0, prompt_caching_savings_spend=0.0, + autorouter_savings_spend=0.0, api_requests=1, successful_requests=1, failed_requests=0, @@ -973,6 +981,7 @@ async def test_get_daily_activity_aggregated_empty_result_set(): "compression_saved_tokens": None, "compression_savings_spend": None, "prompt_caching_savings_spend": None, + "autorouter_savings_spend": None, "api_requests": None, "successful_requests": None, "failed_requests": None, @@ -1016,6 +1025,7 @@ def _no_spend_record(): compression_saved_tokens=None, compression_savings_spend=None, prompt_caching_savings_spend=None, + autorouter_savings_spend=None, api_requests=None, successful_requests=None, failed_requests=None, @@ -1050,3 +1060,54 @@ def test_update_metrics_handles_none_values(): assert metrics.cache_read_input_tokens == 0 assert metrics.cache_creation_input_tokens == 0 assert metrics.compression_saved_tokens == 0 + + +class TestEverySavingsDriverSurvivesTheReadPath: + """A savings driver is only real if it survives the whole read path. + + The write path can price a driver correctly and persist it to all six rollup + tables, and the dashboard can still render a permanent $0.00 because the + aggregation query never summed the column or the response model never + declared it. That failure is silent: the card renders, the number is just + always zero, which is indistinguishable from having saved nothing. These + tests enumerate the drivers from the response model itself, so a driver added + later cannot be half-wired. + """ + + def _drivers(self) -> list[str]: + drivers = [field for field in SpendMetrics.model_fields if field.endswith("_savings_spend")] + assert drivers, "expected the dashboard response to expose at least one savings driver" + return drivers + + def test_every_driver_is_summed_by_the_rollup_query(self): + sql, _ = _build_aggregated_sql_query( + table_name="litellm_dailyuserspend", + entity_id_field="user_id", + entity_id="user-1", + start_date="2026-07-01", + end_date="2026-07-31", + model=None, + api_key=None, + timezone_offset_minutes=None, + ) + for driver in self._drivers(): + assert f"SUM({driver})" in sql, f"{driver} is never summed, so it reads as zero" + + def test_every_driver_is_accumulated_across_rows(self): + for driver in self._drivers(): + record = _no_spend_record() + setattr(record, driver, 1.25) + metrics = update_metrics(SpendMetrics(), record) + assert getattr(metrics, driver) == pytest.approx(1.25), f"{driver} is dropped when accumulating rows" + + def test_every_driver_is_carried_by_a_single_row_conversion(self): + for driver in self._drivers(): + record = _no_spend_record() + setattr(record, driver, 2.5) + assert getattr(_record_to_spend_metrics(record), driver) == pytest.approx(2.5) + + def test_every_driver_has_a_range_total(self): + for driver in self._drivers(): + assert f"total_{driver}" in DailySpendMetadata.model_fields, ( + f"total_{driver} is missing, so the range summary omits the driver" + ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 704cf7a63dd..6d0195b632b 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -6,7 +6,14 @@ sys.path.insert(0, os.path.abspath("../../../..")) import pytest import litellm -from litellm.proxy.spend_tracking.savings import compute_savings_spend +from litellm.router import Router +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.proxy.spend_tracking.savings import ( + _baseline_usage, + compute_autorouter_savings, + compute_savings_spend, +) +from litellm.types.utils import Usage def _anthropic_costs(model: str) -> tuple[float, float]: @@ -16,6 +23,39 @@ def _anthropic_costs(model: str) -> tuple[float, float]: return input_cost, cache_read_cost +def _cached_usage_object() -> dict: + """A cache-heavy Anthropic request, shaped as the spend log records it. + + `prompt_tokens` is the inclusive total: 3 uncached text tokens plus 500 read + from cache plus 12304 written to cache. + """ + return { + "prompt_tokens": 12807, + "completion_tokens": 500, + "total_tokens": 13307, + "prompt_tokens_details": {"cached_tokens": 500, "cache_creation_tokens": 12304, "text_tokens": 3}, + "cache_creation_input_tokens": 12304, + "cache_read_input_tokens": 500, + } + + +def _cost_on(model: str, usage_object: dict) -> float: + prompt_cost, completion_cost = generic_cost_per_token( + model=model, usage=Usage(**usage_object), custom_llm_provider="anthropic" + ) + return prompt_cost + completion_cost + + +def _flat_rates(model: str) -> tuple[float, float, float]: + info = litellm.get_model_info(model=model, custom_llm_provider="anthropic") + input_cost = info["input_cost_per_token"] or 0.0 + return ( + input_cost, + info["output_cost_per_token"] or 0.0, + info.get("cache_creation_input_token_cost") or input_cost, + ) + + def test_compression_savings_priced_at_input_rate(): input_cost, _ = _anthropic_costs("claude-sonnet-5") result = compute_savings_spend( @@ -76,3 +116,421 @@ def test_negative_token_counts_clamp_to_zero(): ) assert result.compression == 0.0 assert result.prompt_caching == 0.0 + + +def _usage(fresh: int, cached: int, written: int, out: int) -> Usage: + """Usage as the spend log records it; `prompt_tokens` is the inclusive total.""" + return Usage( + prompt_tokens=fresh + cached + written, + completion_tokens=out, + total_tokens=fresh + cached + written + out, + prompt_tokens_details={"cached_tokens": cached, "cache_creation_tokens": written, "text_tokens": fresh}, + cache_read_input_tokens=cached, + cache_creation_input_tokens=written, + ) + + +def _savings(baseline: str, selected: str, usage: Usage, continuing: bool = True) -> float: + """Savings for a request, defaulting to a conversation already underway. + + `continuing=True` is the mid-conversation case, where the baseline had the prompt + cached and this request's write is what the switch cost. `continuing=False` is a + conversation's first turn, where nothing was cached for any model. + """ + return compute_autorouter_savings( + baseline_model=baseline, + selected_model=selected, + selected_provider="anthropic", + usage=usage, + conversation_continuing=continuing, + ) + + +def test_switching_models_mid_conversation_charges_the_cold_cache_write(): + """Staying on one model writes the cache once and reads it thereafter. Switching + leaves the new model cold, so it pays to write the whole prompt again; when that + charge outweighs the cheaper rates the route lost money and must report a loss. + + Pricing the baseline as if it too re-wrote the cache credits a charge it never + paid, which is how a losing switch used to read as the largest saving on the page. + """ + usage = _usage(fresh=3, cached=500, written=12304, out=500) + result = _savings("claude-sonnet-5", "claude-haiku-4-5", usage) + + sonnet = litellm.get_model_info("claude-sonnet-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + warm_baseline = ( + 3 * sonnet["input_cost_per_token"] + + 12804 * sonnet["cache_read_input_token_cost"] + + 500 * sonnet["output_cost_per_token"] + ) + actually_paid = ( + 3 * haiku["input_cost_per_token"] + + 500 * haiku["cache_read_input_token_cost"] + + 12304 * haiku["cache_creation_input_token_cost"] + + 500 * haiku["output_cost_per_token"] + ) + assert result == pytest.approx(warm_baseline - actually_paid) + assert result < 0, "a cache-thrashing switch must report a loss, not a saving" + + phantom = 12304 * sonnet["cache_creation_input_token_cost"] + assert result != pytest.approx(warm_baseline + phantom - actually_paid) + + +def test_a_cold_switch_never_beats_turning_caching_off(): + """Switching to a cold model makes it write the whole prompt again. That write is a + real cost of switching, so the same traffic must look worse than if caching were off + entirely. + + The baseline is priced as a warm cache even though this request read nothing: a + switch reads nothing precisely because the new model's cache is empty, and staying + on one model would have had the prompt cached already. Gating the warm baseline on + a read charged the baseline a write it would never repeat, which made a cold switch + report a larger saving than no caching at all. + """ + cold_switch = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(0, 0, 20_000, 1_000)) + caching_off = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(20_000, 0, 0, 1_000)) + + assert cold_switch < caching_off + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + warm_baseline = 20_000 * opus["cache_read_input_token_cost"] + 1_000 * opus["output_cost_per_token"] + actually_paid = 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] + assert cold_switch == pytest.approx(warm_baseline - actually_paid) + + +def test_moving_one_token_between_cache_buckets_does_not_move_the_answer(): + """A continuing conversation writes a few new tokens and reads the rest. Treating the + presence of a write as the signal for a switch made that ordinary increment flip the + result, so a request reading 19,999 and writing 1 landed somewhere entirely different + from one reading 20,000 and writing none. + """ + reads_nothing = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(0, 0, 20_000, 1_000)) + reads_one = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(0, 1, 19_999, 1_000)) + assert reads_one == pytest.approx(reads_nothing, abs=1e-4) + + +def test_multimodal_prompts_are_priced_on_the_baseline_too(): + """The baseline is this same request met by a warm cache, so every field it was + priced on has to survive. Rebuilding the details from the cache buckets alone + dropped the image and audio counts, which priced the baseline as a text-only + request that never ran and shrank the reported saving on multimodal traffic. + """ + details = {"cached_tokens": 0, "cache_creation_tokens": 16_000, "text_tokens": 0, "image_tokens": 4_000} + with_images = Usage( + prompt_tokens=20_000, + completion_tokens=1_000, + total_tokens=21_000, + prompt_tokens_details=details, + ) + baseline = _baseline_usage(with_images, conversation_continuing=True) + + assert baseline.prompt_tokens_details.image_tokens == 4_000, "image tokens must survive into the baseline" + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + priced, _ = generic_cost_per_token(model="claude-opus-5", usage=baseline, custom_llm_provider="anthropic") + text_only = 20_000 * opus["cache_read_input_token_cost"] + assert priced > text_only, "dropping the image tokens undercharges the baseline and hides the saving" + + +def test_the_baseline_is_never_charged_a_cache_write(): + """Carrying the details through must not carry the 5m/1h creation breakdown with + them. `generic_cost_per_token` charges a creation cost whenever that breakdown is + present, even against a zeroed creation count, which would put the phantom write + back on the baseline for every long-cache request. + """ + long_cache = Usage( + prompt_tokens=20_000, + completion_tokens=1_000, + total_tokens=21_000, + prompt_tokens_details={ + "cached_tokens": 0, + "cache_creation_tokens": 20_000, + "text_tokens": 0, + "cache_creation_token_details": {"ephemeral_1h_input_tokens": 20_000}, + }, + ) + baseline = _baseline_usage(long_cache, conversation_continuing=True) + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + priced, _ = generic_cost_per_token(model="claude-opus-5", usage=baseline, custom_llm_provider="anthropic") + assert priced == pytest.approx(20_000 * opus["cache_read_input_token_cost"]), ( + "the baseline reads a warm cache; it never pays to create one" + ) + + +def test_uncached_request_is_the_plain_rate_difference(): + usage = _usage(fresh=2000, cached=0, written=0, out=500) + sonnet = litellm.get_model_info("claude-sonnet-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + assert _savings("claude-sonnet-5", "claude-haiku-4-5", usage) == pytest.approx( + 2000 * (sonnet["input_cost_per_token"] - haiku["input_cost_per_token"]) + + 500 * (sonnet["output_cost_per_token"] - haiku["output_cost_per_token"]) + ) + + +def test_escalation_reports_its_real_cost(): + """Routing up to a pricier model is a real cost; hiding it behind a zero floor + would let the dashboard only ever move in one direction.""" + usage = _usage(fresh=2000, cached=0, written=0, out=500) + assert _savings("claude-haiku-4-5", "claude-sonnet-5", usage) < 0 + + +def test_autorouter_savings_zero_when_model_unchanged(): + assert _savings("claude-opus-5", "claude-opus-5", _usage(3, 500, 12304, 500)) == 0.0 + + +def test_autorouter_savings_unknown_baseline_fails_open_to_zero(): + assert _savings("totally-made-up-model-xyz", "claude-haiku-4-5", _usage(3, 500, 12304, 500)) == 0.0 + + +def test_autorouter_savings_zero_without_baseline(): + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + cache_read_input_tokens=0, + routing_decision=None, + usage_object=_cached_usage_object(), + ) + assert result.autorouter == 0.0 + + +def test_compute_savings_spend_carries_a_losing_switch_through(monkeypatch): + """The signed value must survive into SavingsSpend; clamping it here would put the + dashboard back to only ever showing gains.""" + monkeypatch.setattr(litellm, "autorouter_savings_baseline_model", "claude-sonnet-5") + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + cache_read_input_tokens=0, + routing_decision={"conversation_continuing": True}, + usage_object=_cached_usage_object(), + ) + assert result.autorouter < 0 + + +def test_the_driver_is_off_until_a_baseline_is_configured(): + """No configured counterfactual means there is nothing to measure against, so the + driver reports zero rather than inventing a model the operator never named.""" + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=1000, + cache_read_input_tokens=0, + routing_decision={"conversation_continuing": True}, + usage_object=_cached_usage_object(), + ) + assert result.autorouter == 0.0 + assert result.compression > 0, "the other drivers keep working" + + +def test_malformed_usage_object_does_not_fail_the_spend_write(): + """The daily spend write must survive an unusable usage_object; losing one row's + savings is recoverable, losing the row is not.""" + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=1000, + cache_read_input_tokens=0, + routing_decision={"conversation_continuing": True}, + usage_object={"prompt_tokens": ["not", "a", "number"]}, + ) + assert result.autorouter == 0.0 + assert result.compression > 0 + + +def test_model_without_cache_read_pricing_yields_no_caching_savings(): + """A model with no discounted cache-read rate cannot have saved anything by + reading from cache, so the driver must report zero rather than the full input rate.""" + model = "azure/gpt-3.5-turbo" + assert litellm.get_model_info(model=model).get("cache_read_input_token_cost") is None + result = compute_savings_spend( + model=model, + custom_llm_provider="azure", + compression_saved_tokens=0, + cache_read_input_tokens=5000, + ) + assert result.prompt_caching == 0.0 + + +def test_the_same_deployment_spelled_two_ways_is_not_a_switch(): + """The spend log records a normalized model name while the baseline arrives as the + operator wrote it in config. Comparing the raw strings makes a request that never + changed model look like a switch, and prices one deployment against itself.""" + # Must be a cached request: the baseline arm is priced against a warm cache and the + # selected arm against what was actually paid, so treating one deployment as two + # charges it a cold-cache write it never took, inventing a loss on a request that + # never changed model. An uncached request prices identically either way and would + # make this assertion vacuous. + usage = _usage(fresh=3, cached=500, written=12304, out=500) + assert _savings("anthropic/claude-opus-5", "claude-opus-5", usage) == 0.0 + assert _savings("claude-opus-5", "anthropic/claude-opus-5", usage) == 0.0 + + +def test_baseline_is_priced_under_its_own_provider(): + """Two providers can serve the same bare model name at different rates, so dropping + the provider prices the baseline against a vendor the operator never named. Here it + decides whether routing reads as a saving or a loss.""" + usage = Usage(prompt_tokens=100_000, completion_tokens=10_000, total_tokens=110_000) + azure = compute_autorouter_savings( + baseline_model="azure_ai/deepseek-r1", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=usage, + ) + deepseek = compute_autorouter_savings( + baseline_model="deepseek/deepseek-r1", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=usage, + ) + assert azure != pytest.approx(deepseek) + assert azure > 0 > deepseek + + +def test_unresolvable_baseline_fails_open_to_zero(): + usage = _usage(fresh=2000, cached=0, written=0, out=500) + assert _savings("no-such-provider-xyz/no-such-model", "claude-haiku-4-5", usage) == 0.0 + + +def test_a_first_turn_is_the_rate_difference_not_a_switch_penalty(): + """Nothing was cached anywhere on a conversation's first turn, so the baseline would + have paid the same cache write. Charging it to the selected arm alone reported a + fraction of the real saving; on this shape roughly 4% of it. + """ + usage = _usage(fresh=0, cached=0, written=20_000, out=1_000) + first_turn = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, continuing=False) + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + both_write = (20_000 * opus["cache_creation_input_token_cost"] + 1_000 * opus["output_cost_per_token"]) - ( + 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] + ) + assert first_turn == pytest.approx(both_write) + + mid_conversation = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage) + assert first_turn > mid_conversation * 10, "a first turn must not be priced as a switch" + + +def test_a_first_turn_that_saves_money_never_reports_a_loss(): + """The write premium is fixed by prompt size while the saving grows with completion + length, so charging the write to a first turn made short answers over a large cached + prompt read as losses on requests that genuinely saved. That is the shape most likely + to be on the dashboard, and the sign has to be right. + """ + short_answer = _usage(fresh=0, cached=0, written=20_000, out=200) + assert _savings("anthropic/claude-opus-5", "claude-haiku-4-5", short_answer, continuing=False) > 0 + assert _savings("anthropic/claude-opus-5", "claude-haiku-4-5", short_answer) < 0 + + +def test_an_undetermined_conversation_shape_stays_conservative(): + """The default must charge the write. A caller that cannot be read, or a surface the + router never classified, has said nothing about whether the baseline was warm, and a + savings figure must not inflate on a guess. + """ + usage = _usage(fresh=0, cached=0, written=20_000, out=1_000) + defaulted = compute_autorouter_savings( + baseline_model="anthropic/claude-opus-5", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=usage, + ) + assert defaulted == pytest.approx(_savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage)) + assert defaulted < _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, continuing=False) + + +def test_a_continuing_turn_on_the_same_model_writes_its_growth_on_both_arms(): + """A conversation that grew by a few tokens writes those on whatever model serves + it, and they are new to every model, so the baseline would have written them too. + Moving them into the baseline's read bucket forgives it a write it really owes and + shrinks the reported saving on ordinary steady-state traffic. + """ + usage = _usage(fresh=0, cached=19_900, written=100, out=1_000) + opus = litellm.get_model_info("claude-opus-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + + def cost(info: dict) -> float: + return ( + 19_900 * info["cache_read_input_token_cost"] + + 100 * info["cache_creation_input_token_cost"] + + 1_000 * info["output_cost_per_token"] + ) + + both_write_the_growth = cost(opus) - cost(haiku) + assert _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage) == pytest.approx(both_write_the_growth) + + +def test_a_switch_onto_a_partly_cached_model_still_pays_for_the_write(): + """A model holding a small prefix of this prompt still has to write the rest, and + that write is the switch's cost. Keying the same-model case off reading *anything* + rather than reading *most of it* would hand this request the full rate gap and + inflate the saving by an order of magnitude. + """ + mostly_written = _usage(fresh=0, cached=500, written=19_500, out=1_000) + reported = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", mostly_written) + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + if_treated_as_same_model = ( + 500 * opus["cache_read_input_token_cost"] + + 19_500 * opus["cache_creation_input_token_cost"] + + 1_000 * opus["output_cost_per_token"] + ) - ( + 500 * haiku["cache_read_input_token_cost"] + + 19_500 * haiku["cache_creation_input_token_cost"] + + 1_000 * haiku["output_cost_per_token"] + ) + assert reported < if_treated_as_same_model / 10, "a mostly-cold switch must not be priced as a continuation" + +def test_a_baseline_that_prices_caching_implicitly_still_pays_for_its_prompt(): + """OpenAI, Azure and Gemini entries carry no `cache_creation_input_token_cost`, + because those providers cache implicitly and charge nothing to write. Leaving this + request's written tokens in the creation bucket priced them at the 0.0 the cost + resolver falls back to, so the baseline carried a 20k prompt for free and a first + turn that saved money reported a loss. Those tokens are plain input on such a model. + """ + first_turn = _usage(fresh=0, cached=0, written=20_000, out=1_000) + reported = compute_autorouter_savings( + baseline_model="gpt-5", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=first_turn, + conversation_continuing=False, + ) + + gpt5 = litellm.get_model_info("gpt-5", "openai") + assert gpt5.get("cache_creation_input_token_cost") is None, "pick a baseline with no cache-write rate" + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + baseline_pays_input = 20_000 * gpt5["input_cost_per_token"] + 1_000 * gpt5["output_cost_per_token"] + actually_paid = ( + 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] + ) + assert reported == pytest.approx(baseline_pays_input - actually_paid) + assert reported > 0, "routing a cold first turn onto a cheaper model is a saving, not a loss" + + +def test_a_baseline_with_no_cache_read_rate_is_charged_its_input_rate(): + """The same hole on the other bucket. A baseline whose entry has no + `cache_read_input_token_cost` reads for 0.0, so a continuing turn priced the whole + prompt at nothing and every switch away from it reported a loss. + """ + continuing = _usage(fresh=0, cached=0, written=20_000, out=1_000) + reported = compute_autorouter_savings( + baseline_model="xai/grok-4", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=continuing, + conversation_continuing=True, + ) + + grok = litellm.get_model_info("grok-4", "xai") + assert grok.get("cache_read_input_token_cost") is None, "pick a baseline with no cache-read rate" + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + baseline_pays_input = 20_000 * grok["input_cost_per_token"] + 1_000 * grok["output_cost_per_token"] + actually_paid = ( + 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] + ) + assert reported == pytest.approx(baseline_pays_input - actually_paid) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 3a94b1e0f85..2f249241f21 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -4094,6 +4094,23 @@ class TestRecordRoutingDecision: assert request_kwargs == {} + def test_clearing_the_decision_takes_the_savings_facts_with_it(self): + """A fallback to a plain model group re-enters the hook with the same + `request_kwargs`. The baseline and the conversation shape ride inside the + decision rather than beside it, so one clear cannot leave either behind and + attribute an auto-router saving to a deployment that never routed.""" + decision = { + "router_model_name": "smart-router", + "router_type": "complexity", + "routed_model": "gpt-4o-mini", + "savings_baseline_model": "anthropic/claude-opus-5", + "conversation_continuing": False, + } + request_kwargs: Dict = {"litellm_metadata": {"routing_decision": decision}} + Router._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None) + assert request_kwargs["litellm_metadata"] == {} + + class TestEscalationIsRecordedConsistently: """An escalation keyword records two separate facts on every path: that the caller asked, and whether the tier actually moved. Dropping the ask when there is nowhere @@ -5039,3 +5056,142 @@ class TestClassifierTrustBoundary: assert "Classify only the current message" not in system_prompt assert "using the earlier turns quoted above it as context" in system_prompt assert "rate the work it approves rather than the reply itself" in system_prompt + + +class TestConversationShapeDiscriminator: + """Whether the counterfactual single model would already have had the prompt cached.""" + + @staticmethod + def _router(mock_router_instance, basic_config) -> ComplexityRouter: + return ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, "session_affinity": False}, + ) + + @pytest.mark.asyncio + async def test_a_single_ask_is_a_first_turn(self, mock_router_instance, basic_config): + """Nothing is cached for any model yet, so the baseline would have paid the same + cache write and the saving is the plain rate difference.""" + mock_router_instance.cache = DualCache() + result = await self._router(mock_router_instance, basic_config).async_pre_routing_hook( + model="test-model", + request_kwargs={"metadata": {}}, + messages=[{"role": "user", "content": "Hello!"}], + ) + assert result.routing_decision["conversation_continuing"] is False + + @pytest.mark.asyncio + async def test_a_second_ask_means_the_baseline_was_already_warm(self, mock_router_instance, basic_config): + """An earlier turn was served, so a single-model deployment wrote the prompt then + and would only read it now; this request's write is what switching cost.""" + mock_router_instance.cache = DualCache() + result = await self._router(mock_router_instance, basic_config).async_pre_routing_hook( + model="test-model", + request_kwargs={"metadata": {}}, + messages=[ + {"role": "user", "content": "First question about the codebase"}, + {"role": "assistant", "content": "Here is the answer"}, + {"role": "user", "content": "Hello!"}, + ], + ) + assert result.routing_decision["conversation_continuing"] is True + + @pytest.mark.asyncio + async def test_it_needs_no_session_id(self, mock_router_instance, basic_config): + """The whole point of reading the conversation rather than remembering it: a + caller that sends no session header is still classified correctly.""" + mock_router_instance.cache = DualCache() + router = self._router(mock_router_instance, basic_config) + first = await router.async_pre_routing_hook( + model="test-model", request_kwargs={}, messages=[{"role": "user", "content": "Hello!"}] + ) + later = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[ + {"role": "user", "content": "First question"}, + {"role": "assistant", "content": "Answer"}, + {"role": "user", "content": "Hello!"}, + ], + ) + assert first.routing_decision["conversation_continuing"] is False + assert later.routing_decision["conversation_continuing"] is True + + @pytest.mark.asyncio + async def test_it_touches_no_cache(self, mock_router_instance, basic_config): + """Reading the request instead of remembering it is what removes the routing-path + round-trip, and with it a cache failure that would read as a first turn.""" + cache = AsyncMock() + cache.async_get_cache = AsyncMock(return_value=None) + mock_router_instance.cache = cache + result = await self._router(mock_router_instance, basic_config).async_pre_routing_hook( + model="test-model", + request_kwargs={"metadata": {}}, + messages=[{"role": "user", "content": "Hello!"}], + ) + assert result.routing_decision["conversation_continuing"] is False + assert cache.async_get_cache.await_count == 0 + assert cache.async_set_cache.await_count == 0 + + @pytest.mark.parametrize( + "history", + [ + pytest.param( + [ + {"role": "user", "content": "do X"}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "1", "name": "t", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "1", "content": "r"}]}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "2", "name": "t", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "2", "content": "r"}]}, + ], + id="messages-api-tool-result-blocks", + ), + pytest.param( + [ + {"role": "user", "content": "do X"}, + {"role": "assistant", "tool_calls": [{"id": "1"}]}, + {"role": "tool", "tool_call_id": "1", "content": "r"}, + ], + id="chat-completions-tool-role", + ), + ], + ) + def test_an_agent_loop_on_one_human_ask_is_not_a_first_turn(self, history): + """An agent can run twenty turns on a single human ask: its tool traffic rides + `tool_result` blocks that flatten to empty text and `tool` roles. Counting human + asks read that as a first turn and handed it the untouched-write arithmetic, + which is the one direction this must never fail in, because it inflates.""" + from litellm.router_strategy.complexity_router.complexity_router import _conversation_is_continuing + + assert _conversation_is_continuing(history) is True + + def test_a_system_prompt_does_not_make_a_first_turn_look_continued(self): + from litellm.router_strategy.complexity_router.complexity_router import _conversation_is_continuing + + assert _conversation_is_continuing([{"role": "system", "content": "s"}, {"role": "user", "content": "hi"}]) is False + + def test_unreadable_messages_stay_conservative(self): + """No messages says nothing about the baseline's cache, so it keeps charging the + write and under-claims rather than inflating.""" + from litellm.router_strategy.complexity_router.complexity_router import _conversation_is_continuing + + assert _conversation_is_continuing(None) is True + assert _conversation_is_continuing([]) is True + assert _conversation_is_continuing([{"role": "user", "content": ""}]) is False + + @pytest.mark.asyncio + async def test_the_shape_travels_on_every_pre_routing_response(self): + """A response without it defaults to charging the write, silently undoing the fix + for whichever routing path forgot it.""" + import inspect + + from litellm.router_strategy.complexity_router import complexity_router as module + + source = inspect.getsource(module.ComplexityRouter.async_pre_routing_hook) + inspect.getsource( + module.ComplexityRouter._classify_and_route + ) + builds = source.split("self._build_routing_decision(")[1:] + assert builds + missing = [i for i, block in enumerate(builds) if "conversation_continuing=conversation_continuing" not in block.split("),")[0]] + assert not missing, f"routing decisions {missing} do not carry the conversation shape" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 0f4eacc820c..1ea4113b821 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -23521,6 +23521,11 @@ export interface components { * @default 0 */ total_api_requests: number; + /** + * Total Autorouter Savings Spend + * @default 0 + */ + total_autorouter_savings_spend: number; /** * Total Cache Creation Input Tokens * @default 0 @@ -31427,6 +31432,11 @@ export interface components { * @default 0 */ api_requests: number; + /** + * Autorouter Savings Spend + * @default 0 + */ + autorouter_savings_spend: number; /** * Cache Creation Input Tokens * @default 0 From 22f68c0c6ba334369ab619994b5adb39c7d9f4c6 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 3 Aug 2026 20:46:09 -0700 Subject: [PATCH 112/123] fix(spend): read what a request cost from the record instead of pricing it again (#35736) The auto-router savings driver recomputes what the served request cost, but that request is not a counterfactual: it ran, and the cost calculator already billed it and wrote the number down. Recomputing means restating every pricing dimension the biller applied, and the two this missed were enough to halve it. A request billed at a priority tier is recomputed at standard rates, and a regional host's uplift is dropped entirely, so the driver writes a savings figure into the same rollup row as the `spend` it disagrees with. On `gpt-5.4-mini` at priority the row is billed 0.024 and the driver prices the same usage at 0.012. Neither omission cancels between the two arms, because both are per-model. The uplift is a multiplier read off each model's own entry, so 1.1*A - 1.1*B is 1.1*(A-B) and a model without one does not move at all. Tier coverage is sparser and asymmetric: `gpt-5.6` has priority rates and `gpt-5.4-nano` has none. `cost_breakdown` already carries the answer and already reaches the call site. The cost calculator records it, it rides the standard logging payload into the spend log's metadata, and OTEL, the log drawer and the response headers all read it rather than re-deriving; this driver was the only downstream consumer in the tree still pricing a completed request from its tokens. `input_cost` and `output_cost` sum to exactly what the pricer returns, so the served arm reads them. Tool spend, discount and margin stay out, since the counterfactual cannot be priced with them and charging them to one arm alone would read as the router losing money on every tool call. The baseline never ran, so it is still priced through the cost engine, now on the basis the biller used. `CostBreakdown` carries that basis because it cannot be recovered afterwards: the tier the biller used comes from `optional_params`, which no log record keeps, and the served tier that does survive on the usage object is a different fact with the opposite precedence. Rows written before this shipped carry no basis and price at standard rates, exactly as they do today; there is no backfill. Two smaller things in the same path. The router is passed as a provider rather than a router, so a spend write that was never auto-routed no longer fetches and discards one, and the complexity router resolves its messages once per hook instead of once per consumer. --- litellm/cost_calculator.py | 11 ++ litellm/litellm_core_utils/litellm_logging.py | 6 + litellm/proxy/db/db_spend_update_writer.py | 3 +- litellm/proxy/spend_tracking/savings.py | 124 ++++++++++++++++-- .../complexity_router/complexity_router.py | 15 ++- litellm/types/utils.py | 11 +- .../proxy/spend_tracking/test_savings.py | 83 ++++++++++++ 7 files changed, 239 insertions(+), 14 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index f04a9d61d4a..25d448e48e9 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1051,6 +1051,8 @@ def _store_cost_breakdown_in_logging_obj( cache_read_cost: float | None = None, cache_creation_cost: float | None = None, reasoning_cost: float | None = None, + service_tier: str | None = None, + data_residency: str | None = None, ) -> None: """ Helper function to store cost breakdown in the logging object. @@ -1068,6 +1070,8 @@ def _store_cost_breakdown_in_logging_obj( margin_percent: Margin percentage applied (0.10 = 10%) margin_fixed_amount: Fixed margin amount in USD margin_total_amount: Total margin added in USD + service_tier: Tier the costs above were priced on, already resolved + data_residency: Region uplift the costs above were priced on, already resolved """ if litellm_logging_obj is None: return @@ -1089,6 +1093,8 @@ def _store_cost_breakdown_in_logging_obj( cache_read_cost=cache_read_cost, cache_creation_cost=cache_creation_cost, reasoning_cost=reasoning_cost, + service_tier=service_tier, + data_residency=data_residency, ) except Exception as breakdown_error: @@ -1469,6 +1475,8 @@ def completion_cost( margin_percent=margin_percent, margin_fixed_amount=margin_fixed_amount, margin_total_amount=margin_total_amount, + service_tier=service_tier, + data_residency=data_residency, ) return _final_cost @@ -1657,6 +1665,8 @@ def completion_cost( cache_read_cost=_cache_read_cost, cache_creation_cost=_cache_creation_cost, reasoning_cost=_reasoning_cost, + service_tier=service_tier, + data_residency=data_residency, ) return _final_cost @@ -2351,6 +2361,7 @@ def handle_realtime_stream_cost_calculation( cost_for_built_in_tools_cost_usd_dollar=0.0, total_cost_usd_dollar=total_cost, additional_costs={"transcription_cost": transcription_cost} if transcription_cost > 0 else None, + data_residency=data_residency, ) return total_cost diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 66d82bd18f1..f38b2859259 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1286,6 +1286,8 @@ class Logging(LiteLLMLoggingBaseClass): cache_read_cost: float | None = None, cache_creation_cost: float | None = None, reasoning_cost: float | None = None, + service_tier: str | None = None, + data_residency: str | None = None, ) -> None: """ Helper method to store cost breakdown in the logging object. @@ -1302,6 +1304,8 @@ class Logging(LiteLLMLoggingBaseClass): margin_percent: Margin percentage applied (0.10 = 10%) margin_fixed_amount: Fixed margin amount in USD margin_total_amount: Total margin added in USD + service_tier: Tier the costs above were priced on, already resolved + data_residency: Region uplift the costs above were priced on, already resolved """ self.cost_breakdown = CostBreakdown( @@ -1309,6 +1313,8 @@ class Logging(LiteLLMLoggingBaseClass): output_cost=output_cost, total_cost=total_cost, tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar, + service_tier=service_tier, + data_residency=data_residency, ) if cache_read_cost is not None and cache_read_cost > 0: self.cost_breakdown["cache_read_cost"] = cache_read_cost diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 7c9ee96e809..b24ed4b8282 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -1885,8 +1885,9 @@ class DBSpendUpdateWriter: cache_read_input_tokens=cache_read_input_tokens, routing_decision=_metadata.get("routing_decision"), model_id=payload.get("model_id"), - llm_router=_get_llm_router(), + llm_router=_get_llm_router, usage_object=usage_obj, + cost_breakdown=_metadata.get("cost_breakdown"), ) daily_transaction = BaseDailySpendTransaction( diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 1a1c813d784..a318b11e4c0 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -8,7 +8,7 @@ are known) and summed into the daily tables; tokens cannot be priced after they have been aggregated across models. """ -from collections.abc import Mapping +from collections.abc import Callable, Mapping from typing import TYPE_CHECKING, NamedTuple import litellm @@ -105,11 +105,81 @@ def _model_info(model: _ModelIdentity) -> ModelInfo | None: return None -def _cost_of_usage(model: _ModelIdentity, usage: Usage, model_info: ModelInfo | None = None) -> float | None: +class PricingBasis(NamedTuple): + """The tier and region a request was priced on, as the cost calculator resolved them. + + Read back off the request's recorded ``cost_breakdown`` rather than re-derived. The + tier the biller used comes from ``optional_params``, which no log record carries, and + the served tier that does survive on the usage object is a different fact with the + opposite precedence, so a spend-time re-derivation would disagree with the invoice on + exactly the requests where the tier changed the price. + """ + + service_tier: str | None = None + data_residency: str | None = None + + +_STANDARD_RATES = PricingBasis() + + +def _pricing_basis(cost_breakdown: Mapping[str, object] | None) -> PricingBasis: + """The basis recorded on a request, defaulting to standard rates when absent. + + Rows written before this field shipped carry neither key, and there is no backfill: + they price at standard rates, which is what they already did. + + Both values survive a JSON round trip on the way here, so neither is guaranteed to be + a string. `generic_cost_per_token` calls `.lower()` on both without a type check, and + the resulting `AttributeError` would be swallowed into a silent zero by the caller's + `except`, so anything that is not a string is dropped here instead. + """ + if not cost_breakdown: + return _STANDARD_RATES + service_tier = cost_breakdown.get("service_tier") + data_residency = cost_breakdown.get("data_residency") + return PricingBasis( + service_tier=service_tier if isinstance(service_tier, str) else None, + data_residency=data_residency if isinstance(data_residency, str) else None, + ) + + +def _recorded_token_cost(cost_breakdown: Mapping[str, object] | None) -> float | None: + """What the biller charged for this request's tokens, or ``None`` when unrecorded. + + ``input_cost`` already carries the cache buckets, so it and ``output_cost`` sum to + exactly what `generic_cost_per_token` returns for the same request; the separate + ``cache_read_cost`` and ``cache_creation_cost`` entries decompose that sum rather than + adding to it, and including them would charge those tokens twice. + + Built-in tool cost, discount and margin are deliberately left out. They are properties + of the request and the operator's contract rather than of the model the router picked, + so they land on both sides of the comparison or neither, and only the total the + counterfactual can also be priced on belongs here. + """ + if not cost_breakdown: + return None + input_cost = cost_breakdown.get("input_cost") + output_cost = cost_breakdown.get("output_cost") + if not isinstance(input_cost, (int, float)) or not isinstance(output_cost, (int, float)): + return None + return float(input_cost) + float(output_cost) + + +def _cost_of_usage( + model: _ModelIdentity, + usage: Usage, + model_info: ModelInfo | None = None, + basis: PricingBasis = _STANDARD_RATES, +) -> float | None: """What ``usage`` costs on ``model``, or ``None`` when the model has no pricing.""" try: prompt_cost, completion_cost = generic_cost_per_token( - model=model.model, usage=usage, custom_llm_provider=model.provider, model_info=model_info + model=model.model, + usage=usage, + custom_llm_provider=model.provider, + service_tier=basis.service_tier, + data_residency=basis.data_residency, + model_info=model_info, ) except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings verbose_proxy_logger.debug( @@ -228,6 +298,7 @@ def compute_autorouter_savings( usage: Usage, conversation_continuing: bool = True, selected_info: ModelInfo | None = None, + cost_breakdown: Mapping[str, object] | None = None, ) -> float: """Net dollars the router saved, or cost, by serving this request on ``selected_model``. @@ -237,6 +308,20 @@ def compute_autorouter_savings( dashboard has to be able to say so. Zero when both sides resolve to the same deployment, or when either cannot be resolved or priced. + Only one side of this subtraction is a counterfactual. What the request cost on the + model that served it is a number the operator was actually billed, and the cost + calculator already wrote it down, so ``cost_breakdown`` is read rather than + re-derived. Recomputing it means restating every pricing dimension the biller + applied, and each one omitted is a silent disagreement with the ``spend`` column + beside it; a request billed at a priority tier recomputed at standard rates reads as + half its real cost. + + The baseline has no such record, since it never ran, so it is priced through the same + cost engine on the basis the biller used for this request. An operator running that + one model instead of the router would have sent this request to the same tier and the + same region, because both are properties of the request and the deployment's + contract, not of which model the router happened to pick. + ``conversation_continuing`` says whether the baseline would already have had this prompt cached. It defaults to True because that is the conservative reading: a request whose shape the router could not determine is charged the write and @@ -255,11 +340,16 @@ def compute_autorouter_savings( # name alone reports as zero. if baseline == selected: return 0.0 + basis = _pricing_basis(cost_breakdown) baseline_info = _model_info(baseline) baseline_cost = _cost_of_usage( - baseline, _baseline_usage(usage, conversation_continuing, baseline_info), baseline_info + baseline, _baseline_usage(usage, conversation_continuing, baseline_info), baseline_info, basis ) - selected_cost = _cost_of_usage(selected, usage, selected_info) + # Falls back to pricing the request only when the biller recorded nothing, which is + # every row written before the breakdown carried its basis. + selected_cost = _recorded_token_cost(cost_breakdown) + if selected_cost is None: + selected_cost = _cost_of_usage(selected, usage, selected_info, basis) if baseline_cost is None or selected_cost is None: return 0.0 return baseline_cost - selected_cost @@ -287,7 +377,8 @@ def compute_savings_spend( routing_decision: Mapping[str, object] | None = None, usage_object: Mapping[str, object] | None = None, model_id: str | None = None, - llm_router: "Router | None" = None, + llm_router: "Callable[[], Router | None] | None" = None, + cost_breakdown: Mapping[str, object] | None = None, ) -> SavingsSpend: """ Dollar savings for one request, split by optimization driver. @@ -299,6 +390,17 @@ def compute_savings_spend( baseline the router recorded on its ``routing_decision``, and are zero unless the two differ. That record also says whether the conversation was already underway, which is what tells a mid-conversation switch from a first turn. + + ``llm_router`` is passed as a provider rather than a router because every spend write + calls this and only auto-routed ones need one, so looking it up eagerly at the call + site would fetch and discard it on the rest. + + ``cost_breakdown`` is what the cost calculator recorded for this request, and it + carries both what the request really cost and the tier and region it was priced on. + Only the auto-router driver reads it. Compression and prompt caching price a + hypothetical token delta off flat rate keys, so they are blind to tiered pricing in + the same way; that is pre-existing behaviour on two shipped drivers rather than + something introduced here, and moving those numbers is its own change. """ input_cost, cache_read_cost = _input_and_cache_read_cost(model, custom_llm_provider) compression = max(compression_saved_tokens, 0) * input_cost @@ -311,19 +413,23 @@ def compute_savings_spend( # The counterfactual is one model an operator would have run instead of the router, # configured once for the proxy rather than derived per request. Unset means the # driver is off; a routing decision is what says this request was auto-routed at all. + # Both are checked before anything is resolved, because every spend write reaches + # here and only auto-routed ones can produce a number. + baseline_model = litellm.autorouter_savings_baseline_model decision = routing_decision if isinstance(routing_decision, Mapping) else {} autorouter = ( compute_autorouter_savings( - baseline_model=litellm.autorouter_savings_baseline_model, + baseline_model=baseline_model, selected_model=model, selected_provider=custom_llm_provider, usage=usage, # Absent means the router never recorded a shape, which is the conservative # reading: charge the cache write rather than claim a first turn's saving. conversation_continuing=decision.get("conversation_continuing") is not False, - selected_info=_effective_model_info(llm_router, model_id, model or ""), + selected_info=_effective_model_info(llm_router() if llm_router else None, model_id, model or ""), + cost_breakdown=cost_breakdown, ) - if decision + if decision and baseline_model else 0.0 ) return SavingsSpend(compression=compression, prompt_caching=prompt_caching, autorouter=autorouter) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index ca2e7fcc038..11a1b686c2e 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1428,7 +1428,11 @@ class ComplexityRouter(CustomLogger): if isinstance(metadata, dict): metadata[RETURN_RAW_MODEL_NAME_METADATA_KEY] = True - conversation_continuing = _conversation_is_continuing(self._resolve_messages(messages, request_kwargs)) + # Resolved once for the whole hook. Resolution converts Responses API input into + # chat-completions messages, so it is real work on every non-chat surface, and + # both the conversation shape and the classifier read the same list. + resolved_messages = self._resolve_messages(messages, request_kwargs) + conversation_continuing = _conversation_is_continuing(resolved_messages) use_session_affinity = self.config.session_affinity and not self.config.plugins session_id = self._get_session_id_from_request_kwargs(request_kwargs) if use_session_affinity else None @@ -1440,7 +1444,6 @@ class ComplexityRouter(CustomLogger): routed_model: str | None = pinned_model pin_escalation_keyword: str | None = None if self.escalation_keywords: - resolved_messages = self._resolve_messages(messages, request_kwargs) user_message = _newest_turn_ask(resolved_messages) if resolved_messages else None if user_message is not None: pin_escalation_keyword = self._matched_escalation_keyword(user_message) @@ -1487,6 +1490,7 @@ class ComplexityRouter(CustomLogger): input=input, specific_deployment=specific_deployment, conversation_continuing=conversation_continuing, + resolved_messages=resolved_messages, ) if cache_key is not None and response is not None: await self.litellm_router_instance.cache.async_set_cache( @@ -1504,6 +1508,7 @@ class ComplexityRouter(CustomLogger): input: str | list | None = None, specific_deployment: bool | None = False, conversation_continuing: bool = True, + resolved_messages: Sequence[Mapping[str, object]] | None = None, ) -> PreRoutingHookResponse | None: """ Classifies the request by complexity and returns the appropriate model. @@ -1516,13 +1521,17 @@ class ComplexityRouter(CustomLogger): messages: The messages in the request. input: Optional input for Responses API or embeddings. specific_deployment: Whether a specific deployment was requested. + resolved_messages: Messages the caller already resolved, to avoid converting + the request format a second time. Resolved here when absent, so a direct + caller does not have to. Returns: PreRoutingHookResponse with the routed model, or None if no routing needed. """ from litellm.types.router import PreRoutingHookResponse - resolved_messages = self._resolve_messages(messages, request_kwargs) + if resolved_messages is None: + resolved_messages = self._resolve_messages(messages, request_kwargs) if not resolved_messages: verbose_router_logger.debug("ComplexityRouter: No messages could be resolved, skipping routing") diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 21d69d9925a..c63f4d53572 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2991,9 +2991,18 @@ class CachingDetails(TypedDict): class CostBreakdown(TypedDict, total=False): """ - Detailed cost breakdown for a request + Detailed cost breakdown for a request. + + ``service_tier`` and ``data_residency`` record the pricing basis the cost was + computed on, not what the caller asked for. A consumer that has to price a + counterfactual against this request (what another model would have charged for + it) needs the same basis to compare like with like, and re-deriving it from the + request is not possible after the fact: the tier the biller used comes from + ``optional_params``, which no log record carries. """ + service_tier: Optional[str] + data_residency: Optional[str] input_cost: float # Cost of raw (non-cached) input tokens only cache_read_cost: float # Cost of cache-read tokens (discounted rate) cache_creation_cost: float # Cost of cache-write tokens (premium rate) diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 6d0195b632b..1d6b0da561d 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -534,3 +534,86 @@ def test_a_baseline_with_no_cache_read_rate_is_charged_its_input_rate(): 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] ) assert reported == pytest.approx(baseline_pays_input - actually_paid) + + +def _breakdown(input_cost: float, output_cost: float = 0.0, **extra: object) -> dict: + """A `cost_breakdown` as the cost calculator records it on the spend log.""" + return {"input_cost": input_cost, "output_cost": output_cost, **extra} + + +def test_the_served_arm_is_read_from_the_record_not_repriced(): + """What the request cost on the model that served it is not a counterfactual; the + cost calculator already billed it and wrote the number down. Recomputing it restates + every pricing dimension the biller applied and drops the ones it forgets, so the + driver disagrees with the `spend` column beside it. + + Pinned with a negotiated rate no public map lookup can produce, so re-pricing from + the model name cannot land on this number. Tool spend and margin are recorded too and + must stay out: the baseline cannot be priced with them, so charging them to the + served arm alone would read as the router losing money on every tool call. + """ + usage = _usage(fresh=20_000, cached=0, written=0, out=1_000) + negotiated_input, negotiated_output = 0.0123, 0.0456 + + reported = compute_autorouter_savings( + baseline_model="anthropic/claude-opus-5", + selected_model="gpt-5.5", + selected_provider="openai", + usage=usage, + conversation_continuing=False, + cost_breakdown=_breakdown( + negotiated_input, + negotiated_output, + tool_usage_cost=5.0, + margin_total_amount=2.0, + total_cost=negotiated_input + negotiated_output + 7.0, + ), + ) + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + public = 20_000 * opus["input_cost_per_token"] + 1_000 * opus["output_cost_per_token"] + assert reported == pytest.approx(public - (negotiated_input + negotiated_output)) + + +@pytest.mark.parametrize( + "basis, expected_multiplier", + [ + pytest.param({"service_tier": "priority"}, 2.0, id="priority tier doubles the baseline"), + pytest.param({"data_residency": "eu"}, 1.1, id="eu residency uplifts the baseline"), + pytest.param({}, 1.0, id="no basis recorded prices at standard"), + pytest.param(None, 1.0, id="row predating the field prices at standard"), + pytest.param({"service_tier": True, "data_residency": 17}, 1.0, id="a non-string basis is dropped"), + ], +) +def test_the_baseline_is_priced_on_the_basis_the_request_was_billed_at(basis, expected_multiplier): + """A request billed at a priority tier, or through a regional host, would have been + billed the same way on the single model an operator ran instead of the router, so the + counterfactual carries that basis too. Dropping it prices the two arms from different + books; neither multiplier cancels out of the difference, because both are per-model. + + The served model has no tiered rates and no uplift of its own, so only the baseline + can move: a fix that forwards the basis to the served arm alone leaves these numbers + unchanged. The non-string case guards the JSON round trip, where `.lower()` inside + the pricer would raise and be swallowed into a silent $0.00 for the whole row. + """ + gpt = litellm.get_model_info("gpt-5.5", "openai") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + assert gpt.get("input_cost_per_token_priority") == 2 * gpt["input_cost_per_token"] + assert gpt.get("regional_processing_uplift_multiplier_eu") == 1.1 + assert haiku.get("input_cost_per_token_priority") is None, "served model must not move with the basis" + assert haiku.get("regional_processing_uplift_multiplier_eu") is None + + usage = _usage(fresh=20_000, cached=0, written=0, out=1_000) + served = 20_000 * haiku["input_cost_per_token"] + 1_000 * haiku["output_cost_per_token"] + + reported = compute_autorouter_savings( + baseline_model="openai/gpt-5.5", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=usage, + conversation_continuing=False, + cost_breakdown=None if basis is None else _breakdown(served, **basis), + ) + + baseline = 20_000 * gpt["input_cost_per_token"] + 1_000 * gpt["output_cost_per_token"] + assert reported == pytest.approx(expected_multiplier * baseline - served) From 042ef4cc487974dacc3dab8486663107b6dd4763 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Tue, 4 Aug 2026 05:47:43 +0200 Subject: [PATCH 113/123] perf: install hiredis so redis-py parses replies with its C parser (#35709) --- pyproject.toml | 3 + tests/test_litellm/test_redis.py | 19 +++++++ uv.lock | 94 ++++++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index addf56a3588..9b9fa00f5ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,9 @@ proxy = [ "pyyaml>=6.0.3,<7.0", "rq>=2.7.0,<3.0", "orjson>=3.11.6,<4.0", + # redis-py's C response parser. It arrives with redis (via rq) either way; naming + # it here is what makes redis-py select _HiredisParser instead of the Python one. + "hiredis>=3.0.0,<4.0", "apscheduler>=3.11.2,<4.0", "fastapi-sso>=0.19.0,<1.0", "PyJWT>=2.13.0,<3.0", diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index e0fa800723d..14c242f1096 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -845,3 +845,22 @@ def test_url_config_drops_kwargs_the_connection_cannot_accept(client_only_kwarg, assert pool is not None pool.make_connection() assert pool.connection_kwargs.get("socket_timeout") == 5.0 + + +def test_redis_uses_the_hiredis_response_parser(): + """The C parser must be the one redis-py actually picks. + + hiredis is declared in the `proxy` extra purely for speed; nothing imports it, so + dropping it from pyproject.toml would silently fall back to the pure-Python parser + with no other symptom. redis-py selects it at import time, so asserting on the + selection is what catches that. + """ + from redis._parsers import _HiredisParser + from redis.connection import HIREDIS_AVAILABLE, DefaultParser + + assert HIREDIS_AVAILABLE, "hiredis is not installed; redis-py fell back to the pure-Python parser" + assert DefaultParser is _HiredisParser, f"redis-py selected {DefaultParser.__name__}, expected _HiredisParser" + + client = get_redis_client(host="redis-host", port=6379) + connection = client.connection_pool.make_connection() + assert isinstance(connection._parser, _HiredisParser) diff --git a/uv.lock b/uv.lock index 15e65c9dffd..70048926945 100644 --- a/uv.lock +++ b/uv.lock @@ -3159,6 +3159,98 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/94/3b66b148778ee100dcfd69c2ca22b57b41b44d3063ceec934f209e9184ce/hf_xet-1.5.0-cp37-abi3-win_arm64.whl", hash = "sha256:b6c9df403040248c76d808d3e047d64db2d923bae593eb244c41e425cf6cd7be", size = 3806916, upload-time = "2026-05-06T06:18:21.7Z" }, ] +[[package]] +name = "hiredis" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1f/e2/1654d65851f39fd94e91a77a5655d09d4b64901fdc594020d8348db697b2/hiredis-3.4.0.tar.gz", hash = "sha256:da19331354433af6a2c54c21f2d70ba084933c0d7d2c43578ec5c5b446674ad5", size = 137169, upload-time = "2026-06-03T16:23:46.226Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/48/b0c0e2826eab7543c08980dfab871b3e7c83c47d7496134b04a94df55a1a/hiredis-3.4.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:69d0326f20354ce278cbb86f5ae47cb390e22bb94a66877031038af907c42fa5", size = 138470, upload-time = "2026-06-03T16:21:49.96Z" }, + { url = "https://files.pythonhosted.org/packages/6f/75/2a08a6062228720747570a07ab42e6f5826725f09c9a95d75b7b5b938022/hiredis-3.4.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:4863b99b1bf739eaa60961798efc709f657864fbf5a142cb9b99d3e36a37208e", size = 74496, upload-time = "2026-06-03T16:21:51.14Z" }, + { url = "https://files.pythonhosted.org/packages/e0/3d/28c61f9c628dfaf1f96bb8b8592cb006eaad3747248c95f9ae7f694abb47/hiredis-3.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:98e28c10e43d076f50ce9fa9f4017303d5796c3058b1b651f507c2a7d6ef402c", size = 70083, upload-time = "2026-06-03T16:21:52.125Z" }, + { url = "https://files.pythonhosted.org/packages/31/06/21e254be776b6ecd38e7c955c2fe205f2828c091621fbe400406bd2e382e/hiredis-3.4.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6774f1fe2723001ca0cd42bf5d8b1235301226273915c581c5c1260d4d114c43", size = 304408, upload-time = "2026-06-03T16:21:53.148Z" }, + { url = "https://files.pythonhosted.org/packages/fc/c8/46d48dc674d0aacc66cdc8c400261fad3c08b352dd823e6ffa0a0536259d/hiredis-3.4.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:12eca9aea1450d1a85dc15574a985c227e52abbc2b6466f48ad2aa3b82124701", size = 336932, upload-time = "2026-06-03T16:21:54.391Z" }, + { url = "https://files.pythonhosted.org/packages/50/29/05a3cf8f605f6cdeec2c6f54d022fa51242e3cf77fe4940e89fe1446b068/hiredis-3.4.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:12ea5facb5b08fa23e4c101ec2151f3a3de8ecec412fec58dbde0a6eebca02c7", size = 347528, upload-time = "2026-06-03T16:21:55.541Z" }, + { url = "https://files.pythonhosted.org/packages/26/c9/4e9cd249afc101ac283943295fe3359bdd711a0bb8c667752eb0da80609d/hiredis-3.4.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4de6869be2b33490569dae0712366bb794b7f5e7a8b674de3e092b3e95712d6d", size = 310142, upload-time = "2026-06-03T16:21:56.534Z" }, + { url = "https://files.pythonhosted.org/packages/4a/71/d069db71ba4a5f40bb1390eebaf00e4d161c5c1f48e623880ca22a946618/hiredis-3.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4190bd07dd7879a8a7ddbb2a4f74d402721f3898276e35beb98851b85b5f539c", size = 298868, upload-time = "2026-06-03T16:21:57.559Z" }, + { url = "https://files.pythonhosted.org/packages/0f/cd/83793cc2fb161ddd5d394adc7122ec023b0e1d9289a294d0f80214d910bf/hiredis-3.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e29267ecdd08758926f1a9221af2671d90f475480c40aff409921b1f362f1bd5", size = 328564, upload-time = "2026-06-03T16:21:58.57Z" }, + { url = "https://files.pythonhosted.org/packages/3d/fa/66fed95ab85d85a4dc87acb8df69e22ff943a3bf7a26e791d5a1ff173577/hiredis-3.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:45c6c296056641b5df37cedafe7d1553f33bc247e2f81603a4d038b39261879b", size = 329725, upload-time = "2026-06-03T16:21:59.519Z" }, + { url = "https://files.pythonhosted.org/packages/a9/0d/ac2d1f1c30eb6d7ab5f099da17f76125f1bb0f9274623178508a6c736acf/hiredis-3.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f7c7596fbb2b5202e943180353958e89014e763c7f25877a92f70bbde6cd7f19", size = 309144, upload-time = "2026-06-03T16:22:00.691Z" }, + { url = "https://files.pythonhosted.org/packages/d6/41/ec1dc1c27e8fcbe2dc635d49cc751848972600a7d277569fb9ba77ee501c/hiredis-3.4.0-cp310-cp310-win32.whl", hash = "sha256:1bfb9ccfb13be63883e5f2e5ff7f6fc87bf256f8243af594257dfbed9dbc3cf0", size = 38820, upload-time = "2026-06-03T16:22:01.657Z" }, + { url = "https://files.pythonhosted.org/packages/24/9d/38b85c7fd3ec49c8b1b089288307f8f17e138439be7b079fab2221e113a8/hiredis-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:c2245c46b4ced5f689469e6dcdfc8a0895bf873840a6600f5ea759cdf1b26a8b", size = 40048, upload-time = "2026-06-03T16:22:02.457Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ce/bdc7602ddc7b60ada44be4c4246c1b4d54a0b444a2b5f17ec936c0ce0faf/hiredis-3.4.0-cp310-cp310-win_arm64.whl", hash = "sha256:0bcb630add6bc9ea136fce691ddff0c46aa91cb860df4ca789fe44127eb7e90d", size = 36849, upload-time = "2026-06-03T16:22:03.3Z" }, + { url = "https://files.pythonhosted.org/packages/92/d1/09d7323c76d097ff3f6530228d2422c19817b6052716f9a652ecd6e2f68e/hiredis-3.4.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:7f7fc1535f6e1a190089eae46dee25f0c6b72bb221d377be07092803b8208733", size = 138467, upload-time = "2026-06-03T16:22:04.09Z" }, + { url = "https://files.pythonhosted.org/packages/ed/27/c4ebeb0f7ecc8a23d4356efd3ef2b6243ed74d24584d86ff8065fa14a350/hiredis-3.4.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:ed1dba2695f6de009c67d63b39ff978cb43b8a79362f697acedffb7743e50d21", size = 74504, upload-time = "2026-06-03T16:22:04.998Z" }, + { url = "https://files.pythonhosted.org/packages/ae/d7/4f456f36f5c5224bc11a2fad964116a3cc37259d09dd840628aea5fdbf28/hiredis-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a3796094f616f72976ff51e4dc1a016e753c0f9af5393b2df96920b6bae1e19b", size = 70080, upload-time = "2026-06-03T16:22:05.76Z" }, + { url = "https://files.pythonhosted.org/packages/04/ba/a16d44b2bd71e72a10673faa94d07cc4e9de90240b65ce2511af0cce065b/hiredis-3.4.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ccc5c660e31d788ca534a20f2ccb7a80b946b960e18ed4e1db950fcac122b405", size = 304968, upload-time = "2026-06-03T16:22:06.614Z" }, + { url = "https://files.pythonhosted.org/packages/4a/3a/78ca23fe899f8da7ee2caf9c502ac1a63da15d521f33a3fc617a7adbf2e0/hiredis-3.4.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f3c67f39b112dc35f68d5b59ee111db6121f037d1a60cf3840ecffbb2ec5686b", size = 337465, upload-time = "2026-06-03T16:22:07.622Z" }, + { url = "https://files.pythonhosted.org/packages/ba/11/2df9a12f170e9d61739e7df5f06712141414b2dce2cf385fc1fb6f31a46b/hiredis-3.4.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bca175f02a2b0150ffe7f5dc8bf49c798f34d2c7024d17ace0ec97a7583560e3", size = 348293, upload-time = "2026-06-03T16:22:08.677Z" }, + { url = "https://files.pythonhosted.org/packages/88/07/716ffeb049377d92da6261c5563e554b82336ce3eafb11eb4510c5558be7/hiredis-3.4.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43004b0b48abc628dda1ac3ac4871e1326c126f8cd9f11164d61934d827d7a3b", size = 310697, upload-time = "2026-06-03T16:22:09.661Z" }, + { url = "https://files.pythonhosted.org/packages/5d/03/ef3697bdee359b4521101bdc16e8e4965a5ebd8634b605fc7cf9c01b6b82/hiredis-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8aaaab18314fd25453b5cf59c8cdca4110e419455bcb4c0737d19d4151513e75", size = 299377, upload-time = "2026-06-03T16:22:10.777Z" }, + { url = "https://files.pythonhosted.org/packages/bc/a7/2a12a2f828c2d611b74dcf2229998c4d2570fe6ed6b4903d6a4c3add84af/hiredis-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:5359caad5b57da0bce11d2880f22617ba3710f0866121a924745447848448034", size = 329008, upload-time = "2026-06-03T16:22:11.82Z" }, + { url = "https://files.pythonhosted.org/packages/66/a9/cdfda214af93eeb9f93a83a099d06f26ae5569f188209ddc8a7c977ed446/hiredis-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:44660a91e0fbc803c29b337c1a9194c8d7b4cd3a3868d28f747cbec2df165483", size = 330103, upload-time = "2026-06-03T16:22:12.935Z" }, + { url = "https://files.pythonhosted.org/packages/87/05/cdc7e2e07b56c716426db4644b917b260a4f6fdc8d16cc3bbac4b27d0a17/hiredis-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a315009b441a0105a373a9a780ebb1c6f7d9ead88ac6ea5f2a15791353c6f590", size = 309582, upload-time = "2026-06-03T16:22:14.157Z" }, + { url = "https://files.pythonhosted.org/packages/f2/36/304a0e029cb6e44add3b0d664315de25c483f6e8f8e1d413c68de969a3d0/hiredis-3.4.0-cp311-cp311-win32.whl", hash = "sha256:282c4310af72afbe18b07d416459f4febeaeb805a067a7df790136e0e550fcb2", size = 38823, upload-time = "2026-06-03T16:22:15.14Z" }, + { url = "https://files.pythonhosted.org/packages/f0/19/7ea1fdbee1c42cbac140005e66e60a1198548eea04456e17dab5c285e31b/hiredis-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:bb44efa4fa3e3ed7779ad0ade3c08ed5d75ca7a6336893e9a4f2722093b4168a", size = 40040, upload-time = "2026-06-03T16:22:15.886Z" }, + { url = "https://files.pythonhosted.org/packages/20/e4/2122980b75a3fa8980540e2265028c757564ecc4d813b40298d29dd876ea/hiredis-3.4.0-cp311-cp311-win_arm64.whl", hash = "sha256:4404c557fd49bcfe24dff41f1209e4221c76d1607df2fb2dfd39474b5b086dcb", size = 36851, upload-time = "2026-06-03T16:22:16.644Z" }, + { url = "https://files.pythonhosted.org/packages/d2/84/f74deb132d238a0d5a3eb1618bf7558c65230b279421f909a9753231c516/hiredis-3.4.0-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:9e88048a66dfffec7a3f578f2a2a0fd907c75b5bd85b3c9184f76f0149ea399f", size = 138679, upload-time = "2026-06-03T16:22:17.598Z" }, + { url = "https://files.pythonhosted.org/packages/a2/13/399fe51d399b8d4f5717aa68cb1dafcb8c244b19b1b9b0afaaa526c1be94/hiredis-3.4.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:8b3f1d03046765c0a83558bf1756811101e3947649c7ca22a71d9dc3c92929d1", size = 74657, upload-time = "2026-06-03T16:22:18.819Z" }, + { url = "https://files.pythonhosted.org/packages/a4/cf/6a0bcf454b1642997c4dd007bd89beada43f38b22781afdf475060e427ac/hiredis-3.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:24751054bb11353016d242d09a4a902ecf8f25e3b56fe396cccb6f056fdda016", size = 70115, upload-time = "2026-06-03T16:22:19.649Z" }, + { url = "https://files.pythonhosted.org/packages/98/99/62340215f80e59680c79ae5080c5422311da105870c57bbefc5d87487025/hiredis-3.4.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:258f820cdd6ee6be39ae6a8ea94a76b8856d34113de6604f63bc81327ef06240", size = 306481, upload-time = "2026-06-03T16:22:20.608Z" }, + { url = "https://files.pythonhosted.org/packages/f1/be/97f349e5bb0dcab0ef28b15523443d9bbe81f8ccbd3dadff56594dfa82fe/hiredis-3.4.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3774461209688790734b5db8934400a4456493fc1a172fb5298cc5d72201aceb", size = 339560, upload-time = "2026-06-03T16:22:21.861Z" }, + { url = "https://files.pythonhosted.org/packages/1e/3f/eb6a9632bcc13a3fbefce5de90090052fb1ae1cd3d57faf687f20149d592/hiredis-3.4.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ccdb63363c82ea9cea2d48126bc8e9241437b8b3b36413e967647a17add59643", size = 351549, upload-time = "2026-06-03T16:22:22.969Z" }, + { url = "https://files.pythonhosted.org/packages/1e/8c/440369f727dcb856f3eeda238d6e67781b180feaa831bd28997d8af10c3b/hiredis-3.4.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:452cff764acb30c106d1e33f1bdf03fa9d4a9b0a9c995d722d4d39c998b40582", size = 313066, upload-time = "2026-06-03T16:22:23.987Z" }, + { url = "https://files.pythonhosted.org/packages/ec/d1/3d76c4d5c46cd2e7b38641f7c8b325e0cab7d49d565ea573256eb3837d0c/hiredis-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1fb0a139cd52535f3e5a532816b5c36b3aea95817410fbf28ca4a676026347a5", size = 300827, upload-time = "2026-06-03T16:22:25.287Z" }, + { url = "https://files.pythonhosted.org/packages/c5/bc/d112dd9704ae47243a515fb021ec4d0b5a1b8d83a7a3eff3284c0248412d/hiredis-3.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:163d8c43e2706d23490532ea0de8736fc1493cfa52f0ee65f85b0f074f2fe017", size = 331284, upload-time = "2026-06-03T16:22:26.385Z" }, + { url = "https://files.pythonhosted.org/packages/e9/7b/8a4dc0a15e4658c81a9e79b2c167fbfbf750e0c1c7ef13e00e69d4273ced/hiredis-3.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4b8f52844cd260d7805eca55c834e3e06b4c0d5b53a4178143b92242c2517c0d", size = 332962, upload-time = "2026-06-03T16:22:27.392Z" }, + { url = "https://files.pythonhosted.org/packages/1d/52/d3d0bb234de8deb4cbd432cdc63d001a6cad1f9c05fe07d2fa652f8cf412/hiredis-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03374d663b0e025e4039757ef5fad02e3ff714f7a01e5b34c88de2a9c91359dc", size = 311698, upload-time = "2026-06-03T16:22:28.442Z" }, + { url = "https://files.pythonhosted.org/packages/04/5b/54a052eccaf901703b57d7c28509e74341fa0da08d770f485345397ea1e5/hiredis-3.4.0-cp312-cp312-win32.whl", hash = "sha256:696e0a2118e1df5ccacf8ecf8abe528cf0c4f1f1d867f64c34579bef77778cdb", size = 38921, upload-time = "2026-06-03T16:22:29.39Z" }, + { url = "https://files.pythonhosted.org/packages/a7/64/6508236eda66765fbe873d1d0a0722e38059302e96dc9915b162ff17b35a/hiredis-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:ee6b4beb79a71df67af15a8451366babc2687fcac674d5c6eacec4197e4ce8c1", size = 40090, upload-time = "2026-06-03T16:22:30.204Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1c/7333aba1b4b7cef2591b244140aec0f1aad903397bbaa31c1858722b2fe4/hiredis-3.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:14524fdc751e3960d78d848872576b5442b40baae3cac14fbab1ba7ac523891f", size = 36875, upload-time = "2026-06-03T16:22:31.087Z" }, + { url = "https://files.pythonhosted.org/packages/7b/e5/9e47dda8f1d55e77293c6cdf4169182b7f2f55b56913d1fb16a0ddf63a3d/hiredis-3.4.0-cp313-cp313-macosx_10_15_universal2.whl", hash = "sha256:4f0e3536eea76c03435d411099d165850bc3c9d873efe62843b995027135a763", size = 138688, upload-time = "2026-06-03T16:22:31.825Z" }, + { url = "https://files.pythonhosted.org/packages/1e/07/039bcf7ce8262ed66db736349c121486874826248ccd70c98c2f830ec9da/hiredis-3.4.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:82860f050aabd08c046f304eb57c105bb3d5a7370f79a4a0b74d2b771767cc13", size = 74666, upload-time = "2026-06-03T16:22:32.758Z" }, + { url = "https://files.pythonhosted.org/packages/29/6d/692c50d846a0a36578e9ef0c62c6193ce01a48f353f6961de9de88a30b37/hiredis-3.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:74bcfb26189939daba2a0eb4bad05a6a30773bb2461f3d9967b8ced224bd0de9", size = 70119, upload-time = "2026-06-03T16:22:33.692Z" }, + { url = "https://files.pythonhosted.org/packages/28/5d/c8b9ca711b4d6b7637eae744d6b45ea47f6bded61bac0232bb42ed8c583e/hiredis-3.4.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d95b602ab022f3505288ce51feaa48c072a62e57da55d6a7a38ecb8c5ad67d81", size = 306364, upload-time = "2026-06-03T16:22:34.62Z" }, + { url = "https://files.pythonhosted.org/packages/c4/7e/e940eea3c2ee1aa5947f2e6224f03a1dfd38a5813307259a25f580411820/hiredis-3.4.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:de3e2297a182253dfa4400883a9a4fb46d44946aed3157ea2da873b93e2525c4", size = 339454, upload-time = "2026-06-03T16:22:35.87Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ea/b8147da5c270a2a5b85090c97d0ff7e2fae6e7c5f7749f8c3c2decadd3ac/hiredis-3.4.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:454236d2a5bd917daf38914ce363e71aeef41240e6800f4799e04ee82689bfd2", size = 351457, upload-time = "2026-06-03T16:22:36.95Z" }, + { url = "https://files.pythonhosted.org/packages/33/b5/ff8fe4f812348f09d2943b109cb64c5301af4f601e1cf026518e93a72fff/hiredis-3.4.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:35ab3653569b9867b8d8a3b4c0684a20dc769fe45d4666bedfe9a3391a61b30b", size = 312970, upload-time = "2026-06-03T16:22:38.004Z" }, + { url = "https://files.pythonhosted.org/packages/b3/2a/c90dff527cb2521ee1687e9e30bdf1156f2f4acfd47833b44dc52fec3ec6/hiredis-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:afff0876dafad6d3bb446c907da2836954876243f6bb9d5e44915d175e424aa4", size = 300850, upload-time = "2026-06-03T16:22:39.146Z" }, + { url = "https://files.pythonhosted.org/packages/90/0b/c48e93a1e524198b10ccc26d770368547c0c29d126a992fd4b4aa533f1ac/hiredis-3.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d5c33eb2da5c9ccd281c396e1c618cfe6a91eb841e957f17d2fa520383b3111d", size = 331430, upload-time = "2026-06-03T16:22:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/95/12/ed5bdc482d5c98930ffa264dd707dfb04b83118b2f7f760760c5dfbe6782/hiredis-3.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:04e54fc3bcecf8c7cb2846947b84baf7ce1507caba641bd23590c52fefade865", size = 333021, upload-time = "2026-06-03T16:22:41.363Z" }, + { url = "https://files.pythonhosted.org/packages/e6/42/d4a2e7be82f2b2db7b67ec622806ba099d8fe09d218568f71197922cbe79/hiredis-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5f1ddfe6429f9adc0a8d705afbcd40530fddeafa919873ffbb11f59eda44dbb9", size = 311747, upload-time = "2026-06-03T16:22:42.374Z" }, + { url = "https://files.pythonhosted.org/packages/d6/33/b5ac3420bd803ca9affd68a4a2a6111812bd26bfb9d6b41a721e009d79d9/hiredis-3.4.0-cp313-cp313-win32.whl", hash = "sha256:165e6405b48f9bd66ddb4ad52ce28b0c0041a0308654d7a0cb4357a1939134dc", size = 38921, upload-time = "2026-06-03T16:22:43.513Z" }, + { url = "https://files.pythonhosted.org/packages/03/e7/76e68122b1cf680b93b951a82953fff5b5883dc08ec93f63677eb3653591/hiredis-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:306aae11a52e495aaf0a14e3efcd7b51029e632c74b847bc03159e1e1f6db591", size = 40095, upload-time = "2026-06-03T16:22:44.296Z" }, + { url = "https://files.pythonhosted.org/packages/20/05/9313dc27ed159512dc22b4ecf8a62a84d0aa5fbd500ffdad955b361cb2a8/hiredis-3.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:975a8e75a10425442037dd9c7abbaae31941c34328d9f01b1ca42d9db44ac31d", size = 36884, upload-time = "2026-06-03T16:22:45.134Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ea/cbc922aeaa5af11f1c1235d8b2b04ff8cdf6e3e95c785a500521f32d8d70/hiredis-3.4.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:d3a12ae5685e9621a988af07b5af0ad685c7d19d6a7246ac852e35060178cff4", size = 138762, upload-time = "2026-06-03T16:22:45.927Z" }, + { url = "https://files.pythonhosted.org/packages/d4/e9/e004067ffad9f707174cde04d117c985d5f22dd4d9409f0983892738cb44/hiredis-3.4.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0a70df45cf167b5af99b9fe3e2044716919e30580a869dfa766f2a6467c0c320", size = 74696, upload-time = "2026-06-03T16:22:46.924Z" }, + { url = "https://files.pythonhosted.org/packages/5a/d1/5fe5b6d05e59116d78f9d228d9cc0022efbb84d234333c5fbe6a0c6e13fe/hiredis-3.4.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0a68b0e48509e6e66f4c212e53d98f29178addf83b0701a71bf0fce792954419", size = 70163, upload-time = "2026-06-03T16:22:47.798Z" }, + { url = "https://files.pythonhosted.org/packages/db/93/c86f0a7ae2cd10b72e30476f87aafd1af22992e080feb4b5d2ec1cbdf4e4/hiredis-3.4.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a45822bc8487da8151fe67c788de74b834582b1d510c67b888fcda64bf6ba4bb", size = 306631, upload-time = "2026-06-03T16:22:48.671Z" }, + { url = "https://files.pythonhosted.org/packages/e8/10/3746b028d9c43fab1fa4126fe69c6967df89ab9819140092930322b0550c/hiredis-3.4.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0b82cab9ad7a1574ab273a78942f780c1b1496101eb342b630c46c3e918ca21b", size = 339758, upload-time = "2026-06-03T16:22:49.662Z" }, + { url = "https://files.pythonhosted.org/packages/59/f3/c6fb383854237891039a4d94d3e66dc5eec8a2993fed6020c983d63c5393/hiredis-3.4.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:db13f8039ad8229f77f0e242be14e53bd67e8f3aadeb16f3af30944287cca092", size = 351360, upload-time = "2026-06-03T16:22:50.779Z" }, + { url = "https://files.pythonhosted.org/packages/70/b7/32110aa458690722a1069c7349b8ebe374a6ba0bdf9ef8925a9f37a74978/hiredis-3.4.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54b6267918c66d8ba4a3cf519db1235a4bd56d2a0969ca5b2ae3c6b6b7d9ed79", size = 313070, upload-time = "2026-06-03T16:22:51.966Z" }, + { url = "https://files.pythonhosted.org/packages/bb/23/bccfa0fb7b1b529cff35c8725cfd99a2d18fa4123f52f52bf03e84210855/hiredis-3.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:88396e6a24b80c86f4dc180964d9cc467ba3aa3c886af6532fe077c5a5dc0c3c", size = 300927, upload-time = "2026-06-03T16:22:53.085Z" }, + { url = "https://files.pythonhosted.org/packages/3e/0f/e1e2295ee863efc7ce8c88ec10bcc4b1504352373998cb493f10e900dbe5/hiredis-3.4.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:73dd607b47863633d8070f1eb3bab1b3b097ee747783fe69c0dd0f93ec673d8b", size = 331764, upload-time = "2026-06-03T16:22:54.194Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/11b1de2ac85dfd7a8713d72a6ed7ac0f1a6e28d906bd362e0df3a27f5c86/hiredis-3.4.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e6e8d5fa63ec2a0738d188488e828818cbe4cb4d37c0c706836cf3888d82c53d", size = 333144, upload-time = "2026-06-03T16:22:55.277Z" }, + { url = "https://files.pythonhosted.org/packages/6f/10/4b104565c936d51b4b02597352ec068937c9d6a73a3c4c9609c08ae3923e/hiredis-3.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d77901d058923a09ed25063ea6fb2842c153bbe75060a46e3949e73ad12ce352", size = 311593, upload-time = "2026-06-03T16:22:56.573Z" }, + { url = "https://files.pythonhosted.org/packages/70/ae/c9eda3c116bef50fcf0dc7e44379e3577f3627caca4ffd7af04675b02d98/hiredis-3.4.0-cp314-cp314-win32.whl", hash = "sha256:05384fcfe5851b5af868bf24265c14ab86f38562679f9c6f712895b67a98163c", size = 39662, upload-time = "2026-06-03T16:22:57.683Z" }, + { url = "https://files.pythonhosted.org/packages/f2/c8/cedb336a0386a97271761ace460a362cb2433c6cdf1d1ba760ad99225734/hiredis-3.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:53233656e4fecf9f8ec654f1f4c5d445bf1c2957d7f63ffdedbba2682c9d1584", size = 40682, upload-time = "2026-06-03T16:22:58.526Z" }, + { url = "https://files.pythonhosted.org/packages/4c/ea/3a05247ce4e2afe56f59d24b73ba38e37f2b324dba8290beba56fbd9fd1f/hiredis-3.4.0-cp314-cp314-win_arm64.whl", hash = "sha256:3348ba4e101f3a96c927447ff2edcb3e0026dc6df375ba117485a43edcbb6980", size = 37541, upload-time = "2026-06-03T16:22:59.307Z" }, + { url = "https://files.pythonhosted.org/packages/35/14/caeaa1be1205ebdc1cf6760c5f6882afbdb3b82a6bdf0559d01205b1c857/hiredis-3.4.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:3159c54fe560aa30bf1ab76e65c4c23dc45ad79d7cf4aecc25ec9942f5ea4cea", size = 139787, upload-time = "2026-06-03T16:23:00.139Z" }, + { url = "https://files.pythonhosted.org/packages/49/85/8f52b485b9d835e0f8da063a635290d916a6f5ab60c18db5411ecea344d1/hiredis-3.4.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:be4a41496a0a48c3abf57ef1bbeb11980060ce9c7a1dd8b92caa028a813a9c59", size = 75136, upload-time = "2026-06-03T16:23:01.705Z" }, + { url = "https://files.pythonhosted.org/packages/9f/09/ee568562f36f481395d5cea3ab75fd9350cd77d98d55ee5f9b395f3fc358/hiredis-3.4.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a2f9a9a591b3eaade523f3e778dfcd8684965ee6e954ae25cd2fd6d8c75e881d", size = 70772, upload-time = "2026-06-03T16:23:02.765Z" }, + { url = "https://files.pythonhosted.org/packages/7f/0d/3cb03fbbe72f86541f42ee49dba95ff428c87908815152970fbf24bdcf4c/hiredis-3.4.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c2852eaa26c0a73be4a30118cd5ad6a77c095d224ccb5ac38e40cb865747d22", size = 315571, upload-time = "2026-06-03T16:23:03.826Z" }, + { url = "https://files.pythonhosted.org/packages/52/fc/c8667282e41153bc20930aeba8ba0dff989cbaa9eb7594f8bcac02558dea/hiredis-3.4.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:18ff3d9b23ebe6c8248c3debca2402ad209d60c48495e7ed76407c2fe54cb9b4", size = 348131, upload-time = "2026-06-03T16:23:05.077Z" }, + { url = "https://files.pythonhosted.org/packages/99/13/5431ace8330904b2b9d9ce5425c13b7a8fa2b443ff272a92f248c07e6400/hiredis-3.4.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:94f83352295bf3d332678689ecd4ce190a4d233a20ad2f432724efd3ce03e49a", size = 359915, upload-time = "2026-06-03T16:23:06.293Z" }, + { url = "https://files.pythonhosted.org/packages/be/57/30dab05cf2a70905e5d2807edd4afa30a4747599070faf80f18e61375e11/hiredis-3.4.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:393d5e7c8c67cdddf7109a8e925d885e788f3f43e5b1043f84390df40c59944b", size = 321426, upload-time = "2026-06-03T16:23:07.447Z" }, + { url = "https://files.pythonhosted.org/packages/33/6f/0a6e030d96d927000735b39aa8b8fef03b43fafdf4a79c80755be351a0f5/hiredis-3.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7e7ab4c1c8c4d365b02d9e82cdf25b01a065edf2ededd7b5acb043201ff80203", size = 309862, upload-time = "2026-06-03T16:23:08.672Z" }, + { url = "https://files.pythonhosted.org/packages/11/48/26b2771d2b2403124c1f97c2a6d45df0ba3fa59f0c2d4d244e90543722fb/hiredis-3.4.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:cfe23f8dcf2c0f4e03d107ff68a9ee9707f9d76abeddbe59633e5de1564a650c", size = 339568, upload-time = "2026-06-03T16:23:09.755Z" }, + { url = "https://files.pythonhosted.org/packages/07/b1/01c18f676d5dea65e894c01ffae8da2f15df1fceed1c69b16877ba57be60/hiredis-3.4.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a7e76904148c229549db7240a4f9963deb8bb328c0c0844fc9f2320aca05b530", size = 341424, upload-time = "2026-06-03T16:23:10.964Z" }, + { url = "https://files.pythonhosted.org/packages/fb/58/ab3a5672e506f282e1dd6dfb1c0c3f7e17f02398280c2a2994f8d7b478ba/hiredis-3.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:92b570225f6097430615a82543c3eb7974ca354738a6cef38053138f7d983151", size = 320386, upload-time = "2026-06-03T16:23:12.174Z" }, + { url = "https://files.pythonhosted.org/packages/15/af/3f26324cca720f56ace408883c1c7311ce71b571e82e6434515f7ba4eb59/hiredis-3.4.0-cp314-cp314t-win32.whl", hash = "sha256:decc176d86127c620b5d280b3fe5f97a788be58ca945971f3852c3bf54f4d5ad", size = 40516, upload-time = "2026-06-03T16:23:13.179Z" }, + { url = "https://files.pythonhosted.org/packages/8b/18/e011a424a9608ff152ebeb7bbae2be3163e5716e92cf75baddcb5a8fc312/hiredis-3.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:05c852c58fec65d4c9fb861372dd7391d8b2ce96c960ba8714145f8cd85cd0ec", size = 41453, upload-time = "2026-06-03T16:23:14.091Z" }, + { url = "https://files.pythonhosted.org/packages/43/5f/829287555ce7286be8d6c87c69f93aa1f38fe67c46740806416142231cf3/hiredis-3.4.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7ff29c9f5d3c91fda948c2fde58f457b3244550781d3bc0891b1b9d93c10f47f", size = 37968, upload-time = "2026-06-03T16:23:14.948Z" }, +] + [[package]] name = "hpack" version = "4.1.0" @@ -4178,6 +4270,7 @@ proxy = [ { name = "fastapi-sso" }, { name = "granian" }, { name = "gunicorn" }, + { name = "hiredis" }, { name = "inquirerpy" }, { name = "litellm-enterprise" }, { name = "litellm-proxy-extras" }, @@ -4359,6 +4452,7 @@ requires-dist = [ { name = "grpcio", marker = "extra == 'grpc'", specifier = "==1.78.0" }, { name = "grpcio", marker = "extra == 'proxy-runtime'", specifier = "==1.78.0" }, { name = "gunicorn", marker = "extra == 'proxy'", specifier = ">=23.0.0,<24.0" }, + { name = "hiredis", marker = "extra == 'proxy'", specifier = ">=3.0.0,<4.0" }, { name = "httpx", specifier = ">=0.28.0,<1.0" }, { name = "importlib-metadata", specifier = ">=8.0.0,<9.0" }, { name = "inquirerpy", marker = "extra == 'cli'", specifier = ">=0.3.4,<1.0" }, From 20399812103a622482f6a149daf82ee3f3f04d5c Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 3 Aug 2026 20:50:11 -0700 Subject: [PATCH 114/123] feat(ui): show auto-router savings on the cost-optimization dashboard (#35522) Adds the auto-router as a third optimization driver beside compression and prompt caching: a summary card, a donut segment, and a series in the savings graph across both the cumulative and per-day views. The number is signed, because a switch that thrashes the prompt cache can cost more than the cheaper rates save and an operator needs to see that. The donut plots only drivers that saved, since a negative slice has no meaning, while the card and the range total keep the sign. `usd()` sizes and signs off the magnitude so a small loss renders as -$0.01 rather than "$-0.00". The card's popover states the counterfactual and its two consequences: that a switch pays to re-warm the cache, and that a first turn the router could not identify is charged that write and therefore under-reported. --- .../_components/UsageTab.test.tsx | 112 +++++++++++++++++- .../_components/UsageTab.tsx | 74 +++++++----- .../_components/costOptimizationUtils.test.ts | 59 ++++++++- .../_components/costOptimizationUtils.ts | 30 ++++- .../src/components/UsagePage/types.ts | 1 + 5 files changed, 238 insertions(+), 38 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx index 5c26ac30477..fe3e792eeee 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx @@ -29,12 +29,14 @@ vi.mock("@/components/shared/charts", () => ({ colors, showLegend, maxBarSize, + stack, }: { data: unknown; categories: string[]; colors?: readonly string[]; showLegend?: boolean; maxBarSize?: number; + stack?: boolean; }) => (
({ data-colors={(colors ?? []).join(",")} data-show-legend={String(showLegend ?? true)} data-max-bar-size={maxBarSize === undefined ? "" : String(maxBarSize)} + data-stack={String(stack ?? false)} data-series={JSON.stringify(data)} /> ), @@ -216,8 +219,8 @@ describe("UsageTab", () => { const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); expect(slices).toEqual([ - { driver: "Compression", usd: expect.closeTo(0.14, 5) }, - { driver: "Prompt caching", usd: expect.closeTo(0.016, 5) }, + { driver: "Compression", color: "emerald", usd: expect.closeTo(0.14, 5) }, + { driver: "Prompt caching", color: "blue", usd: expect.closeTo(0.016, 5) }, ]); }); @@ -225,7 +228,110 @@ describe("UsageTab", () => { const { getByTestId } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]); const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); - expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]); + expect(slices).toEqual([{ driver: "Compression", color: "emerald", usd: expect.closeTo(0.04, 5) }]); + }); + + it("does not stack the per-day drivers, because one of them can be negative", async () => { + // Stacking sums the series into one bar. Auto-router savings go negative when a + // model switch pays for a cold cache, and that segment would be drawn below the + // axis while the rest of the bar still read as the day's total. + const { getByRole, getByTestId } = renderWith([ + day("2026-07-12", { + compression_savings_spend: 0.1, + prompt_caching_savings_spend: 0.02, + autorouter_savings_spend: -0.05, + }), + ]); + + await userEvent.click(getByRole("tab", { name: "Per day" })); + const bars = getByTestId("bar-chart"); + expect(bars.getAttribute("data-stack")).toBe("false"); + expect(readSeries(bars)[0]).toMatchObject({ "Auto-router": -0.05 }); + }); + + it("lays the savings header out with the card's own slots so nothing shifts between tabs", async () => { + // The subtitle differs in length between the tabs ("Running total saved" vs "Saved + // per day"). Hand-rolled rows made it compete with the legend and the toggle for + // width, so the header grew a line on one tab and the chart moved with it. CardHeader + // sizes the action column to its content and gives the rest to the title column. + const { getByRole, getByTestId, container } = renderWith(twoDays()); + + const header = () => { + const legend = getByTestId("chart-legend"); + const action = legend.closest('[data-slot="card-action"]') as HTMLElement; + const cardHeader = action.parentElement as HTMLElement; + const description = cardHeader.querySelector('[data-slot="card-description"]') as HTMLElement; + return { action, cardHeader, description }; + }; + + const before = header(); + expect(before.action).toBeTruthy(); + expect(before.description).toBeTruthy(); + // the toggle rides in the same action slot as the legend, so neither moves alone + expect(before.action.contains(getByRole("tablist"))).toBe(true); + // the subtitle lives outside that slot, so its length cannot reposition the controls + expect(before.action.contains(before.description)).toBe(false); + expect(before.description.textContent).toContain("Running total saved"); + + await userEvent.click(getByRole("tab", { name: "Per day" })); + + const after = header(); + expect(after.action).toBe(before.action); + expect(after.cardHeader).toBe(before.cardHeader); + expect(after.action.contains(after.description)).toBe(false); + expect(after.description.textContent).toContain("Saved per day"); + expect(container.textContent).toContain("Savings"); + }); + + it("subtracts a losing auto-router route from the total and keeps it out of the donut", () => { + // Switching models leaves the new one with a cold cache, so a route can cost more + // than the baseline would have. A negative slice is meaningless in a donut, but the + // total has to keep the loss or the page can only ever report good news. + const { getByText, getByTestId } = renderWith([ + day("2026-07-12", { + compression_savings_spend: 0.1, + prompt_caching_savings_spend: 0.02, + autorouter_savings_spend: -0.05, + }), + ]); + + expect(getByText("$0.0700")).toBeInTheDocument(); + expect(getByText("-$0.0500")).toBeInTheDocument(); + + const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); + expect(slices.map((d: { driver: string }) => d.driver)).toEqual(["Compression", "Prompt caching"]); + expect(getByTestId("donut-chart").getAttribute("data-label")).toBe("$0.1200"); + }); + + it("carries auto-router savings into the summary card, donut slice, and cumulative series", () => { + const { getByText, getByTestId } = renderWith([ + day("2026-07-12", { + compression_savings_spend: 0.04, + prompt_caching_savings_spend: 0.006, + autorouter_savings_spend: 0.02, + }), + day("2026-07-13", { + compression_savings_spend: 0.1, + prompt_caching_savings_spend: 0.01, + autorouter_savings_spend: 0.05, + }), + ]); + + // Total saved now sums three drivers, and the auto-router card carries its own total. + expect(getByText("$0.2260")).toBeInTheDocument(); + expect(getByText("$0.0700")).toBeInTheDocument(); + + // The driver donut gains a third slice priced from the range totals. + const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); + expect(slices).toEqual([ + { driver: "Compression", color: "emerald", usd: expect.closeTo(0.14, 5) }, + { driver: "Prompt caching", color: "blue", usd: expect.closeTo(0.016, 5) }, + { driver: "Auto-router", color: "amber", usd: expect.closeTo(0.07, 5) }, + ]); + + // And the cumulative line accumulates the auto-router series alongside the others. + const series = readSeries(getByTestId("area-chart")); + expect(series[2]["Auto-router"]).toBeCloseTo(0.07, 5); }); it("renders spend-by-tool bars from the tool spend endpoint", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx index ec37418e0b5..b6287602210 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx @@ -5,7 +5,7 @@ import { Info } from "lucide-react"; import { AreaChart, BarChart, CustomLegend, DonutChart, SEQUENTIAL_COLOR_RAMP } from "@/components/shared/charts"; import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Card, CardAction, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { getToolSpend, ToolSpendResponse } from "@/components/networking"; @@ -16,6 +16,8 @@ import { formatRangeLabel, localIsoDay, MAX_POINTS_WITH_DOTS, + SAVINGS_COLORS, + SAVINGS_DRIVERS, SAVINGS_SERIES, SavingsAccumulation, SavingsPoint, @@ -38,8 +40,6 @@ const EMPTY_TOOL_SPEND: ToolSpendResponse = { end_date: null, }; -const SAVINGS_COLORS = ["emerald", "blue"] as const; - const shortDate = (iso: string): string => new Date(`${iso}T00:00:00`).toLocaleDateString("en-US", { month: "short", day: "numeric" }); @@ -47,6 +47,7 @@ const isoDay = (d: Date): string => d.toISOString().slice(0, 10); const compressionOf = (m: SpendMetrics): number => m.compression_savings_spend ?? 0; const cachingOf = (m: SpendMetrics): number => m.prompt_caching_savings_spend ?? 0; +const autorouterOf = (m: SpendMetrics): number => m.autorouter_savings_spend ?? 0; const savedTokensOf = (m: SpendMetrics): number => m.compression_saved_tokens ?? 0; const SummaryCard = ({ label, value, hint, info }: { label: string; value: string; hint?: string; info?: string }) => ( @@ -105,8 +106,9 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { const compressionTotal = useMemo(() => results.reduce((sum, d) => sum + compressionOf(d.metrics), 0), [results]); const cachingTotal = useMemo(() => results.reduce((sum, d) => sum + cachingOf(d.metrics), 0), [results]); + const autorouterTotal = useMemo(() => results.reduce((sum, d) => sum + autorouterOf(d.metrics), 0), [results]); const savedTokensTotal = useMemo(() => results.reduce((sum, d) => sum + savedTokensOf(d.metrics), 0), [results]); - const totalSaved = compressionTotal + cachingTotal; + const totalSaved = compressionTotal + cachingTotal + autorouterTotal; const [accumulation, setAccumulation] = useState("cumulative"); @@ -122,6 +124,7 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { date: shortDate(d.date), Compression: compressionOf(d.metrics), "Prompt caching": cachingOf(d.metrics), + "Auto-router": autorouterOf(d.metrics), })), [results], ); @@ -143,14 +146,19 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { .filter(Boolean) .join(" \u00b7 "); + // A driver can come out negative (auto-router pays a cold-cache write on every + // model switch), and a negative slice has no meaning in a donut, so only drivers + // that actually saved are plotted; the range total keeps the signed truth. const byDriver = useMemo( () => - [ - { driver: "Compression", usd: compressionTotal }, - { driver: "Prompt caching", usd: cachingTotal }, - ].filter((d) => d.usd > 0), - [compressionTotal, cachingTotal], + SAVINGS_DRIVERS.map(({ name, color }) => ({ + driver: name, + color, + usd: { Compression: compressionTotal, "Prompt caching": cachingTotal, "Auto-router": autorouterTotal }[name], + })).filter((d) => d.usd > 0), + [compressionTotal, cachingTotal, autorouterTotal], ); + const plottedDriverTotal = useMemo(() => byDriver.reduce((sum, d) => sum + d.usd, 0), [byDriver]); const topTools = useMemo(() => topToolsBySpend(toolSpend?.by_tool ?? []), [toolSpend]); const topToolNames = useMemo(() => topTools.map((t) => t.tool_name), [topTools]); @@ -174,11 +182,11 @@ const UsageTab: React.FC = ({ accessToken, activity }) => {
-
+
= ({ accessToken, activity }) => { hint="Cache read discount" info="Tokens the provider served from cache, priced at the discount between the input and cache-read rates." /> +
+ {/* CardHeader's own slots rather than hand-rolled rows: the action column is + sized to its content and the title column takes the rest, so the subtitle + never competes with the controls for width and neither moves when it grows. + The controls wrap within their column instead of pushing past the card */} -
-
- Savings -

{savingsSubtitle}

-
-
- - setAccumulation(value as SavingsAccumulation)}> - - Cumulative - {intervalLabel} - - -
-
+ Savings + {savingsSubtitle} + + + setAccumulation(value as SavingsAccumulation)}> + + Cumulative + {intervalLabel} + + +
{accumulation === "cumulative" ? ( @@ -225,12 +239,14 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { showDots={overTime.length <= MAX_POINTS_WITH_DOTS} /> ) : ( + // Not stacked: a driver can be negative once a model switch is charged + // for its cold cache, and stacking would draw that segment below the axis + // while the remaining bar still read as the day's total @@ -247,10 +263,10 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { data={byDriver} index="driver" category="usd" - colors={["emerald", "blue"]} + colors={byDriver.map((d) => d.color)} valueFormatter={usd} showLabel - label={usd(totalSaved)} + label={usd(plottedDriverTotal)} />
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts index dc08799a3c4..14fb26c53ef 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts @@ -3,6 +3,9 @@ import { describe, expect, it } from "vitest"; import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; import type { ToolSpendDailyEntry, ToolSpendEntry } from "@/components/networking"; import { + SAVINGS_COLORS, + SAVINGS_DRIVERS, + SAVINGS_SERIES, buildDailyToolSeries, computeCacheLeakage, formatRangeLabel, @@ -10,6 +13,7 @@ import { localIsoDay, toCumulative, topToolsBySpend, + usd, withStartAnchor, } from "./costOptimizationUtils"; @@ -239,22 +243,25 @@ describe("localIsoDay", () => { }); describe("toCumulative", () => { - const point = (date: string, compression: number, caching: number) => ({ + const point = (date: string, compression: number, caching: number, autorouter: number = 0) => ({ date, Compression: compression, "Prompt caching": caching, + "Auto-router": autorouter, }); it("turns each reading into everything saved up to that point", () => { const running = toCumulative([point("Jul 1", 1, 10), point("Jul 2", 2, 20), point("Jul 3", 3, 30)]); expect(running.map((p) => p.Compression)).toEqual([1, 3, 6]); expect(running.map((p) => p["Prompt caching"])).toEqual([10, 30, 60]); + expect(running.map((p) => p["Auto-router"])).toEqual([0, 0, 0]); }); it("accumulates each driver on its own, so one flat series cannot lift the other", () => { const running = toCumulative([point("Jul 1", 0, 5), point("Jul 2", 0, 5)]); expect(running.map((p) => p.Compression)).toEqual([0, 0]); expect(running.map((p) => p["Prompt caching"])).toEqual([5, 10]); + expect(running.map((p) => p["Auto-router"])).toEqual([0, 0]); }); it("never falls, even across a quiet interval", () => { @@ -267,13 +274,19 @@ describe("toCumulative", () => { expect(running.map((p) => p.date)).toEqual(["9am", "10am"]); expect(toCumulative([])).toEqual([]); }); + + it("accumulates auto-router savings like other drivers", () => { + const running = toCumulative([point("Jul 1", 1, 1, 5), point("Jul 2", 1, 1, 10)]); + expect(running.map((p) => p["Auto-router"])).toEqual([5, 15]); + }); }); describe("withStartAnchor", () => { - const point = (date: string, compression: number, caching: number) => ({ + const point = (date: string, compression: number, caching: number, autorouter: number = 0) => ({ date, Compression: compression, "Prompt caching": caching, + "Auto-router": autorouter, }); it("lifts a single-day cumulative off a floating dot by prepending a $0 origin", () => { @@ -285,6 +298,7 @@ describe("withStartAnchor", () => { const anchored = withStartAnchor([point("Jul 16", 5, 1), point("Jul 17", 9, 4)], "Jul 16"); expect(anchored.map((p) => p.Compression)).toEqual([0, 5, 9]); expect(anchored.map((p) => p["Prompt caching"])).toEqual([0, 1, 4]); + expect(anchored.map((p) => p["Auto-router"])).toEqual([0, 0, 0]); }); it("leaves an empty series alone so the chart's own no-data state can show", () => { @@ -306,3 +320,44 @@ describe("formatRangeLabel", () => { expect(formatRangeLabel(new Date(2026, 6, 23), undefined)).toBe(""); }); }); + +describe("usd", () => { + it("keeps four decimals for sub-dollar amounts so small savings stay visible", () => { + expect(usd(0.05)).toBe("$0.0500"); + expect(usd(1.5)).toBe("$1.50"); + expect(usd(0)).toBe("$0.00"); + }); + + it("signs a loss ahead of the symbol and keeps its precision", () => { + // A driver can be negative once a model switch is charged for its cold cache. + // Sizing decimals off the raw value would render this as "$-0.00". + expect(usd(-0.05)).toBe("-$0.0500"); + expect(usd(-0.0004)).toBe("-$0.0004"); + expect(usd(-12.4)).toBe("-$12.40"); + }); +}); + +describe("savings driver colours", () => { + it("keeps a driver's colour when a driver above it is filtered out", () => { + // Charts colour by position in the data they are given, and the donut is given + // only drivers that saved something. Compression is zero on any deployment not + // running the compression guardrail, so the survivors must not slide onto the + // colours of the drivers dropped above them. + const totals = { Compression: 0, "Prompt caching": 4, "Auto-router": 7 } as const; + const plotted = SAVINGS_DRIVERS.map(({ name, color }) => ({ name, color, usd: totals[name] })).filter( + (d) => d.usd > 0, + ); + + expect(plotted.map((d) => [d.name, d.color])).toEqual([ + ["Prompt caching", "blue"], + ["Auto-router", "amber"], + ]); + }); + + it("agrees with the legend, which is built from the unfiltered list", () => { + const legend = new Map(SAVINGS_SERIES.map((name, i) => [SAVINGS_COLORS[i], name])); + for (const { name, color } of SAVINGS_DRIVERS) { + expect(legend.get(color)).toBe(name); + } + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts index 32eb6ae198d..d63266c5ee7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts @@ -3,8 +3,11 @@ import { ToolSpendDailyEntry, ToolSpendEntry } from "@/components/networking"; import { formatNumberWithCommas } from "@/utils/dataUtils"; export const usd = (value: number): string => { - const decimals = value > 0 && value < 1 ? 4 : 2; - return `$${formatNumberWithCommas(value, decimals)}`; + // Sized and signed off the magnitude: a driver can come out negative, and a small + // loss rendered at two decimals would read as "$-0.00" + const magnitude = Math.abs(value); + const decimals = magnitude > 0 && magnitude < 1 ? 4 : 2; + return `${value < 0 ? "-" : ""}$${formatNumberWithCommas(magnitude, decimals)}`; }; export const pct = (ratio: number): string => `${formatNumberWithCommas(ratio * 100, 1)}%`; @@ -161,9 +164,27 @@ export type SavingsPoint = { date: string; Compression: number; "Prompt caching": number; + "Auto-router": number; }; -export const SAVINGS_SERIES = ["Compression", "Prompt caching"] as const; +/** + * The savings drivers, each owning its own colour. + * + * One list rather than a names list beside a colours list, because the donut is + * given only the drivers that saved anything and charts assign colours by position + * in the data they receive. Two lists that line up by index therefore stop lining + * up the moment a driver is filtered out: the survivors slide down and inherit the + * colours of the drivers above them, while the legend still reports the original + * mapping. Colour travels with the driver so filtering cannot separate them. + */ +export const SAVINGS_DRIVERS = [ + { name: "Compression", color: "emerald" }, + { name: "Prompt caching", color: "blue" }, + { name: "Auto-router", color: "amber" }, +] as const; + +export const SAVINGS_SERIES = SAVINGS_DRIVERS.map((d) => d.name); +export const SAVINGS_COLORS = SAVINGS_DRIVERS.map((d) => d.color); /** * Running total of each series across the selected window. The total restarts @@ -179,6 +200,7 @@ export const toCumulative = (points: readonly SavingsPoint[]): SavingsPoint[] => date: point.date, Compression: (previous?.Compression ?? 0) + point.Compression, "Prompt caching": (previous?.["Prompt caching"] ?? 0) + point["Prompt caching"], + "Auto-router": (previous?.["Auto-router"] ?? 0) + point["Auto-router"], }, ]; }, []); @@ -193,7 +215,7 @@ export const toCumulative = (points: readonly SavingsPoint[]): SavingsPoint[] => export const withStartAnchor = (cumulative: readonly SavingsPoint[], startLabel: string): SavingsPoint[] => cumulative.length === 0 ? [...cumulative] - : [{ date: startLabel, Compression: 0, "Prompt caching": 0 }, ...cumulative]; + : [{ date: startLabel, Compression: 0, "Prompt caching": 0, "Auto-router": 0 }, ...cumulative]; /** "Jul 16 – Jul 23", collapsing to a single date when the range is one day. */ export const formatRangeLabel = (from: Date | undefined, to: Date | undefined): string => { diff --git a/ui/litellm-dashboard/src/components/UsagePage/types.ts b/ui/litellm-dashboard/src/components/UsagePage/types.ts index bf33fa37111..b10fc79be15 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/types.ts +++ b/ui/litellm-dashboard/src/components/UsagePage/types.ts @@ -11,6 +11,7 @@ export interface SpendMetrics { compression_saved_tokens?: number; compression_savings_spend?: number; prompt_caching_savings_spend?: number; + autorouter_savings_spend?: number; } export type DailyData = { From c9887a1f94bc1e7e4bdfe64d640f0509a0bc19dd Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Tue, 4 Aug 2026 06:34:52 +0200 Subject: [PATCH 115/123] perf: build log messages lazily so filtered-out log records cost nothing (#35703) --- litellm/_redis.py | 8 +- litellm/a2a_protocol/card_resolver.py | 4 +- .../a2a_protocol/exception_mapping_utils.py | 8 +- .../litellm_completion_bridge/handler.py | 14 +- .../transformation.py | 8 +- litellm/a2a_protocol/main.py | 20 +- .../providers/bedrock_agentcore/handler.py | 6 +- .../bedrock_agentcore/transformation.py | 2 +- .../providers/pydantic_ai_agents/handler.py | 4 +- .../pydantic_ai_agents/transformation.py | 10 +- .../providers/watsonx_orchestrate/handler.py | 6 +- .../watsonx_orchestrate/transformation.py | 2 +- litellm/a2a_protocol/streaming_iterator.py | 10 +- litellm/anthropic_beta_headers_manager.py | 8 +- litellm/batches/batch_utils.py | 4 +- litellm/batches/main.py | 4 +- litellm/caching/azure_blob_cache.py | 10 +- litellm/caching/caching.py | 6 +- litellm/caching/caching_handler.py | 2 +- litellm/caching/dual_cache.py | 6 +- litellm/caching/gcs_cache.py | 9 +- litellm/caching/redis_cache.py | 21 +- litellm/caching/redis_cluster_cache.py | 2 +- litellm/caching/redis_semantic_cache.py | 2 +- litellm/caching/s3_cache.py | 17 +- .../transformation.py | 42 +- litellm/cost_calculator.py | 33 +- litellm/experimental_mcp_client/client.py | 105 ++--- .../google_genai/adapters/transformation.py | 9 +- .../SlackAlerting/batching_handler.py | 4 +- .../SlackAlerting/slack_alerting.py | 10 +- .../anthropic_cache_control_hook.py | 10 +- litellm/integrations/argilla.py | 14 +- litellm/integrations/arize/_utils.py | 2 +- litellm/integrations/arize/arize_phoenix.py | 2 +- .../arize/arize_phoenix_prompt_manager.py | 2 +- .../azure_sentinel/azure_sentinel.py | 8 +- .../azure_storage/azure_storage.py | 34 +- .../bitbucket/bitbucket_prompt_manager.py | 2 +- .../integrations/braintrust_mock_client.py | 2 +- litellm/integrations/cloudzero/cloudzero.py | 18 +- litellm/integrations/custom_batch_logger.py | 2 +- litellm/integrations/custom_guardrail.py | 4 +- litellm/integrations/custom_logger.py | 20 +- litellm/integrations/custom_secret_manager.py | 2 +- litellm/integrations/datadog/datadog.py | 24 +- .../datadog/datadog_cost_management.py | 8 +- .../integrations/datadog/datadog_llm_obs.py | 38 +- .../integrations/datadog/datadog_metrics.py | 8 +- litellm/integrations/deepeval/api.py | 4 +- litellm/integrations/gcs_bucket/gcs_bucket.py | 12 +- .../gcs_bucket/gcs_bucket_mock_client.py | 4 +- litellm/integrations/gcs_pubsub/pub_sub.py | 6 +- .../generic_api/generic_api_callback.py | 37 +- .../gitlab/gitlab_prompt_manager.py | 2 +- litellm/integrations/lago.py | 8 +- litellm/integrations/langfuse/langfuse.py | 29 +- .../integrations/langfuse/langfuse_otel.py | 4 +- .../langfuse/langfuse_prompt_management.py | 4 +- litellm/integrations/langsmith.py | 20 +- litellm/integrations/literal_ai.py | 10 +- litellm/integrations/logfire_logger.py | 4 +- litellm/integrations/mlflow.py | 2 +- litellm/integrations/mock_client_factory.py | 18 +- litellm/integrations/newrelic/newrelic.py | 41 +- litellm/integrations/opentelemetry.py | 3 +- litellm/integrations/opik/opik.py | 20 +- .../opik/opik_payload_builder/extractors.py | 6 +- .../opik_payload_builder/payload_builders.py | 2 +- litellm/integrations/posthog.py | 30 +- litellm/integrations/prometheus.py | 69 ++-- litellm/integrations/prometheus_services.py | 2 +- litellm/integrations/rubrik.py | 60 +-- litellm/integrations/s3.py | 14 +- litellm/integrations/s3_v2.py | 59 +-- litellm/integrations/sqs.py | 12 +- litellm/integrations/traceloop.py | 4 +- .../vector_store_pre_call_hook.py | 20 +- litellm/integrations/weave/weave_otel.py | 4 +- .../websearch_interception/handler.py | 131 +++--- .../websearch_interception/transformation.py | 10 +- litellm/integrations/weights_biases.py | 4 +- litellm/interactions/streaming_iterator.py | 2 +- .../exception_mapping_utils.py | 12 +- litellm/litellm_core_utils/fallback_utils.py | 2 +- litellm/litellm_core_utils/litellm_logging.py | 83 ++-- .../litellm_core_utils/llm_cost_calc/utils.py | 12 +- .../llm_response_utils/get_api_base.py | 2 +- .../logging_callback_manager.py | 13 +- litellm/litellm_core_utils/logging_utils.py | 8 +- litellm/litellm_core_utils/logging_worker.py | 6 +- .../prompt_templates/factory.py | 15 +- .../litellm_core_utils/realtime_streaming.py | 12 +- .../litellm_core_utils/streaming_handler.py | 10 +- litellm/litellm_core_utils/token_counter.py | 21 +- litellm/llms/__init__.py | 14 +- .../chat/guardrail_translation/handler.py | 8 +- .../llms/anthropic/count_tokens/handler.py | 16 +- .../anthropic/count_tokens/token_counter.py | 4 +- .../adapters/streaming_iterator.py | 2 +- .../editors/clear_tool_uses.py | 4 +- .../messages/mcp_handler.py | 5 +- .../responses_adapters/streaming_iterator.py | 2 +- litellm/llms/anthropic/files/handler.py | 4 +- .../azure/chat/o_series_transformation.py | 2 +- litellm/llms/azure/common_utils.py | 13 +- litellm/llms/azure/cost_calculation.py | 5 +- .../llms/azure/image_generation/__init__.py | 2 +- .../responses/o_series_transformation.py | 2 +- .../llms/azure/responses/transformation.py | 10 +- litellm/llms/azure_ai/agents/handler.py | 24 +- .../llms/azure_ai/agents/transformation.py | 2 +- .../anthropic/count_tokens/handler.py | 16 +- .../anthropic/count_tokens/token_counter.py | 4 +- litellm/llms/azure_ai/chat/transformation.py | 2 +- litellm/llms/azure_ai/cost_calculator.py | 2 +- .../azure_ai/image_generation/__init__.py | 2 +- litellm/llms/azure_ai/ocr/common_utils.py | 4 +- .../document_intelligence/transformation.py | 12 +- litellm/llms/azure_ai/ocr/transformation.py | 12 +- .../files/azure_blob_storage_backend.py | 10 +- .../base_llm/files/storage_backend_factory.py | 2 +- .../base_managed_resource.py | 6 +- litellm/llms/bedrock/base_aws_llm.py | 32 +- .../bedrock/chat/agentcore/transformation.py | 53 +-- .../bedrock/chat/converse_transformation.py | 8 +- .../chat/invoke_agent/transformation.py | 24 +- litellm/llms/bedrock/chat/invoke_handler.py | 2 +- .../count_tokens/bedrock_token_counter.py | 4 +- litellm/llms/bedrock/count_tokens/handler.py | 20 +- litellm/llms/bedrock/files/transformation.py | 3 +- litellm/llms/bedrock/realtime/handler.py | 16 +- .../llms/bedrock/realtime/transformation.py | 10 +- .../bedrock_mantle/chat/transformation.py | 2 +- .../black_forest_labs/image_edit/handler.py | 8 +- .../image_generation/handler.py | 8 +- .../llms/custom_httpx/aiohttp_transport.py | 10 +- litellm/llms/custom_httpx/http_handler.py | 17 +- litellm/llms/custom_httpx/llm_http_handler.py | 46 ++- litellm/llms/databricks/common_utils.py | 2 +- litellm/llms/databricks/streaming_utils.py | 8 +- litellm/llms/gemini/files/transformation.py | 10 +- .../llms/gemini/realtime/transformation.py | 10 +- litellm/llms/gigachat/authenticator.py | 4 +- litellm/llms/gigachat/chat/transformation.py | 2 +- litellm/llms/gigachat/file_handler.py | 16 +- litellm/llms/github_copilot/authenticator.py | 34 +- .../embedding/transformation.py | 2 +- .../responses/transformation.py | 9 +- litellm/llms/groq/chat/transformation.py | 2 +- .../llms/huggingface/chat/transformation.py | 2 +- litellm/llms/langflow/chat/transformation.py | 6 +- litellm/llms/langgraph/chat/sse_iterator.py | 6 +- litellm/llms/langgraph/chat/transformation.py | 12 +- .../litellm_proxy/skills/code_execution.py | 12 +- litellm/llms/litellm_proxy/skills/handler.py | 10 +- .../litellm_proxy/skills/prompt_injection.py | 8 +- .../litellm_proxy/skills/sandbox_executor.py | 18 +- litellm/llms/manus/files/transformation.py | 6 +- .../llms/manus/responses/transformation.py | 10 +- litellm/llms/mistral/ocr/transformation.py | 6 +- litellm/llms/ollama/common_utils.py | 4 +- .../llms/ollama/completion/transformation.py | 2 +- .../openai/chat/o_series_transformation.py | 2 +- litellm/llms/openai/cost_calculation.py | 20 +- .../image_generation/cost_calculator.py | 2 +- litellm/llms/openai/openai.py | 2 +- .../openai/responses/count_tokens/handler.py | 14 +- .../responses/count_tokens/token_counter.py | 4 +- .../llms/openai/responses/transformation.py | 14 +- litellm/llms/openai_like/dynamic_config.py | 5 +- litellm/llms/openai_like/json_loader.py | 2 +- .../llms/perplexity/chat/transformation.py | 6 +- .../image_generation/transformation.py | 6 +- .../runwayml/text_to_speech/transformation.py | 6 +- litellm/llms/sagemaker/common_utils.py | 10 +- litellm/llms/sap/credentials.py | 8 +- litellm/llms/together_ai/chat.py | 2 +- .../vertex_ai/agent_engine/transformation.py | 14 +- litellm/llms/vertex_ai/common_utils.py | 3 +- litellm/llms/vertex_ai/cost_calculator.py | 6 +- .../vertex_and_google_ai_studio_gemini.py | 35 +- .../vertex_ai/ocr/deepseek_transformation.py | 4 +- litellm/llms/vertex_ai/ocr/transformation.py | 10 +- .../llms/vertex_ai/rag_engine/ingestion.py | 10 +- litellm/llms/vertex_ai/vertex_llm_base.py | 25 +- litellm/llms/watsonx/chat/transformation.py | 2 +- litellm/llms/xai/chat/transformation.py | 6 +- litellm/main.py | 18 +- litellm/ocr/main.py | 16 +- .../mcp_server/auth/user_api_key_auth_mcp.py | 100 ++--- litellm/proxy/_experimental/mcp_server/db.py | 2 +- .../mcp_server/discoverable_endpoints.py | 7 +- .../mcp_server/mcp_server_manager.py | 130 +++--- .../mcp_server/openapi_to_mcp_generator.py | 11 +- .../mcp_server/rest_endpoints.py | 26 +- .../mcp_server/semantic_tool_filter.py | 17 +- .../proxy/_experimental/mcp_server/server.py | 137 ++++--- .../_experimental/mcp_server/sse_transport.py | 22 +- .../_experimental/mcp_server/tool_registry.py | 4 +- .../_experimental/mcp_server/toolset_db.py | 2 +- .../mcp_server/ui_session_utils.py | 2 +- .../proxy/agent_endpoints/a2a_endpoints.py | 14 +- litellm/proxy/agent_endpoints/a2a_routing.py | 6 +- .../auth/agent_permission_handler.py | 14 +- litellm/proxy/agent_endpoints/endpoints.py | 28 +- .../agent_endpoints/model_list_helpers.py | 4 +- .../claude_code_marketplace.py | 26 +- .../proxy/anthropic_endpoints/endpoints.py | 4 +- litellm/proxy/auth/auth_checks.py | 47 ++- litellm/proxy/auth/auth_exception_handler.py | 4 +- litellm/proxy/auth/auth_utils.py | 23 +- litellm/proxy/auth/handle_jwt.py | 36 +- litellm/proxy/auth/litellm_license.py | 23 +- litellm/proxy/auth/model_checks.py | 8 +- litellm/proxy/auth/oauth2_proxy_hook.py | 2 +- litellm/proxy/auth/resolvers/store.py | 4 +- litellm/proxy/auth/route_checks.py | 5 +- litellm/proxy/auth/user_api_key_auth.py | 31 +- litellm/proxy/batches_endpoints/endpoints.py | 29 +- litellm/proxy/caching_routes.py | 4 +- litellm/proxy/client/cli/interface.py | 2 +- litellm/proxy/common_request_processing.py | 20 +- litellm/proxy/common_utils/callback_utils.py | 6 +- .../proxy/common_utils/custom_openapi_spec.py | 14 +- litellm/proxy/common_utils/debug_utils.py | 25 +- .../common_utils/encrypt_decrypt_utils.py | 4 +- .../expired_ui_session_key_cleanup_manager.py | 2 +- litellm/proxy/common_utils/get_routes.py | 2 +- .../proxy/common_utils/http_parsing_utils.py | 38 +- .../common_utils/key_rotation_manager.py | 8 +- .../proxy/common_utils/load_config_utils.py | 24 +- .../common_utils/openapi_schema_compat.py | 4 +- .../proxy/common_utils/performance_utils.py | 20 +- litellm/proxy/custom_prompt_management.py | 5 +- litellm/proxy/db/check_migration.py | 3 +- litellm/proxy/db/create_views.py | 2 +- litellm/proxy/db/db_spend_update_writer.py | 47 ++- .../db_transaction_queue/pod_lock_manager.py | 4 +- .../db_transaction_queue/spend_log_cleanup.py | 33 +- litellm/proxy/db/dynamo_db.py | 4 +- litellm/proxy/db/prisma_client.py | 23 +- .../example_config_yaml/custom_guardrail.py | 4 +- .../proxy/fine_tuning_endpoints/endpoints.py | 15 +- .../proxy/guardrails/guardrail_endpoints.py | 38 +- .../guardrails/guardrail_hooks/aim/aim.py | 4 +- .../guardrail_hooks/azure/prompt_shield.py | 4 +- .../guardrail_hooks/azure/text_moderation.py | 4 +- .../guardrail_hooks/bedrock_guardrails.py | 2 +- .../cato_networks/cato_networks.py | 4 +- .../crowdstrike_aidr/crowdstrike_aidr.py | 10 +- .../custom_code/custom_code_guardrail.py | 19 +- .../guardrail_hooks/custom_code/primitives.py | 20 +- .../hiddenlayer/hiddenlayer.py | 8 +- .../guardrails/guardrail_hooks/lasso/lasso.py | 27 +- .../litellm_content_filter/content_filter.py | 93 +++-- .../litellm_content_filter/patterns.py | 2 +- .../llm_as_a_judge/__init__.py | 6 +- .../mcp_end_user_permission.py | 11 +- .../guardrails/guardrail_hooks/noma/noma.py | 28 +- .../guardrails/guardrail_hooks/onyx/onyx.py | 10 +- .../guardrail_hooks/openai/moderations.py | 2 +- .../guardrail_hooks/pangea/pangea.py | 15 +- .../panw_prisma_airs/panw_prisma_airs.py | 82 ++-- .../guardrail_hooks/pillar/pillar.py | 34 +- .../guardrails/guardrail_hooks/presidio.py | 10 +- .../prompt_security/prompt_security.py | 8 +- .../guardrail_hooks/qualifire/qualifire.py | 8 +- .../semantic_guard/route_loader.py | 2 +- .../semantic_guard/semantic_guard.py | 9 +- .../guardrail_hooks/tool_permission.py | 16 +- .../zscaler_ai_guard/zscaler_ai_guard.py | 39 +- .../proxy/guardrails/guardrail_registry.py | 32 +- litellm/proxy/guardrails/init_guardrails.py | 6 +- .../health_endpoints/_health_endpoints.py | 20 +- litellm/proxy/hooks/azure_content_safety.py | 2 +- litellm/proxy/hooks/batch_rate_limiter.py | 16 +- litellm/proxy/hooks/batch_redis_get.py | 2 +- litellm/proxy/hooks/cache_control_check.py | 2 +- litellm/proxy/hooks/dynamic_rate_limiter.py | 10 +- .../proxy/hooks/dynamic_rate_limiter_v3.py | 24 +- .../proxy/hooks/key_management_event_hooks.py | 15 +- litellm/proxy/hooks/litellm_skills/main.py | 52 +-- litellm/proxy/hooks/max_budget_limiter.py | 2 +- .../proxy/hooks/mcp_semantic_filter/hook.py | 60 +-- .../proxy/hooks/model_max_budget_limiter.py | 4 +- .../proxy/hooks/parallel_request_limiter.py | 4 +- .../hooks/parallel_request_limiter_v3.py | 76 ++-- .../proxy/hooks/prompt_injection_detection.py | 2 +- .../proxy/hooks/proxy_track_cost_callback.py | 12 +- litellm/proxy/hooks/responses_id_security.py | 9 +- .../hooks/user_management_event_hooks.py | 2 +- litellm/proxy/image_endpoints/endpoints.py | 2 +- litellm/proxy/litellm_pre_call_utils.py | 54 +-- .../cache_settings_endpoints.py | 9 +- .../common_daily_activity.py | 4 +- .../management_endpoints/common_utils.py | 4 +- .../cost_tracking_settings.py | 20 +- .../customer_endpoints.py | 22 +- .../fallback_management_endpoints.py | 10 +- .../internal_user_endpoints.py | 36 +- .../key_management_endpoints.py | 56 +-- .../management_v1/budgets.py | 2 +- .../management_v1/spend_logs.py | 4 +- .../mcp_management_endpoints.py | 34 +- ...model_access_group_management_endpoints.py | 28 +- .../model_management_endpoints.py | 24 +- .../organization_endpoints.py | 8 +- .../policy_endpoints/endpoints.py | 2 +- .../router_settings_endpoints.py | 4 +- .../management_endpoints/scim/scim_v2.py | 38 +- .../sso/custom_microsoft_sso.py | 8 +- .../management_endpoints/sso/saml_sso.py | 4 +- .../tag_management_endpoints.py | 8 +- .../team_callback_endpoints.py | 6 +- .../management_endpoints/team_endpoints.py | 20 +- litellm/proxy/management_endpoints/ui_sso.py | 171 ++++---- .../proxy/management_helpers/audit_logs.py | 2 +- .../object_permission_utils.py | 6 +- litellm/proxy/ocr_endpoints/endpoints.py | 7 +- .../openai_files_endpoints/common_utils.py | 22 +- .../openai_files_endpoints/files_endpoints.py | 14 +- .../storage_backend_service.py | 8 +- .../llm_passthrough_endpoints.py | 16 +- .../anthropic_passthrough_logging_handler.py | 14 +- .../assembly_passthrough_logging_handler.py | 4 +- .../openai_passthrough_logging_handler.py | 12 +- ...tex_ai_live_passthrough_logging_handler.py | 12 +- .../vertex_passthrough_logging_handler.py | 14 +- .../pass_through_endpoints.py | 79 ++-- .../passthrough_endpoint_router.py | 8 +- .../streaming_handler.py | 6 +- .../policy_engine/attachment_registry.py | 34 +- .../policy_engine/condition_evaluator.py | 4 +- litellm/proxy/policy_engine/init_policies.py | 14 +- .../proxy/policy_engine/pipeline_executor.py | 11 +- .../proxy/policy_engine/policy_endpoints.py | 32 +- .../proxy/policy_engine/policy_registry.py | 46 ++- .../policy_engine/policy_resolve_endpoints.py | 4 +- .../proxy/policy_engine/policy_resolver.py | 16 +- .../proxy/policy_engine/policy_validator.py | 8 +- litellm/proxy/prisma_migration.py | 6 +- litellm/proxy/prometheus_cleanup.py | 8 +- litellm/proxy/prompts/init_prompts.py | 2 +- litellm/proxy/prompts/prompt_endpoints.py | 10 +- litellm/proxy/prompts/prompt_registry.py | 12 +- litellm/proxy/proxy_cli.py | 4 +- litellm/proxy/proxy_server.py | 386 ++++++++++-------- litellm/proxy/rag_endpoints/endpoints.py | 29 +- litellm/proxy/rerank_endpoints/endpoints.py | 2 +- .../proxy/response_api_endpoints/endpoints.py | 13 +- .../response_polling/background_streaming.py | 17 +- .../proxy/response_polling/polling_handler.py | 12 +- litellm/proxy/route_llm_request.py | 4 +- litellm/proxy/search_endpoints/endpoints.py | 11 +- .../search_tool_management.py | 30 +- .../search_endpoints/search_tool_registry.py | 12 +- .../spend_tracking/cloudzero_endpoints.py | 16 +- .../spend_management_endpoints.py | 12 +- .../proxy/spend_tracking/vantage_endpoints.py | 16 +- litellm/proxy/types_utils/utils.py | 13 +- .../proxy_setting_endpoints.py | 7 +- litellm/proxy/utils.py | 69 ++-- .../management_endpoints.py | 35 +- .../vector_store_files_endpoints/endpoints.py | 24 +- litellm/rag/ingestion/base_ingestion.py | 2 +- litellm/rag/ingestion/bedrock_ingestion.py | 58 +-- .../rag/ingestion/file_parsers/pdf_parser.py | 6 +- litellm/rag/ingestion/gemini_ingestion.py | 14 +- litellm/rag/ingestion/s3_vectors_ingestion.py | 62 +-- litellm/rag/ingestion/vertex_ai_ingestion.py | 26 +- litellm/repositories/config_repository.py | 2 +- litellm/rerank_api/main.py | 4 +- .../responses/mcp/chat_completions_handler.py | 4 +- .../mcp/litellm_proxy_mcp_handler.py | 18 +- .../responses/mcp/mcp_streaming_iterator.py | 22 +- litellm/responses/utils.py | 4 +- litellm/router.py | 236 ++++++----- .../auto_router/auto_router.py | 2 +- .../router_strategy/base_routing_strategy.py | 6 +- litellm/router_strategy/budget_limiter.py | 23 +- .../complexity_router/complexity_router.py | 32 +- litellm/router_strategy/lar1_routing.py | 8 +- litellm/router_strategy/lowest_cost.py | 10 +- litellm/router_strategy/lowest_latency.py | 6 +- litellm/router_strategy/lowest_tpm_rpm.py | 12 +- litellm/router_strategy/lowest_tpm_rpm_v2.py | 14 +- .../quality_router/quality_router.py | 16 +- litellm/router_strategy/simple_shuffle.py | 11 +- litellm/router_utils/batch_utils.py | 4 +- litellm/router_utils/cooldown_cache.py | 4 +- litellm/router_utils/cooldown_callbacks.py | 3 +- litellm/router_utils/cooldown_handlers.py | 8 +- .../router_utils/fallback_event_handlers.py | 6 +- litellm/router_utils/handle_error.py | 4 +- .../router_utils/pattern_match_deployments.py | 2 +- .../io_token_rate_limit_check.py | 32 +- .../pre_call_checks/model_rate_limit_check.py | 16 +- litellm/router_utils/search_api_router.py | 15 +- litellm/sandbox/main.py | 2 +- litellm/search/main.py | 4 +- .../cyberark_secret_manager.py | 22 +- .../get_azure_ad_token_provider.py | 2 +- .../hashicorp_secret_manager.py | 32 +- litellm/secret_managers/main.py | 5 +- .../guardrail_hooks/zscaler_ai_guard.py | 2 +- litellm/types/videos/utils.py | 4 +- litellm/utils.py | 103 +++-- .../vector_stores/vector_store_registry.py | 9 +- .../test_mcp_client.py | 8 +- .../integrations/newrelic/test_newrelic.py | 12 +- .../test_anthropic_cache_control_hook.py | 10 +- .../llms/azure/test_azure_common_utils.py | 2 +- .../mcp_server/test_mcp_server.py | 25 +- .../mcp_server/test_rest_endpoints.py | 14 +- .../proxy/auth/test_auth_checks.py | 8 +- .../proxy/auth/test_model_checks_fallbacks.py | 10 +- .../test_team_endpoints.py | 3 +- tests/test_litellm/test_logging.py | 65 +++ 419 files changed, 3919 insertions(+), 3140 deletions(-) diff --git a/litellm/_redis.py b/litellm/_redis.py index 693f9582705..e05e9d4eb20 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -651,7 +651,9 @@ def get_redis_async_client( if arg in args: url_kwargs[arg] = redis_kwargs[arg] else: - verbose_logger.debug(f"REDIS: ignoring argument: {arg}. Not an allowed async_redis.Redis.from_url arg.") + verbose_logger.debug( + "REDIS: ignoring argument: %s. Not an allowed async_redis.Redis.from_url arg.", arg + ) return async_redis.Redis.from_url(**url_kwargs) # Check for Redis Sentinel @@ -805,6 +807,6 @@ def _pretty_print_redis_config(redis_kwargs: dict) -> None: # Fallback to simple logging if rich is not available masker = SensitiveDataMasker() masked_redis_kwargs = masker.mask_dict(redis_kwargs) - verbose_logger.info(f"Redis configuration: {masked_redis_kwargs}") + verbose_logger.info("Redis configuration: %s", masked_redis_kwargs) except Exception as e: - verbose_logger.error(f"Error pretty printing Redis configuration: {e}") + verbose_logger.error("Error pretty printing Redis configuration: %s", e) diff --git a/litellm/a2a_protocol/card_resolver.py b/litellm/a2a_protocol/card_resolver.py index e4cce56d0e4..81a7813d14c 100644 --- a/litellm/a2a_protocol/card_resolver.py +++ b/litellm/a2a_protocol/card_resolver.py @@ -148,13 +148,13 @@ class LiteLLMA2ACardResolver(_A2ACardResolver): # type: ignore[misc] last_error = None for path in paths: try: - verbose_logger.debug(f"Attempting to fetch agent card from {self.base_url}{path}") + verbose_logger.debug("Attempting to fetch agent card from %s%s", self.base_url, path) return await super().get_agent_card( relative_card_path=path, http_kwargs=http_kwargs, ) except Exception as e: - verbose_logger.debug(f"Failed to fetch agent card from {self.base_url}{path}: {e}") + verbose_logger.debug("Failed to fetch agent card from %s%s: %s", self.base_url, path, e) last_error = e continue diff --git a/litellm/a2a_protocol/exception_mapping_utils.py b/litellm/a2a_protocol/exception_mapping_utils.py index 4d24dd4f1d8..16979667fe5 100644 --- a/litellm/a2a_protocol/exception_mapping_utils.py +++ b/litellm/a2a_protocol/exception_mapping_utils.py @@ -192,9 +192,11 @@ async def handle_a2a_localhost_retry( request_type = "streaming " if is_streaming else "" verbose_logger.warning( - f"A2A {request_type}request to '{error.localhost_url}' failed: {error.original_error}. " - f"Agent card contains localhost/internal URL. " - f"Retrying with base_url '{error.base_url}'." + "A2A %srequest to '%s' failed: %s. Agent card contains localhost/internal URL. Retrying with base_url '%s'.", + request_type, + error.localhost_url, + error.original_error, + error.base_url, ) # Fix the agent card URL diff --git a/litellm/a2a_protocol/litellm_completion_bridge/handler.py b/litellm/a2a_protocol/litellm_completion_bridge/handler.py index 1d46e5c700f..21366602d1a 100644 --- a/litellm/a2a_protocol/litellm_completion_bridge/handler.py +++ b/litellm/a2a_protocol/litellm_completion_bridge/handler.py @@ -76,7 +76,7 @@ class A2ACompletionBridgeHandler: ) if a2a_provider_config is not None: - verbose_logger.info(f"A2A: Using provider config for {custom_llm_provider}") + verbose_logger.info("A2A: Using provider config for %s", custom_llm_provider) return await a2a_provider_config.handle_non_streaming( request_id=request_id, @@ -103,7 +103,7 @@ class A2ACompletionBridgeHandler: else: full_model = model - verbose_logger.info(f"A2A completion bridge: model={full_model}, api_base={api_base}") + verbose_logger.info("A2A completion bridge: model=%s, api_base=%s", full_model, api_base) # Build completion params dict completion_params: dict[str, Any] = { @@ -143,7 +143,7 @@ class A2ACompletionBridgeHandler: request_id=request_id, ) - verbose_logger.info(f"A2A completion bridge completed: request_id={request_id}") + verbose_logger.info("A2A completion bridge completed: request_id=%s", request_id) return a2a_response @@ -185,7 +185,7 @@ class A2ACompletionBridgeHandler: ) if a2a_provider_config is not None: - verbose_logger.info(f"A2A: Using provider config for {custom_llm_provider} (streaming)") + verbose_logger.info("A2A: Using provider config for %s (streaming)", custom_llm_provider) async for chunk in a2a_provider_config.handle_streaming( request_id=request_id, @@ -221,7 +221,7 @@ class A2ACompletionBridgeHandler: else: full_model = model - verbose_logger.info(f"A2A completion bridge streaming: model={full_model}, api_base={api_base}") + verbose_logger.info("A2A completion bridge streaming: model=%s, api_base=%s", full_model, api_base) # Build completion params dict completion_params: dict[str, Any] = { @@ -300,7 +300,9 @@ class A2ACompletionBridgeHandler: ) yield completed_event - verbose_logger.info(f"A2A completion bridge streaming completed: request_id={request_id}, chunks={chunk_count}") + verbose_logger.info( + "A2A completion bridge streaming completed: request_id=%s, chunks=%s", request_id, chunk_count + ) # Convenience functions that delegate to the class methods diff --git a/litellm/a2a_protocol/litellm_completion_bridge/transformation.py b/litellm/a2a_protocol/litellm_completion_bridge/transformation.py index a63221b4a77..e216abb6c6d 100644 --- a/litellm/a2a_protocol/litellm_completion_bridge/transformation.py +++ b/litellm/a2a_protocol/litellm_completion_bridge/transformation.py @@ -109,7 +109,7 @@ class A2ACompletionBridgeTransformation: extra_body = {**extra_body, "metadata": merged_metadata} completion_params["extra_body"] = extra_body - verbose_logger.debug(f"A2A -> completion forward metadata keys={list(forward_metadata.keys())}") + verbose_logger.debug("A2A -> completion forward metadata keys=%s", list(forward_metadata.keys())) @staticmethod def a2a_message_to_openai_messages( @@ -145,7 +145,9 @@ class A2ACompletionBridgeTransformation: # once at run level via extra_body.metadata (LangGraph POST /runs/wait shape). openai_message: dict[str, Any] = {"role": openai_role, "content": content} - verbose_logger.debug(f"A2A -> OpenAI transform: role={role} -> {openai_role}, content_length={len(content)}") + verbose_logger.debug( + "A2A -> OpenAI transform: role=%s -> %s, content_length=%s", role, openai_role, len(content) + ) return [openai_message] @@ -186,7 +188,7 @@ class A2ACompletionBridgeTransformation: "result": a2a_message, } - verbose_logger.debug(f"OpenAI -> A2A transform: content_length={len(content)}") + verbose_logger.debug("OpenAI -> A2A transform: content_length=%s", len(content)) return a2a_response diff --git a/litellm/a2a_protocol/main.py b/litellm/a2a_protocol/main.py index 52d35a988c6..ec2d3ccf1f7 100644 --- a/litellm/a2a_protocol/main.py +++ b/litellm/a2a_protocol/main.py @@ -204,7 +204,7 @@ async def _send_message_via_completion_bridge( Requires request; api_base is optional for providers that derive endpoint from model. """ - verbose_logger.info(f"A2A using completion bridge: provider={custom_llm_provider}, api_base={api_base}") + verbose_logger.info("A2A using completion bridge: provider=%s, api_base=%s", custom_llm_provider, api_base) from litellm.a2a_protocol.litellm_completion_bridge.handler import ( A2ACompletionBridgeHandler, @@ -463,7 +463,7 @@ async def asend_message( agent_name = _get_a2a_model_info(a2a_client, kwargs) - verbose_logger.info(f"A2A send_message request_id={request.id}, agent={agent_name}") + verbose_logger.info("A2A send_message request_id=%s, agent=%s", request.id, agent_name) # Get agent card URL for localhost retry logic agent_card = _get_a2a_client_agent_card(a2a_client) @@ -478,7 +478,7 @@ async def asend_message( agent_name=agent_name, ) - verbose_logger.info(f"A2A send_message completed, request_id={request.id}") + verbose_logger.info("A2A send_message completed, request_id=%s", request.id) # Wrap in LiteLLM response type for _hidden_params support response = LiteLLMSendMessageResponse.from_a2a_response(a2a_response, request_id=str(request.id)) @@ -640,7 +640,7 @@ async def asend_message_streaming( raise ValueError("request is required for completion bridge") # api_base is optional for providers that derive endpoint from model (e.g., bedrock/agentcore) - verbose_logger.info(f"A2A streaming using completion bridge: provider={custom_llm_provider}") + verbose_logger.info("A2A streaming using completion bridge: provider=%s", custom_llm_provider) from litellm.a2a_protocol.litellm_completion_bridge.handler import ( A2ACompletionBridgeHandler, @@ -697,7 +697,7 @@ async def asend_message_streaming( proxy_server_request=proxy_server_request, ) - verbose_logger.info(f"A2A send_message_streaming request_id={request.id}, agent={agent_name}") + verbose_logger.info("A2A send_message_streaming request_id=%s, agent=%s", request.id, agent_name) agent_card = _get_a2a_client_agent_card(a2a_client) card_url = get_agent_card_url(agent_card) if agent_card else None @@ -759,7 +759,7 @@ async def create_a2a_client( "The 'a2a' package is required for A2A agent invocation. Install it with: pip install a2a-sdk" ) - verbose_logger.info(f"Creating A2A client for {base_url}") + verbose_logger.info("Creating A2A client for %s", base_url) # Use get_async_httpx_client with per-agent params so that different agents # (with different extra_headers) get separate cached clients. The params @@ -781,7 +781,7 @@ async def create_a2a_client( httpx_client = _async_handler.client if extra_headers: httpx_client.headers.update(extra_headers) - verbose_proxy_logger.debug(f"A2A client created with extra_headers={list(extra_headers.keys())}") + verbose_proxy_logger.debug("A2A client created with extra_headers=%s", list(extra_headers.keys())) a2a_client = await create_client( # pyright: ignore[reportOptionalCall] base_url, @@ -798,7 +798,7 @@ async def create_a2a_client( if agent_card is not None: a2a_client._litellm_agent_card = agent_card # type: ignore[attr-defined] - verbose_logger.info(f"A2A client created for {base_url}") + verbose_logger.info("A2A client created for %s", base_url) return a2a_client @@ -824,7 +824,7 @@ async def aget_agent_card( "The 'a2a' package is required for A2A agent invocation. Install it with: pip install a2a-sdk" ) - verbose_logger.info(f"Fetching agent card from {base_url}") + verbose_logger.info("Fetching agent card from %s", base_url) # Use LiteLLM's cached httpx client http_handler = get_async_httpx_client( @@ -839,5 +839,5 @@ async def aget_agent_card( ) agent_card = await resolver.get_agent_card() - verbose_logger.info(f"Fetched agent card: {agent_card.name if hasattr(agent_card, 'name') else 'unknown'}") + verbose_logger.info("Fetched agent card: %s", agent_card.name if hasattr(agent_card, "name") else "unknown") return agent_card diff --git a/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py b/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py index 56f5f806e7b..d19137ef4a9 100644 --- a/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py +++ b/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py @@ -53,7 +53,7 @@ class BedrockAgentCoreA2AHandler: agent_extra_headers=agent_extra_headers, ) - verbose_logger.info(f"BedrockAgentCore A2A: Sending non-streaming request to {url}") + verbose_logger.info("BedrockAgentCore A2A: Sending non-streaming request to %s", url) client = get_async_httpx_client( llm_provider=cast(Any, httpxSpecialProvider.A2AProvider), @@ -67,7 +67,7 @@ class BedrockAgentCoreA2AHandler: response_data = response.json() if "error" in response_data: - verbose_logger.warning(f"BedrockAgentCore A2A: Agent returned error: {response_data['error']}") + verbose_logger.warning("BedrockAgentCore A2A: Agent returned error: %s", response_data["error"]) return response_data @@ -100,7 +100,7 @@ class BedrockAgentCoreA2AHandler: agent_extra_headers=agent_extra_headers, ) - verbose_logger.info(f"BedrockAgentCore A2A: Sending streaming request to {url}") + verbose_logger.info("BedrockAgentCore A2A: Sending streaming request to %s", url) client = get_async_httpx_client( llm_provider=cast(Any, httpxSpecialProvider.A2AProvider), diff --git a/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py b/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py index f9343d2d3b4..0c1e01e7f9f 100644 --- a/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py +++ b/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py @@ -195,5 +195,5 @@ class BedrockAgentCoreA2ATransformation: event = json.loads(data_str) yield event except json.JSONDecodeError: - verbose_logger.debug(f"BedrockAgentCore A2A: Skipping non-JSON SSE line: {data_str[:100]}") + verbose_logger.debug("BedrockAgentCore A2A: Skipping non-JSON SSE line: %s", data_str[:100]) continue diff --git a/litellm/a2a_protocol/providers/pydantic_ai_agents/handler.py b/litellm/a2a_protocol/providers/pydantic_ai_agents/handler.py index 86cb2d47ad3..da1fc1eb657 100644 --- a/litellm/a2a_protocol/providers/pydantic_ai_agents/handler.py +++ b/litellm/a2a_protocol/providers/pydantic_ai_agents/handler.py @@ -47,7 +47,7 @@ class PydanticAIHandler: """ if api_base is None: raise ValueError("api_base is required for Pydantic AI agents") - verbose_logger.info(f"Pydantic AI: Routing to Pydantic AI agent at {api_base}") + verbose_logger.info("Pydantic AI: Routing to Pydantic AI agent at %s", api_base) # Send request directly to Pydantic AI agent response_data = await PydanticAITransformation.send_non_streaming_request( @@ -92,7 +92,7 @@ class PydanticAIHandler: """ if api_base is None: raise ValueError("api_base is required for Pydantic AI agents") - verbose_logger.info(f"Pydantic AI: Faking streaming for Pydantic AI agent at {api_base}") + verbose_logger.info("Pydantic AI: Faking streaming for Pydantic AI agent at %s", api_base) # Get raw task response first (not the transformed A2A format) raw_response = await PydanticAITransformation.send_and_get_raw_response( diff --git a/litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py b/litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py index 37127f2fcab..d8b22282d3a 100644 --- a/litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py +++ b/litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py @@ -118,7 +118,7 @@ class PydanticAITransformation: status = result.get("status", {}) state = status.get("state", "") - verbose_logger.debug(f"Pydantic AI: Poll attempt {attempt + 1}/{max_attempts}, state={state}") + verbose_logger.debug("Pydantic AI: Poll attempt %s/%s, state=%s", attempt + 1, max_attempts, state) if state == "completed": return poll_data @@ -173,7 +173,7 @@ class PydanticAITransformation: # FastA2A uses root endpoint (/) not /messages endpoint = api_base.rstrip("/") - verbose_logger.info(f"Pydantic AI: Sending non-streaming request to {endpoint}") + verbose_logger.info("Pydantic AI: Sending non-streaming request to %s", endpoint) # Send request to Pydantic AI agent using shared async HTTP client client = get_async_httpx_client( @@ -200,7 +200,7 @@ class PydanticAITransformation: # Need to poll for completion task_id = result.get("id") if task_id: - verbose_logger.info(f"Pydantic AI: Task {task_id} submitted, polling for completion...") + verbose_logger.info("Pydantic AI: Task %s submitted, polling for completion...", task_id) response_data = await PydanticAITransformation._poll_for_completion( client=client, endpoint=endpoint, @@ -209,7 +209,7 @@ class PydanticAITransformation: agent_extra_headers=agent_extra_headers, ) - verbose_logger.info(f"Pydantic AI: Received completed response for request_id={request_id}") + verbose_logger.info("Pydantic AI: Received completed response for request_id=%s", request_id) return response_data @@ -518,4 +518,4 @@ class PydanticAITransformation: } yield completed_event - verbose_logger.info(f"Pydantic AI: Fake streaming completed for request_id={request_id}") + verbose_logger.info("Pydantic AI: Fake streaming completed for request_id=%s", request_id) diff --git a/litellm/a2a_protocol/providers/watsonx_orchestrate/handler.py b/litellm/a2a_protocol/providers/watsonx_orchestrate/handler.py index efb2b38b912..2c7c04cec0b 100644 --- a/litellm/a2a_protocol/providers/watsonx_orchestrate/handler.py +++ b/litellm/a2a_protocol/providers/watsonx_orchestrate/handler.py @@ -135,7 +135,7 @@ class WatsonxOrchestrateHandler: response.raise_for_status() result: dict[str, Any] = response.json() status = result.get("status", "") - verbose_logger.debug(f"WXO: Poll {attempt + 1}/{max_attempts} run='{run_id}' status='{status}'") + verbose_logger.debug("WXO: Poll %s/%s run='%s' status='%s'", attempt + 1, max_attempts, run_id, status) if status in WatsonxOrchestrateTransformation.TERMINAL_STATES: return result @@ -297,8 +297,8 @@ class WatsonxOrchestrateHandler: response.raise_for_status() except httpx.TransportError as exc: verbose_logger.warning( - f"WXO: Streaming request failed before a run was submitted " - f"({exc!r}), falling back to non-streaming + fake streaming", + "WXO: Streaming request failed before a run was submitted (%r), falling back to non-streaming + fake streaming", + exc, exc_info=True, ) result = await WatsonxOrchestrateHandler.handle_non_streaming( diff --git a/litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py b/litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py index ab7b8abb3ba..18b0795aa8a 100644 --- a/litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py +++ b/litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py @@ -214,4 +214,4 @@ class WatsonxOrchestrateTransformation: }, } - verbose_logger.debug(f"WXO: Fake streaming completed for request_id={request_id}") + verbose_logger.debug("WXO: Fake streaming completed for request_id=%s", request_id) diff --git a/litellm/a2a_protocol/streaming_iterator.py b/litellm/a2a_protocol/streaming_iterator.py index 79056ca336f..f954cb187b5 100644 --- a/litellm/a2a_protocol/streaming_iterator.py +++ b/litellm/a2a_protocol/streaming_iterator.py @@ -138,13 +138,15 @@ class A2AStreamingIterator: ) verbose_logger.info( - f"A2A streaming completed: prompt_tokens={prompt_tokens}, " - f"completion_tokens={completion_tokens}, total_tokens={total_tokens}, " - f"response_cost={response_cost}" + "A2A streaming completed: prompt_tokens=%s, completion_tokens=%s, total_tokens=%s, response_cost=%s", + prompt_tokens, + completion_tokens, + total_tokens, + response_cost, ) except Exception as e: - verbose_logger.debug(f"Error in A2A streaming completion handler: {e}") + verbose_logger.debug("Error in A2A streaming completion handler: %s", e) def _build_logging_result(self, usage: litellm.Usage) -> dict[str, Any]: """Build a result dict for logging.""" diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index 542885b5130..063a83e6f38 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -51,7 +51,7 @@ class GetAnthropicBetaHeadersConfig: ) return content except Exception as e: - verbose_logger.error(f"Failed to load local beta headers config: {e}") + verbose_logger.error("Failed to load local beta headers config: %s", e) # Return empty config as fallback return { "anthropic": {}, @@ -246,7 +246,9 @@ def filter_and_transform_beta_headers( # Check if header is in the mapping if header not in provider_mapping: - verbose_logger.debug(f"Dropping unknown beta header '{header}' for provider '{provider}' (not in mapping)") + verbose_logger.debug( + "Dropping unknown beta header '%s' for provider '%s' (not in mapping)", header, provider + ) continue # Get the mapped header value @@ -254,7 +256,7 @@ def filter_and_transform_beta_headers( # Skip if header is unsupported (null value) if mapped_header is None: - verbose_logger.debug(f"Dropping unsupported beta header '{header}' for provider '{provider}'") + verbose_logger.debug("Dropping unsupported beta header '%s' for provider '%s'", header, provider) continue # Add the mapped header diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index 2e28aaa14df..4f90b50eaa2 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -258,10 +258,10 @@ async def _fetch_batch_output_file_content( if is_base64_unified_file_id: try: file_id = is_base64_unified_file_id.split("llm_output_file_id,")[1].split(";")[0] - verbose_logger.debug(f"Extracted LLM output file ID from unified file ID: {file_id}") + verbose_logger.debug("Extracted LLM output file ID from unified file ID: %s", file_id) except (IndexError, AttributeError) as e: verbose_logger.error( - f"Failed to extract LLM output file ID from unified file ID: {batch.output_file_id}, error: {e}" + "Failed to extract LLM output file ID from unified file ID: %s, error: %s", batch.output_file_id, e ) # Build kwargs for afile_content with credentials from litellm_params diff --git a/litellm/batches/main.py b/litellm/batches/main.py index 3a057d41744..33f2f4613bf 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -182,7 +182,7 @@ def create_batch( ) except Exception as e: verbose_logger.exception( - f"litellm.batches.main.py::create_batch() - Error inferring custom_llm_provider - {e}" + "litellm.batches.main.py::create_batch() - Error inferring custom_llm_provider - %s", e ) _is_async = kwargs.pop("acreate_batch", False) is True @@ -890,7 +890,7 @@ def cancel_batch( ) except Exception as e: verbose_logger.exception( - f"litellm.batches.main.py::cancel_batch() - Error inferring custom_llm_provider - {e}" + "litellm.batches.main.py::cancel_batch() - Error inferring custom_llm_provider - %s", e ) optional_params = GenericLiteLLMParams(**kwargs) litellm_params = get_litellm_params( diff --git a/litellm/caching/azure_blob_cache.py b/litellm/caching/azure_blob_cache.py index 80ad645ec7b..755b491f9a0 100644 --- a/litellm/caching/azure_blob_cache.py +++ b/litellm/caching/azure_blob_cache.py @@ -67,7 +67,10 @@ class AzureBlobCache(BaseCache): cached_response = json.loads(as_str) verbose_logger.debug( - f"Got Azure Blob Cache: key: {key}, cached_response {cached_response}. Type Response {type(cached_response)}" + "Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s", + key, + cached_response, + type(cached_response), ) return cached_response @@ -84,7 +87,10 @@ class AzureBlobCache(BaseCache): as_str = as_bytes.decode("utf-8") cached_response = json.loads(as_str) verbose_logger.debug( - f"Got Azure Blob Cache: key: {key}, cached_response {cached_response}. Type Response {type(cached_response)}" + "Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s", + key, + cached_response, + type(cached_response), ) return cached_response except ResourceNotFoundError: diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 9542be0999a..758a14afb17 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -676,7 +676,7 @@ class Cache: cache_key, cached_data, kwargs = self._add_cache_logic(result=result, **kwargs) self.cache.set_cache(cache_key, cached_data, **kwargs) except Exception as e: - verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {e}") + verbose_logger.exception("LiteLLM Cache: Excepton add_cache: %s", e) async def async_add_cache(self, result, dynamic_cache_object: BaseCache | None = None, **kwargs): """ @@ -695,7 +695,7 @@ class Cache: else: await self.cache.async_set_cache(cache_key, cached_data, **kwargs) except Exception as e: - verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {e}") + verbose_logger.exception("LiteLLM Cache: Excepton add_cache: %s", e) def _convert_to_cached_embedding( self, @@ -874,7 +874,7 @@ class Cache: else: await self.cache.async_set_cache_pipeline(cache_list=cache_list, **kwargs) except Exception as e: - verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {e}") + verbose_logger.exception("LiteLLM Cache: Excepton add_cache: %s", e) def should_use_cache(self, **kwargs): """ diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index aed38d6ef65..2655a5ad683 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -271,7 +271,7 @@ class LLMCachingHandler: embedding_all_elements_cache_hit=embedding_all_elements_cache_hit, ) - verbose_logger.debug(f"CACHE RESULT: {cached_result}") + verbose_logger.debug("CACHE RESULT: %s", cached_result) return CachingHandlerResponse( cached_result=cached_result, final_embedding_cached_response=final_embedding_cached_response, diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index b641c600a0e..a242f4a818e 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -147,7 +147,7 @@ class DualCache(BaseCache): return result except Exception as e: - verbose_logger.error(f"LiteLLM Cache: Excepton async add_cache: {e}") + verbose_logger.error("LiteLLM Cache: Excepton async add_cache: %s", e) raise e def get_cache( @@ -347,7 +347,7 @@ class DualCache(BaseCache): if self.redis_cache is not None and local_only is False: await self.redis_cache.async_set_cache(key, value, **kwargs) except Exception as e: - verbose_logger.exception(f"LiteLLM Cache: Excepton async add_cache: {e}") + verbose_logger.exception("LiteLLM Cache: Excepton async add_cache: %s", e) # async_batch_set_cache async def async_set_cache_pipeline(self, cache_list: list, local_only: bool = False, **kwargs): @@ -366,7 +366,7 @@ class DualCache(BaseCache): cache_list=cache_list, ttl=kwargs.pop("ttl", None), **kwargs ) except Exception as e: - verbose_logger.exception(f"LiteLLM Cache: Excepton async add_cache: {e}") + verbose_logger.exception("LiteLLM Cache: Excepton async add_cache: %s", e) async def async_increment_cache( self, diff --git a/litellm/caching/gcs_cache.py b/litellm/caching/gcs_cache.py index d74c68de770..1e1508669b3 100644 --- a/litellm/caching/gcs_cache.py +++ b/litellm/caching/gcs_cache.py @@ -71,12 +71,15 @@ class GCSCache(BaseCache): if response.status_code == 200: cached_response = json.loads(response.text) verbose_logger.debug( - f"Got GCS Cache: key: {key}, cached_response {cached_response}. Type Response {type(cached_response)}" + "Got GCS Cache: key: %s, cached_response %s. Type Response %s", + key, + cached_response, + type(cached_response), ) return cached_response return None except Exception as e: - verbose_logger.error(f"GCS Caching: get_cache() - Got exception from GCS: {e}") + verbose_logger.error("GCS Caching: get_cache() - Got exception from GCS: %s", e) async def async_get_cache(self, key, **kwargs): try: @@ -89,7 +92,7 @@ class GCSCache(BaseCache): return json.loads(response.text) return None except Exception as e: - verbose_logger.error(f"GCS Caching: async_get_cache() - Got exception from GCS: {e}") + verbose_logger.error("GCS Caching: async_get_cache() - Got exception from GCS: %s", e) def flush_cache(self): pass diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index e3c0e3616f0..2dfd123d46d 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -346,7 +346,8 @@ class RedisCache(BaseCache): verbose_logger.debug("Ignoring async redis ping. No running event loop.") else: verbose_logger.error( - f"Error connecting to Async Redis client - {e}", + "Error connecting to Async Redis client - %s", + e, extra={"error": str(e)}, ) self._handle_async_ping_error(e) @@ -1139,7 +1140,7 @@ class RedisCache(BaseCache): return decoded_results except Exception as e: - verbose_logger.error(f"Error occurred in batch get cache - {e}") + verbose_logger.error("Error occurred in batch get cache - %s", e) return key_value_dict @_redis_circuit_breaker_guard @@ -1257,7 +1258,7 @@ class RedisCache(BaseCache): parent_otel_span=parent_otel_span, ) ) - verbose_logger.error(f"Error occurred in async batch get cache - {e}") + verbose_logger.error("Error occurred in async batch get cache - %s", e) _record_swallowed_redis_failure(self._circuit_breaker, e) return key_value_dict @@ -1292,7 +1293,7 @@ class RedisCache(BaseCache): error=e, call_type=f"sync_ping <- {_get_call_stack_info()}", ) - verbose_logger.error(f"LiteLLM Redis Cache PING: - Got exception from REDIS : {e}") + verbose_logger.error("LiteLLM Redis Cache PING: - Got exception from REDIS : %s", e) raise e async def ping(self) -> bool: @@ -1326,7 +1327,7 @@ class RedisCache(BaseCache): call_type=f"async_ping <- {_get_call_stack_info()}", ) ) - verbose_logger.error(f"LiteLLM Redis Cache PING: - Got exception from REDIS : {e}") + verbose_logger.error("LiteLLM Redis Cache PING: - Got exception from REDIS : %s", e) raise e @_redis_circuit_breaker_guard @@ -1388,7 +1389,7 @@ class RedisCache(BaseCache): else: return {"status": "failed", "message": "Redis ping returned False"} except Exception as e: - verbose_logger.error(f"Redis connection test failed: {e}") + verbose_logger.error("Redis connection test failed: %s", e) return { "status": "failed", "message": f"Redis connection failed: {e}", @@ -1426,7 +1427,7 @@ class RedisCache(BaseCache): # Execute the pipeline and return results results = await pipe.execute() # only return float values - verbose_logger.debug(f"Increment ASYNC Redis Cache PIPELINE: results: {results}") + verbose_logger.debug("Increment ASYNC Redis Cache PIPELINE: results: %s", results) return [r for r in results if isinstance(r, float)] @_redis_circuit_breaker_guard @@ -1513,7 +1514,7 @@ class RedisCache(BaseCache): return None return ttl except Exception as e: - verbose_logger.debug(f"Redis TTL Error: {e}") + verbose_logger.debug("Redis TTL Error: %s", e) _record_swallowed_redis_failure(self._circuit_breaker, e) return None @@ -1565,7 +1566,7 @@ class RedisCache(BaseCache): call_type=f"async_rpush <- {_get_call_stack_info()}", ) ) - verbose_logger.error(f"LiteLLM Redis Cache RPUSH: - Got exception from REDIS : {e}") + verbose_logger.error("LiteLLM Redis Cache RPUSH: - Got exception from REDIS : %s", e) raise e async def _pipeline_rpush_helper( @@ -1711,7 +1712,7 @@ class RedisCache(BaseCache): call_type=f"async_lpop <- {_get_call_stack_info()}", ) ) - verbose_logger.error(f"LiteLLM Redis Cache LPOP: - Got exception from REDIS : {e}") + verbose_logger.error("LiteLLM Redis Cache LPOP: - Got exception from REDIS : %s", e) raise e async def _pipeline_lpop_helper( diff --git a/litellm/caching/redis_cluster_cache.py b/litellm/caching/redis_cluster_cache.py index 127a5c3bd29..926712e38ec 100644 --- a/litellm/caching/redis_cluster_cache.py +++ b/litellm/caching/redis_cluster_cache.py @@ -100,7 +100,7 @@ class RedisClusterCache(RedisCache): except Exception as e: from litellm._logging import verbose_logger - verbose_logger.error(f"Redis Cluster connection test failed: {e}") + verbose_logger.error("Redis Cluster connection test failed: %s", e) return { "status": "failed", "message": f"Redis Cluster connection failed: {e}", diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index f55274d446d..4fe42d1908e 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -138,7 +138,7 @@ class RedisSemanticCache(BaseCache): cache_vectorizer=cache_vectorizer, ) except Exception as e: - verbose_logger.error(f"Redis semantic-cache index build failed: {e}") + verbose_logger.error("Redis semantic-cache index build failed: %s", e) raise @classmethod diff --git a/litellm/caching/s3_cache.py b/litellm/caching/s3_cache.py index 5e185de7526..baad0e29c5e 100644 --- a/litellm/caching/s3_cache.py +++ b/litellm/caching/s3_cache.py @@ -104,12 +104,12 @@ class S3Cache(BaseCache): Compatible with Python 3.8+. """ try: - verbose_logger.debug(f"Set ASYNC S3 Cache: Key={key}. Value={value}") + verbose_logger.debug("Set ASYNC S3 Cache: Key=%s. Value=%s", key, value) loop = asyncio.get_event_loop() func = partial(self.set_cache, key, value, **kwargs) await loop.run_in_executor(None, func) except Exception as e: - verbose_logger.error(f"S3 Caching: async_set_cache() - Got exception from S3: {e}") + verbose_logger.error("S3 Caching: async_set_cache() - Got exception from S3: %s", e) def get_cache(self, key, **kwargs): import botocore @@ -138,17 +138,20 @@ class S3Cache(BaseCache): if not isinstance(cached_response, dict): cached_response = dict(cached_response) verbose_logger.debug( - f"Got S3 Cache: key: {key}, cached_response {cached_response}. Type Response {type(cached_response)}" + "Got S3 Cache: key: %s, cached_response %s. Type Response %s", + key, + cached_response, + type(cached_response), ) return cached_response except botocore.exceptions.ClientError as e: # type: ignore if e.response["Error"]["Code"] == "NoSuchKey": - verbose_logger.debug(f"S3 Cache: The specified key '{key}' does not exist in the S3 bucket.") + verbose_logger.debug("S3 Cache: The specified key '%s' does not exist in the S3 bucket.", key) return None except Exception as e: - verbose_logger.error(f"S3 Caching: get_cache() - Got exception from S3: {e}") + verbose_logger.error("S3 Caching: get_cache() - Got exception from S3: %s", e) async def async_get_cache(self, key, **kwargs): """ @@ -156,13 +159,13 @@ class S3Cache(BaseCache): Compatible with Python 3.8+. """ try: - verbose_logger.debug(f"Get ASYNC S3 Cache: key: {key}") + verbose_logger.debug("Get ASYNC S3 Cache: key: %s", key) loop = asyncio.get_event_loop() func = partial(self.get_cache, key, **kwargs) result = await loop.run_in_executor(None, func) return result except Exception as e: - verbose_logger.error(f"S3 Caching: async_get_cache() - Got exception from S3: {e}") + verbose_logger.error("S3 Caching: async_get_cache() - Got exception from S3: %s", e) return None def flush_cache(self): diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 3825854852d..16c90d5a20a 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -408,7 +408,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): self._map_optional_params_to_responses_api_request(optional_params, responses_api_request) stream = optional_params.get("stream") or litellm_params.get("stream", False) - verbose_logger.debug(f"Chat provider: Stream parameter: {stream}") + verbose_logger.debug("Chat provider: Stream parameter: %s", stream) # Ensure stream is properly set in the request if stream: @@ -418,7 +418,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): previous_response_id = optional_params.get("previous_response_id") if previous_response_id: # Use the existing session handler for responses API - verbose_logger.debug(f"Chat provider: Warning ignoring previous response ID: {previous_response_id}") + verbose_logger.debug("Chat provider: Warning ignoring previous response ID: %s", previous_response_id) # Convert back to responses API format for the actual request @@ -438,7 +438,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): "client": client, } - verbose_logger.debug(f"Chat provider: Final request model={api_model}, input_items={len(input_items)}") + verbose_logger.debug("Chat provider: Final request model=%s, input_items=%s", api_model, len(input_items)) self._merge_responses_api_request_into_request_data(request_data, responses_api_request, instructions) @@ -776,29 +776,29 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): """Convert chat completion content to responses API format""" from litellm.types.llms.openai import ChatCompletionImageObject - verbose_logger.debug(f"Chat provider: Converting content to responses format - input type: {type(content)}") + verbose_logger.debug("Chat provider: Converting content to responses format - input type: %s", type(content)) if content is None: return [self._convert_content_str_to_input_text("", role)] elif isinstance(content, str): result = [self._convert_content_str_to_input_text(content, role)] - verbose_logger.debug(f"Chat provider: String content -> {result}") + verbose_logger.debug("Chat provider: String content -> %s", result) return result elif isinstance(content, list): result = [] for i, item in enumerate(content): - verbose_logger.debug(f"Chat provider: Processing content item {i}: {type(item)} = {item}") + verbose_logger.debug("Chat provider: Processing content item %s: %s = %s", i, type(item), item) if isinstance(item, str): converted = self._convert_content_str_to_input_text(item, role) result.append(converted) - verbose_logger.debug(f"Chat provider: -> {converted}") + verbose_logger.debug("Chat provider: -> %s", converted) elif isinstance(item, dict): # Handle multimodal content original_type = item.get("type") if original_type == "text": converted = self._convert_content_str_to_input_text(item.get("text", ""), role) result.append(converted) - verbose_logger.debug(f"Chat provider: text -> {converted}") + verbose_logger.debug("Chat provider: text -> %s", converted) elif original_type == "image_url": # Map to responses API image format converted = cast( @@ -808,14 +808,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ), ) result.append(converted) - verbose_logger.debug(f"Chat provider: image_url -> {converted}") + verbose_logger.debug("Chat provider: image_url -> %s", converted) else: # Try to map other types to responses API format item_type = original_type or "input_text" if item_type == "image": converted = {"type": "input_image", **item} result.append(converted) - verbose_logger.debug(f"Chat provider: image -> {converted}") + verbose_logger.debug("Chat provider: image -> %s", converted) elif item_type == "file": # Map Chat Completion file to Responses API input_file # {"type": "file", "file": {"file_data": "...", "filename": "..."}} @@ -827,7 +827,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if key in file_data: converted[key] = file_data[key] result.append(converted) - verbose_logger.debug(f"Chat provider: file -> {converted}") + verbose_logger.debug("Chat provider: file -> %s", converted) elif item_type in [ "input_text", "input_image", @@ -839,17 +839,17 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ]: # Already in responses API format result.append(item) - verbose_logger.debug(f"Chat provider: passthrough -> {item}") + verbose_logger.debug("Chat provider: passthrough -> %s", item) else: # Default to input_text for unknown types converted = self._convert_content_str_to_input_text(str(item.get("text", item)), role) result.append(converted) - verbose_logger.debug(f"Chat provider: unknown({original_type}) -> {converted}") - verbose_logger.debug(f"Chat provider: Final converted content: {result}") + verbose_logger.debug("Chat provider: unknown(%s) -> %s", original_type, converted) + verbose_logger.debug("Chat provider: Final converted content: %s", result) return result else: result = [self._convert_content_str_to_input_text(str(content), role)] - verbose_logger.debug(f"Chat provider: Other content type -> {result}") + verbose_logger.debug("Chat provider: Other content type -> %s", result) return result def _convert_tools_to_responses_format(self, tools: list[dict[str, Any]]) -> list["ALL_RESPONSES_API_TOOL_PARAMS"]: @@ -1032,13 +1032,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): annotation_dict = annotation else: # Skip unsupported annotation types - verbose_logger.debug(f"Skipping unsupported annotation type: {type(annotation)}") + verbose_logger.debug("Skipping unsupported annotation type: %s", type(annotation)) continue result.append(annotation_dict) # type: ignore except Exception as e: # Skip malformed annotations - verbose_logger.debug(f"Skipping malformed annotation: {annotation}, error: {e}") + verbose_logger.debug("Skipping malformed annotation: %s, error: %s", annotation, e) continue return result if result else None @@ -1122,11 +1122,11 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): ): return ModelResponseStream(**parsed_chunk) - verbose_logger.debug(f"Chat provider: Processing event type: {event_type}") + verbose_logger.debug("Chat provider: Processing event type: %s", event_type) if event_type == "response.created": # Initial response creation event - verbose_logger.debug(f"Chat provider: response.created -> {parsed_chunk}") + verbose_logger.debug("Chat provider: response.created -> %s", parsed_chunk) return ModelResponseStream( choices=[ StreamingChoices( @@ -1345,7 +1345,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): else: pass # For any unhandled event types, create a minimal valid chunk or skip - verbose_logger.debug(f"Chat provider: Unhandled event type '{event_type}', creating empty chunk") + verbose_logger.debug("Chat provider: Unhandled event type '%s', creating empty chunk", event_type) # Return a minimal valid chunk for unknown events return ModelResponseStream( @@ -1368,7 +1368,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): Returns: ModelResponseStream: OpenAI-formatted streaming chunk """ - verbose_logger.debug(f"Chat provider: transform_streaming_response called with chunk: {chunk}") + verbose_logger.debug("Chat provider: transform_streaming_response called with chunk: %s", chunk) return self._with_stream_scoped_id( OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(chunk) ) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 25d448e48e9..bd03feb03c2 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -273,7 +273,7 @@ def _get_additional_costs( completion_tokens=completion_tokens, ) except Exception as e: - verbose_logger.debug(f"Error calculating additional costs: {e}") + verbose_logger.debug("Error calculating additional costs: %s", e) return None @@ -715,7 +715,7 @@ def _get_provider_for_cost_calc( _, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model) except Exception as e: verbose_logger.debug( - f"litellm.cost_calculator.py::_get_provider_for_cost_calc() - Error inferring custom_llm_provider - {e}" + "litellm.cost_calculator.py::_get_provider_for_cost_calc() - Error inferring custom_llm_provider - %s", e ) return None @@ -896,7 +896,7 @@ def _get_usage_object( elif isinstance(usage_obj, BaseModel): return Usage(**usage_obj.model_dump()) else: - verbose_logger.debug(f"Unknown usage object type: {type(usage_obj)}, usage_obj: {usage_obj}") + verbose_logger.debug("Unknown usage object type: %s, usage_obj: %s", type(usage_obj), usage_obj) return None @@ -994,16 +994,17 @@ def _apply_cost_margin( if custom_llm_provider and custom_llm_provider in litellm.cost_margin_config: margin_config = litellm.cost_margin_config[custom_llm_provider] if verbose_logger.isEnabledFor(logging.DEBUG): - verbose_logger.debug(f"Found provider-specific margin config for {custom_llm_provider}: {margin_config}") + verbose_logger.debug("Found provider-specific margin config for %s: %s", custom_llm_provider, margin_config) elif "global" in litellm.cost_margin_config: margin_config = litellm.cost_margin_config["global"] if verbose_logger.isEnabledFor(logging.DEBUG): - verbose_logger.debug(f"Using global margin config: {margin_config}") + verbose_logger.debug("Using global margin config: %s", margin_config) else: if verbose_logger.isEnabledFor(logging.DEBUG): verbose_logger.debug( - f"No margin config found. Provider: {custom_llm_provider}, " - f"Available configs: {list(litellm.cost_margin_config.keys())}" + "No margin config found. Provider: %s, Available configs: %s", + custom_llm_provider, + list(litellm.cost_margin_config.keys()), ) if margin_config is not None: @@ -1098,7 +1099,7 @@ def _store_cost_breakdown_in_logging_obj( ) except Exception as breakdown_error: - verbose_logger.debug(f"Error storing cost breakdown: {breakdown_error}") + verbose_logger.debug("Error storing cost breakdown: %s", breakdown_error) # Don't fail the main cost calculation if breakdown storage fails @@ -1225,7 +1226,7 @@ def completion_cost( for idx, model in enumerate(potential_model_names): try: if verbose_logger.isEnabledFor(logging.DEBUG): - verbose_logger.debug(f"selected model name for cost calculation: {model}") + verbose_logger.debug("selected model name for cost calculation: %s", model) if completion_response is not None and ( isinstance(completion_response, BaseModel) or isinstance(completion_response, dict) @@ -1321,7 +1322,8 @@ def completion_cost( ) # strip the llm provider from the model name -> for image gen cost calculation except Exception as e: verbose_logger.debug( - f"litellm.cost_calculator.py::completion_cost() - Error inferring custom_llm_provider - {e}" + "litellm.cost_calculator.py::completion_cost() - Error inferring custom_llm_provider - %s", + e, ) if CostCalculatorUtils._call_type_has_image_response(call_type) and isinstance( completion_response, ImageResponse @@ -1672,7 +1674,7 @@ def completion_cost( return _final_cost except Exception as e: verbose_logger.debug( - f"litellm.cost_calculator.py::completion_cost() - Error calculating cost for model={model} - {e}" + "litellm.cost_calculator.py::completion_cost() - Error calculating cost for model=%s - %s", model, e ) if idx == len(potential_model_names) - 1: raise e @@ -1888,7 +1890,7 @@ def vector_store_search_cost( ) if config is None: - verbose_logger.debug(f"Vector store search is not supported for {custom_llm_provider}") + verbose_logger.debug("Vector store search is not supported for %s", custom_llm_provider) return 0.0, 0.0 return config.calculate_vector_store_cost( @@ -1976,7 +1978,7 @@ def default_image_cost_calculator( # gpt-image-1 models use low, medium, high quality. If user did not specify quality, use medium fot gpt-image-1 model family model_name_with_v2_quality = f"{ImageGenerationRequestQuality.HIGH.value}/{base_model_name}" - verbose_logger.debug(f"Looking up cost for models: {model_name_with_quality}, {base_model_name}") + verbose_logger.debug("Looking up cost for models: %s, %s", model_name_with_quality, base_model_name) model_without_provider = f"{size_str}/{model.split('/')[-1]}" model_with_quality_without_provider = f"{quality}/{model_without_provider}" if quality else model_without_provider @@ -2046,7 +2048,7 @@ def default_video_cost_calculator( model_name_without_custom_llm_provider = model.replace(f"{custom_llm_provider}/", "") base_model_name = f"{custom_llm_provider}/{model_name_without_custom_llm_provider}" - verbose_logger.debug(f"Looking up cost for video model: {base_model_name}") + verbose_logger.debug("Looking up cost for video model: %s", base_model_name) model_without_provider = model.split("/")[-1] @@ -2082,7 +2084,8 @@ def default_video_cost_calculator( # If no cost information found, return 0 verbose_logger.info( - f"No cost information found for video model {model}. Please add pricing to model_prices_and_context_window.json" + "No cost information found for video model %s. Please add pricing to model_prices_and_context_window.json", + model, ) return 0.0 diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index 8815c38192b..c9d73363242 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -364,7 +364,7 @@ class MCPClient: try: await session_ctx.__aexit__(None, None, None) except BaseException as e: - verbose_logger.debug(f"Error during session context exit: {e}") + verbose_logger.debug("Error during session context exit: %s", e) except BaseException as e: in_flight_error = e raise @@ -372,7 +372,7 @@ class MCPClient: try: await transport_ctx.__aexit__(None, None, None) except BaseException as exit_error: - verbose_logger.debug(f"Error during transport context exit: {exit_error}") + verbose_logger.debug("Error during transport context exit: %s", exit_error) root_cause = _first_non_cancelled_cause(exit_error) if root_cause is not None and isinstance(in_flight_error, asyncio.CancelledError): raise root_cause from in_flight_error @@ -402,7 +402,7 @@ class MCPClient: try: await http_client.aclose() except BaseException as e: - verbose_logger.debug(f"Error during http_client cleanup: {e}") + verbose_logger.debug("Error during http_client cleanup: %s", e) def update_auth_value(self, mcp_auth_value: str | dict[str, str]): """ @@ -464,7 +464,7 @@ class MCPClient: """Create an httpx.AsyncClient with LiteLLM's SSL configuration.""" # Get unified SSL configuration using the same logic as http_handler.py ssl_config = get_ssl_configuration(self.ssl_verify) - verbose_logger.debug(f"MCP client using SSL configuration: {type(ssl_config).__name__}") + verbose_logger.debug("MCP client using SSL configuration: %s", type(ssl_config).__name__) # The MCP SDK's sse_client and streamable_http_client call this factory without # passing auth=, so the fallback is used: a v2-resolved auth if present, else the # SigV4 aws_auth. Both are None for the common case — no behavior change. @@ -490,7 +490,7 @@ class MCPClient: MCP client (triggering the upstream OAuth flow) rather than masking them as "connected, no tools". """ - verbose_logger.debug(f"MCP client listing tools from {self.server_url or 'stdio'}") + verbose_logger.debug("MCP client listing tools from %s", self.server_url or "stdio") async def _list_tools_operation(session: ClientSession): return await session.list_tools() @@ -499,7 +499,9 @@ class MCPClient: result = await self.run_with_session(_list_tools_operation, quiet_on_error=raise_on_error) tool_count = len(result.tools) tool_names = [tool.name for tool in result.tools] - verbose_logger.info(f"MCP client listed {tool_count} tools from {self.server_url or 'stdio'}: {tool_names}") + verbose_logger.info( + "MCP client listed %s tools from %s: %s", tool_count, self.server_url or "stdio", tool_names + ) return result.tools except asyncio.CancelledError: verbose_logger.warning("MCP client list_tools was cancelled") @@ -555,7 +557,7 @@ class MCPClient: an upstream 401 so it can re-mint the exchanged token and retry once; every other caller keeps the default and gets graceful ``isError`` degradation. """ - verbose_logger.info(f"MCP client calling tool '{call_tool_request_params.name}'") + verbose_logger.info("MCP client calling tool '%s'", call_tool_request_params.name) async def on_progress(progress: float, total: float | None, message: str | None): percentage = (progress / total * 100) if total else 0 @@ -568,7 +570,7 @@ class MCPClient: try: await host_progress_callback(progress, total) except Exception as e: - verbose_logger.warning(f"Failed to forward to Host: {e}") + verbose_logger.warning("Failed to forward to Host: %s", e) async def _call_tool_operation(session: ClientSession): verbose_logger.debug("MCP client sending tool call to session") @@ -580,16 +582,16 @@ class MCPClient: try: tool_result = await self.run_with_session(_call_tool_operation, quiet_on_error=raise_on_error) - verbose_logger.info(f"MCP client tool call '{call_tool_request_params.name}' completed successfully") + verbose_logger.info("MCP client tool call '%s' completed successfully", call_tool_request_params.name) return tool_result except asyncio.CancelledError: - verbose_logger.warning(f"MCP client tool call timed out after {self.timeout}s for {self.server_url}") + verbose_logger.warning("MCP client tool call timed out after %ss for %s", self.timeout, self.server_url) raise except Exception as e: import traceback error_trace = traceback.format_exc() - verbose_logger.debug(f"MCP client tool call traceback:\n{error_trace}") + verbose_logger.debug("MCP client tool call traceback:\n%s", error_trace) # Log detailed error information error_type = type(e).__name__ # When the caller opted into raise_on_error it owns the exception and logs it at the @@ -619,7 +621,7 @@ class MCPClient: async def list_prompts(self) -> list[Prompt]: """List available prompts from the server.""" - verbose_logger.debug(f"MCP client listing tools from {self.server_url or 'stdio'}") + verbose_logger.debug("MCP client listing tools from %s", self.server_url or "stdio") async def _list_prompts_operation(session: ClientSession): return await session.list_prompts() @@ -629,7 +631,7 @@ class MCPClient: prompt_count = len(result.prompts) prompt_names = [prompt.name for prompt in result.prompts] verbose_logger.info( - f"MCP client listed {prompt_count} tools from {self.server_url or 'stdio'}: {prompt_names}" + "MCP client listed %s tools from %s: %s", prompt_count, self.server_url or "stdio", prompt_names ) return result.prompts except asyncio.CancelledError: @@ -638,11 +640,11 @@ class MCPClient: except Exception as e: error_type = type(e).__name__ verbose_logger.error( - f"MCP client list_prompts failed - " - f"Error Type: {error_type}, " - f"Error: {e}, " - f"Server: {self.server_url or 'stdio'}, " - f"Transport: {self.transport_type}" + "MCP client list_prompts failed - Error Type: %s, Error: %s, Server: %s, Transport: %s", + error_type, + e, + self.server_url or "stdio", + self.transport_type, ) # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: @@ -655,7 +657,7 @@ class MCPClient: async def get_prompt(self, get_prompt_request_params: GetPromptRequestParams) -> GetPromptResult: """Fetch a prompt definition from the MCP server.""" - verbose_logger.info(f"MCP client fetching prompt '{get_prompt_request_params.name}'") + verbose_logger.info("MCP client fetching prompt '%s'", get_prompt_request_params.name) async def _get_prompt_operation(session: ClientSession): verbose_logger.debug("MCP client sending get_prompt request to session") @@ -666,7 +668,7 @@ class MCPClient: try: get_prompt_result = await self.run_with_session(_get_prompt_operation) - verbose_logger.info(f"MCP client get_prompt '{get_prompt_request_params.name}' completed successfully") + verbose_logger.info("MCP client get_prompt '%s' completed successfully", get_prompt_request_params.name) return get_prompt_result except asyncio.CancelledError: verbose_logger.warning("MCP client get_prompt was cancelled") @@ -675,16 +677,16 @@ class MCPClient: import traceback error_trace = traceback.format_exc() - verbose_logger.debug(f"MCP client get_prompt traceback:\n{error_trace}") + verbose_logger.debug("MCP client get_prompt traceback:\n%s", error_trace) # Log detailed error information error_type = type(e).__name__ verbose_logger.error( - f"MCP client get_prompt failed - " - f"Error Type: {error_type}, " - f"Error: {e}, " - f"Prompt: {get_prompt_request_params.name}, " - f"Server: {self.server_url or 'stdio'}, " - f"Transport: {self.transport_type}" + "MCP client get_prompt failed - Error Type: %s, Error: %s, Prompt: %s, Server: %s, Transport: %s", + error_type, + e, + get_prompt_request_params.name, + self.server_url or "stdio", + self.transport_type, ) # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: @@ -696,7 +698,7 @@ class MCPClient: async def list_resources(self) -> list[Resource]: """List available resources from the server.""" - verbose_logger.debug(f"MCP client listing resources from {self.server_url or 'stdio'}") + verbose_logger.debug("MCP client listing resources from %s", self.server_url or "stdio") async def _list_resources_operation(session: ClientSession): return await session.list_resources() @@ -706,7 +708,7 @@ class MCPClient: resource_count = len(result.resources) resource_names = [resource.name for resource in result.resources] verbose_logger.info( - f"MCP client listed {resource_count} resources from {self.server_url or 'stdio'}: {resource_names}" + "MCP client listed %s resources from %s: %s", resource_count, self.server_url or "stdio", resource_names ) return result.resources except asyncio.CancelledError: @@ -715,11 +717,11 @@ class MCPClient: except Exception as e: error_type = type(e).__name__ verbose_logger.error( - f"MCP client list_resources failed - " - f"Error Type: {error_type}, " - f"Error: {e}, " - f"Server: {self.server_url or 'stdio'}, " - f"Transport: {self.transport_type}" + "MCP client list_resources failed - Error Type: %s, Error: %s, Server: %s, Transport: %s", + error_type, + e, + self.server_url or "stdio", + self.transport_type, ) # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: @@ -732,7 +734,7 @@ class MCPClient: async def list_resource_templates(self) -> list[ResourceTemplate]: """List available resource templates from the server.""" - verbose_logger.debug(f"MCP client listing resource templates from {self.server_url or 'stdio'}") + verbose_logger.debug("MCP client listing resource templates from %s", self.server_url or "stdio") async def _list_resource_templates_operation(session: ClientSession): return await session.list_resource_templates() @@ -742,7 +744,10 @@ class MCPClient: resource_template_count = len(result.resourceTemplates) resource_template_names = [resourceTemplate.name for resourceTemplate in result.resourceTemplates] verbose_logger.info( - f"MCP client listed {resource_template_count} resource templates from {self.server_url or 'stdio'}: {resource_template_names}" + "MCP client listed %s resource templates from %s: %s", + resource_template_count, + self.server_url or "stdio", + resource_template_names, ) return result.resourceTemplates except asyncio.CancelledError: @@ -751,11 +756,11 @@ class MCPClient: except Exception as e: error_type = type(e).__name__ verbose_logger.error( - f"MCP client list_resource_templates failed - " - f"Error Type: {error_type}, " - f"Error: {e}, " - f"Server: {self.server_url or 'stdio'}, " - f"Transport: {self.transport_type}" + "MCP client list_resource_templates failed - Error Type: %s, Error: %s, Server: %s, Transport: %s", + error_type, + e, + self.server_url or "stdio", + self.transport_type, ) # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: @@ -768,7 +773,7 @@ class MCPClient: async def read_resource(self, url: AnyUrl) -> ReadResourceResult: """Fetch resource contents from the MCP server.""" - verbose_logger.info(f"MCP client fetching resource '{url}'") + verbose_logger.info("MCP client fetching resource '%s'", url) async def _read_resource_operation(session: ClientSession): verbose_logger.debug("MCP client sending read_resource request to session") @@ -776,7 +781,7 @@ class MCPClient: try: read_resource_result = await self.run_with_session(_read_resource_operation) - verbose_logger.info(f"MCP client read_resource '{url}' completed successfully") + verbose_logger.info("MCP client read_resource '%s' completed successfully", url) return read_resource_result except asyncio.CancelledError: verbose_logger.warning("MCP client read_resource was cancelled") @@ -785,16 +790,16 @@ class MCPClient: import traceback error_trace = traceback.format_exc() - verbose_logger.debug(f"MCP client read_resource traceback:\n{error_trace}") + verbose_logger.debug("MCP client read_resource traceback:\n%s", error_trace) # Log detailed error information error_type = type(e).__name__ verbose_logger.error( - f"MCP client read_resource failed - " - f"Error Type: {error_type}, " - f"Error: {e}, " - f"Url: {url}, " - f"Server: {self.server_url or 'stdio'}, " - f"Transport: {self.transport_type}" + "MCP client read_resource failed - Error Type: %s, Error: %s, Url: %s, Server: %s, Transport: %s", + error_type, + e, + url, + self.server_url or "stdio", + self.transport_type, ) # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: diff --git a/litellm/google_genai/adapters/transformation.py b/litellm/google_genai/adapters/transformation.py index 7c2800db07b..b13fb71690b 100644 --- a/litellm/google_genai/adapters/transformation.py +++ b/litellm/google_genai/adapters/transformation.py @@ -104,9 +104,10 @@ class GoogleGenAIStreamWrapper(AdapterCompletionStreamWrapper): except json.JSONDecodeError: # This can happen if the stream is abruptly cut off mid-argument string. verbose_logger.warning( - f"Could not parse tool call arguments at end of stream for index {tool_call_index}. " - f"Name: {tool_call_data['name']}. " - f"Partial args: {tool_call_data['arguments']}" + "Could not parse tool call arguments at end of stream for index %s. Name: %s. Partial args: %s", + tool_call_index, + tool_call_data["name"], + tool_call_data["arguments"], ) if parts: final_chunk = { @@ -662,7 +663,7 @@ class GoogleGenAIAdapter: # Optimization: Skip chunks that have no new data if not function_name and not args_chunk: - verbose_logger.debug(f"Skipping empty tool call chunk for index: {tool_call_index}") + verbose_logger.debug("Skipping empty tool call chunk for index: %s", tool_call_index) continue if function_name: diff --git a/litellm/integrations/SlackAlerting/batching_handler.py b/litellm/integrations/SlackAlerting/batching_handler.py index da905b606a5..12b1f772616 100644 --- a/litellm/integrations/SlackAlerting/batching_handler.py +++ b/litellm/integrations/SlackAlerting/batching_handler.py @@ -68,8 +68,8 @@ async def send_to_webhook(slackAlertingInstance: SlackAlertingType, item, count) data=json.dumps(payload), ) if response.status_code != 200: - verbose_proxy_logger.debug(f"Error sending slack alert to url={item['url']}. Error={response.text}") + verbose_proxy_logger.debug("Error sending slack alert to url=%s. Error=%s", item["url"], response.text) except Exception as e: - verbose_proxy_logger.debug(f"Error sending slack alert: {e}") + verbose_proxy_logger.debug("Error sending slack alert: %s", e) finally: _print_alerting_payload_warning(payload, slackAlertingInstance=slackAlertingInstance) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 114924e7359..0d842a5889a 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -1467,7 +1467,7 @@ Model Info: try: await self._flush_digest_buckets() except Exception as e: - verbose_proxy_logger.debug(f"Error flushing digest buckets: {e}") + verbose_proxy_logger.debug("Error flushing digest buckets: %s", e) await self.flush_queue() async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -1502,7 +1502,7 @@ Model Info: ) except Exception as e: verbose_proxy_logger.error( - f"[Non-Blocking Error] Slack Alerting: Got error in logging LLM deployment latency: {e}" + "[Non-Blocking Error] Slack Alerting: Got error in logging LLM deployment latency: %s", e ) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): @@ -1522,7 +1522,7 @@ Model Info: ) ) except Exception as e: - verbose_logger.debug(f"Exception raises -{e}") + verbose_logger.debug("Exception raises -%s", e) if isinstance(kwargs.get("exception", ""), APIError): if "outage_alerts" in self.alert_types: @@ -1662,9 +1662,9 @@ Model Info: ) except ValueError as ve: - verbose_proxy_logger.error(f"Invalid time range format: {ve}") + verbose_proxy_logger.error("Invalid time range format: %s", ve) except Exception as e: - verbose_proxy_logger.error(f"Error sending spend report: {e}") + verbose_proxy_logger.error("Error sending spend report: %s", e) async def send_monthly_spend_report(self): """ """ diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 751c8c01aae..f52b6bd8415 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -143,8 +143,8 @@ class AnthropicCacheControlHook(CustomPromptManagement): if limit_reached: verbose_logger.warning( - f"AnthropicCacheControlHook: Reached the Anthropic limit of " - f"{MAX_CACHE_CONTROL_BLOCKS} cache_control blocks. Skipping further injection." + "AnthropicCacheControlHook: Reached the Anthropic limit of %s cache_control blocks. Skipping further injection.", + MAX_CACHE_CONTROL_BLOCKS, ) return messages @@ -174,8 +174,10 @@ class AnthropicCacheControlHook(CustomPromptManagement): return [targetted_index] verbose_logger.warning( - f"AnthropicCacheControlHook: Provided index {original_index} is out of bounds for message list of length {len(messages)}. " - f"Targeted index was {targetted_index}. Skipping cache control injection for this point." + "AnthropicCacheControlHook: Provided index %s is out of bounds for message list of length %s. Targeted index was %s. Skipping cache control injection for this point.", + original_index, + len(messages), + targetted_index, ) return [] diff --git a/litellm/integrations/argilla.py b/litellm/integrations/argilla.py index d41291f9f98..b1cda6a5593 100644 --- a/litellm/integrations/argilla.py +++ b/litellm/integrations/argilla.py @@ -185,9 +185,9 @@ class ArgillaLogger(CustomBatchLogger): ) if response.status_code >= 300: - verbose_logger.error(f"Argilla Error: {response.status_code} - {response.text}") + verbose_logger.error("Argilla Error: %s - %s", response.status_code, response.text) else: - verbose_logger.debug(f"Batch of {len(self.log_queue)} runs successfully created") + verbose_logger.debug("Batch of %s runs successfully created", len(self.log_queue)) self.log_queue.clear() except Exception: @@ -204,7 +204,7 @@ class ArgillaLogger(CustomBatchLogger): random_sample = random.random() if random_sample > sampling_rate: verbose_logger.info( - f"Skipping Langsmith logging. Sampling rate={sampling_rate}, random_sample={random_sample}" + "Skipping Langsmith logging. Sampling rate=%s, random_sample=%s", sampling_rate, random_sample ) return # Skip logging verbose_logger.debug( @@ -217,7 +217,7 @@ class ArgillaLogger(CustomBatchLogger): return self.log_queue.append(data) - verbose_logger.debug(f"Langsmith, event added to queue. Will flush in {self.flush_interval} seconds...") + verbose_logger.debug("Langsmith, event added to queue. Will flush in %s seconds...", self.flush_interval) if len(self.log_queue) >= self.batch_size: self._send_batch() @@ -231,7 +231,7 @@ class ArgillaLogger(CustomBatchLogger): random_sample = random.random() if random_sample > sampling_rate: verbose_logger.info( - f"Skipping Langsmith logging. Sampling rate={sampling_rate}, random_sample={random_sample}" + "Skipping Langsmith logging. Sampling rate=%s, random_sample=%s", sampling_rate, random_sample ) return # Skip logging verbose_logger.debug( @@ -272,7 +272,7 @@ class ArgillaLogger(CustomBatchLogger): random_sample = random.random() if random_sample > sampling_rate: verbose_logger.info( - f"Skipping Langsmith logging. Sampling rate={sampling_rate}, random_sample={random_sample}" + "Skipping Langsmith logging. Sampling rate=%s, random_sample=%s", sampling_rate, random_sample ) return # Skip logging verbose_logger.info("Langsmith Failure Event Logging!") @@ -325,7 +325,7 @@ class ArgillaLogger(CustomBatchLogger): response.raise_for_status() if response.status_code >= 300: - verbose_logger.error(f"Argilla Error: {response.status_code} - {response.text}") + verbose_logger.error("Argilla Error: %s - %s", response.status_code, response.text) else: verbose_logger.debug("Batch of %s runs successfully created", len(self.log_queue)) except httpx.HTTPStatusError: diff --git a/litellm/integrations/arize/_utils.py b/litellm/integrations/arize/_utils.py index 032f490860d..fe5d235e51b 100644 --- a/litellm/integrations/arize/_utils.py +++ b/litellm/integrations/arize/_utils.py @@ -461,7 +461,7 @@ def set_attributes(span: "Span", kwargs, response_obj, attributes: type[BaseLLMO _set_response_attributes(span=span, response_obj=response_obj_for_attrs) except Exception as e: - verbose_logger.error(f"[Arize/Phoenix] Failed to set OpenInference span attributes: {e}") + verbose_logger.error("[Arize/Phoenix] Failed to set OpenInference span attributes: %s", e) if hasattr(span, "record_exception"): span.record_exception(e) diff --git a/litellm/integrations/arize/arize_phoenix.py b/litellm/integrations/arize/arize_phoenix.py index db698dd6b77..ae8a6994488 100644 --- a/litellm/integrations/arize/arize_phoenix.py +++ b/litellm/integrations/arize/arize_phoenix.py @@ -425,7 +425,7 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore endpoint = "http://localhost:6006/v1/traces" protocol = "otlp_http" verbose_logger.debug( - f"No PHOENIX_COLLECTOR_ENDPOINT found, using default local Phoenix endpoint: {endpoint}" + "No PHOENIX_COLLECTOR_ENDPOINT found, using default local Phoenix endpoint: %s", endpoint ) otlp_auth_headers = None diff --git a/litellm/integrations/arize/arize_phoenix_prompt_manager.py b/litellm/integrations/arize/arize_phoenix_prompt_manager.py index ca74835e167..9985bc20af0 100644 --- a/litellm/integrations/arize/arize_phoenix_prompt_manager.py +++ b/litellm/integrations/arize/arize_phoenix_prompt_manager.py @@ -339,7 +339,7 @@ class ArizePhoenixPromptManager(CustomPromptManagement): # Log error but don't fail the call import litellm - litellm._logging.verbose_proxy_logger.error(f"Error in Arize Phoenix prompt pre_call_hook: {e}") + litellm._logging.verbose_proxy_logger.error("Error in Arize Phoenix prompt pre_call_hook: %s", e) return messages, litellm_params def get_available_prompts(self) -> list[str]: diff --git a/litellm/integrations/azure_sentinel/azure_sentinel.py b/litellm/integrations/azure_sentinel/azure_sentinel.py index e0ed0cd7cf3..29bbac2912a 100644 --- a/litellm/integrations/azure_sentinel/azure_sentinel.py +++ b/litellm/integrations/azure_sentinel/azure_sentinel.py @@ -203,7 +203,7 @@ class AzureSentinelLogger(CustomBatchLogger): await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"Azure Sentinel Layer Error - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Azure Sentinel Layer Error - %s\n%s", e, traceback.format_exc()) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): """ @@ -233,7 +233,7 @@ class AzureSentinelLogger(CustomBatchLogger): await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"Azure Sentinel Layer Error - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Azure Sentinel Layer Error - %s\n%s", e, traceback.format_exc()) async def async_log_audit_log_event(self, audit_log: StandardAuditLogPayload) -> None: """ @@ -256,7 +256,7 @@ class AzureSentinelLogger(CustomBatchLogger): await self.async_send_audit_batch() except Exception as e: - verbose_logger.exception(f"Azure Sentinel Audit Log Layer Error - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Azure Sentinel Audit Log Layer Error - %s\n%s", e, traceback.format_exc()) async def async_send_batch(self): """ @@ -323,7 +323,7 @@ class AzureSentinelLogger(CustomBatchLogger): ) except Exception as e: - verbose_logger.exception(f"Azure Sentinel Error sending batch API - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Azure Sentinel Error sending batch API - %s\n%s", e, traceback.format_exc()) finally: log_queue.clear() diff --git a/litellm/integrations/azure_storage/azure_storage.py b/litellm/integrations/azure_storage/azure_storage.py index d2dd3d37dc7..142a0a56967 100644 --- a/litellm/integrations/azure_storage/azure_storage.py +++ b/litellm/integrations/azure_storage/azure_storage.py @@ -53,7 +53,9 @@ class AzureBlobStorageLogger(CustomBatchLogger): self.log_queue: list[StandardLoggingPayload] = [] super().__init__(**kwargs, flush_lock=self.flush_lock) except Exception as e: - verbose_logger.exception(f"AzureBlobStorageLogger: Got exception on init AzureBlobStorageLogger client {e}") + verbose_logger.exception( + "AzureBlobStorageLogger: Got exception on init AzureBlobStorageLogger client %s", e + ) raise e async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -77,7 +79,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): self.log_queue.append(standard_logging_payload) except Exception as e: - verbose_logger.exception(f"AzureBlobStorageLogger Layer Error - {e}") + verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): """ @@ -99,7 +101,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): self.log_queue.append(standard_logging_payload) except Exception as e: - verbose_logger.exception(f"AzureBlobStorageLogger Layer Error - {e}") + verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e) async def async_send_batch(self): """ @@ -122,7 +124,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): await self.async_upload_payload_to_azure_blob_storage(payload=payload) except Exception as e: - verbose_logger.exception(f"AzureBlobStorageLogger Error sending batch API - {e}") + verbose_logger.exception("AzureBlobStorageLogger Error sending batch API - %s", e) async def async_upload_payload_to_azure_blob_storage(self, payload: StandardLoggingPayload): """ @@ -148,16 +150,16 @@ class AzureBlobStorageLogger(CustomBatchLogger): await self._append_data(async_client, base_url, json_payload) await self._flush_data(async_client, base_url, len(payload_bytes)) - verbose_logger.debug(f"Successfully uploaded log to Azure Blob Storage: {filename}") + verbose_logger.debug("Successfully uploaded log to Azure Blob Storage: %s", filename) except Exception as e: - verbose_logger.exception(f"Error uploading to Azure Blob Storage: {e}") + verbose_logger.exception("Error uploading to Azure Blob Storage: %s", e) raise e async def _create_file(self, client: AsyncHTTPHandler, base_url: str): """Helper method to create the file resource""" try: - verbose_logger.debug(f"Creating file resource at: {base_url}") + verbose_logger.debug("Creating file resource at: %s", base_url) headers = { "x-ms-version": AZURE_STORAGE_MSFT_VERSION, "Content-Length": "0", @@ -167,13 +169,13 @@ class AzureBlobStorageLogger(CustomBatchLogger): response.raise_for_status() verbose_logger.debug("Successfully created file resource") except Exception as e: - verbose_logger.exception(f"Error creating file resource: {e}") + verbose_logger.exception("Error creating file resource: %s", e) raise async def _append_data(self, client: AsyncHTTPHandler, base_url: str, json_payload: str): """Helper method to append data to the file""" try: - verbose_logger.debug(f"Appending data to file: {base_url}") + verbose_logger.debug("Appending data to file: %s", base_url) headers = { "x-ms-version": AZURE_STORAGE_MSFT_VERSION, "Content-Type": "application/json", @@ -187,13 +189,13 @@ class AzureBlobStorageLogger(CustomBatchLogger): response.raise_for_status() verbose_logger.debug("Successfully appended data") except Exception as e: - verbose_logger.exception(f"Error appending data: {e}") + verbose_logger.exception("Error appending data: %s", e) raise async def _flush_data(self, client: AsyncHTTPHandler, base_url: str, position: int): """Helper method to flush the data""" try: - verbose_logger.debug(f"Flushing data at position {position}") + verbose_logger.debug("Flushing data at position %s", position) headers = { "x-ms-version": AZURE_STORAGE_MSFT_VERSION, "Content-Length": "0", @@ -203,7 +205,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): response.raise_for_status() verbose_logger.debug("Successfully flushed data") except Exception as e: - verbose_logger.exception(f"Error flushing data: {e}") + verbose_logger.exception("Error flushing data: %s", e) raise ####### Helper methods to managing Authentication to Azure Storage ####### @@ -227,7 +229,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): ) # Token typically expires in 1 hour self.token_expiry = datetime.now() + timedelta(hours=1) - verbose_logger.debug(f"New token will expire at {self.token_expiry}") + verbose_logger.debug("New token will expire at %s", self.token_expiry) def get_azure_ad_token_from_azure_storage( self, @@ -322,7 +324,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): # check if the directory exists if not await directory_client.exists(): await directory_client.create_directory() - verbose_logger.debug(f"Created directory: {today}") + verbose_logger.debug("Created directory: %s", today) # Create a file client file_name = f"{payload.get('id') or str(uuid.uuid4())}.json" @@ -340,7 +342,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): # Flush the content to finalize the file await file_client.flush_data(position=len(content), offset=0) - verbose_logger.debug(f"Successfully uploaded and wrote to {today}/{file_name}") + verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, file_name) except Exception as e: - verbose_logger.exception(f"Error occurred: {e}") + verbose_logger.exception("Error occurred: %s", e) diff --git a/litellm/integrations/bitbucket/bitbucket_prompt_manager.py b/litellm/integrations/bitbucket/bitbucket_prompt_manager.py index c76466b2f40..3a61c900600 100644 --- a/litellm/integrations/bitbucket/bitbucket_prompt_manager.py +++ b/litellm/integrations/bitbucket/bitbucket_prompt_manager.py @@ -320,7 +320,7 @@ class BitBucketPromptManager(CustomPromptManagement): # Log error but don't fail the call import litellm - litellm._logging.verbose_proxy_logger.error(f"Error in BitBucket prompt pre_call_hook: {e}") + litellm._logging.verbose_proxy_logger.error("Error in BitBucket prompt pre_call_hook: %s", e) return messages, litellm_params def _parse_prompt_to_messages(self, prompt_content: str) -> list[AllMessageValues]: diff --git a/litellm/integrations/braintrust_mock_client.py b/litellm/integrations/braintrust_mock_client.py index e2b732d6e9c..c775a8f3ab8 100644 --- a/litellm/integrations/braintrust_mock_client.py +++ b/litellm/integrations/braintrust_mock_client.py @@ -89,7 +89,7 @@ def _mock_http_handler_post( """Monkey-patched HTTPHandler.post that intercepts Braintrust calls with endpoint-specific responses.""" # Only mock Braintrust API calls if isinstance(url, str) and _is_braintrust_url(url): - verbose_logger.info(f"[BRAINTRUST MOCK] POST to {url}") + verbose_logger.info("[BRAINTRUST MOCK] POST to %s", url) time.sleep(_MOCK_LATENCY_SECONDS) # Return appropriate mock response based on endpoint if "/project" in url: diff --git a/litellm/integrations/cloudzero/cloudzero.py b/litellm/integrations/cloudzero/cloudzero.py index 52b41f74fce..5cab952cfea 100644 --- a/litellm/integrations/cloudzero/cloudzero.py +++ b/litellm/integrations/cloudzero/cloudzero.py @@ -38,7 +38,7 @@ class CloudZeroLogger(CustomLogger): self.connection_id = connection_id or os.getenv("CLOUDZERO_CONNECTION_ID") self.timezone = timezone or os.getenv("CLOUDZERO_TIMEZONE", "UTC") verbose_logger.debug( - f"CloudZero Logger initialized with connection ID: {self.connection_id}, timezone: {self.timezone}" + "CloudZero Logger initialized with connection ID: %s, timezone: %s", self.connection_id, self.timezone ) async def initialize_cloudzero_export_job(self): @@ -130,7 +130,7 @@ class CloudZeroLogger(CustomLogger): verbose_logger.debug("CloudZero Logger: No usage data found to export") return - verbose_logger.debug(f"CloudZero Logger: Processing {len(data)} records") + verbose_logger.debug("CloudZero Logger: Processing %s records", len(data)) # Transform data to CloudZero CBF format transformer = CBFTransformer() @@ -147,13 +147,13 @@ class CloudZeroLogger(CustomLogger): user_timezone=self.timezone, ) - verbose_logger.debug(f"CloudZero Logger: Transmitting {len(cbf_data)} records to CloudZero") + verbose_logger.debug("CloudZero Logger: Transmitting %s records to CloudZero", len(cbf_data)) streamer.send_batched(cbf_data, operation=operation) - verbose_logger.debug(f"CloudZero Logger: Successfully exported {len(cbf_data)} records to CloudZero") + verbose_logger.debug("CloudZero Logger: Successfully exported %s records to CloudZero", len(cbf_data)) except Exception as e: - verbose_logger.error(f"CloudZero Logger: Error exporting usage data: {e}") + verbose_logger.error("CloudZero Logger: Error exporting usage data: %s", e) raise async def dry_run_export_usage_data(self, limit: int | None = 10000): @@ -191,7 +191,7 @@ class CloudZeroLogger(CustomLogger): }, } - verbose_logger.debug(f"CloudZero Dry Run: Processing {len(data)} records...") + verbose_logger.debug("CloudZero Dry Run: Processing %s records...", len(data)) # Convert usage data to dict format for response usage_data_sample = data.head(50).to_dicts() # Return first 50 rows @@ -229,7 +229,7 @@ class CloudZeroLogger(CustomLogger): ) total_tokens = sum(record.get("usage/amount", 0) for record in cbf_data_dict) - verbose_logger.debug(f"CloudZero Logger: Dry run completed for {len(cbf_data)} records") + verbose_logger.debug("CloudZero Logger: Dry run completed for %s records", len(cbf_data)) return { "usage_data": usage_data_sample, @@ -244,8 +244,8 @@ class CloudZeroLogger(CustomLogger): } except Exception as e: - verbose_logger.error(f"CloudZero Logger: Error in dry run export: {e}") - verbose_logger.error(f"CloudZero Dry Run Error: {e}") + verbose_logger.error("CloudZero Logger: Error in dry run export: %s", e) + verbose_logger.error("CloudZero Dry Run Error: %s", e) raise def _display_cbf_data_on_screen(self, cbf_data): diff --git a/litellm/integrations/custom_batch_logger.py b/litellm/integrations/custom_batch_logger.py index 98a8e4ba739..7559bc83cfa 100644 --- a/litellm/integrations/custom_batch_logger.py +++ b/litellm/integrations/custom_batch_logger.py @@ -47,7 +47,7 @@ class CustomBatchLogger(CustomLogger): async def periodic_flush(self): while True: await asyncio.sleep(self.flush_interval) - verbose_logger.debug(f"CustomLogger periodic flush after {self.flush_interval} seconds") + verbose_logger.debug("CustomLogger periodic flush after %s seconds", self.flush_interval) await self.flush_queue() async def flush_queue(self): diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 743c539c36f..d9d65375ea8 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -864,7 +864,7 @@ class CustomGuardrail(CustomLogger): if premium_user is not True: verbose_logger.warning( - f"Trying to use premium guardrail without premium user {CommonProxyErrors.not_premium_user.value}" + "Trying to use premium guardrail without premium user %s", CommonProxyErrors.not_premium_user.value ) return False return True @@ -1028,7 +1028,7 @@ class CustomGuardrail(CustomLogger): else: guardrail_response = "allow" - verbose_logger.debug(f"Guardrail response: {response}") + verbose_logger.debug("Guardrail response: %s", response) self.add_standard_logging_guardrail_information_to_request_data( guardrail_json_response=guardrail_response, diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 9915224ba09..9df0cf6e84d 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -915,19 +915,19 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac for callback_obj in all_callbacks: if hasattr(callback_obj, "increment_callback_logging_failure"): - verbose_logger.debug(f"Incrementing callback failure metric for {callback_name}") + verbose_logger.debug("Incrementing callback failure metric for %s", callback_name) callback_obj.increment_callback_logging_failure(callback_name=callback_name) # type: ignore return verbose_logger.debug( - f"No callback with increment_callback_logging_failure method found for {callback_name}. " - "Ensure 'prometheus' is in your callbacks config." + "No callback with increment_callback_logging_failure method found for %s. Ensure 'prometheus' is in your callbacks config.", + callback_name, ) except Exception as e: from litellm._logging import verbose_logger - verbose_logger.debug(f"Error in handle_callback_failure for {callback_name}: {e}") + verbose_logger.debug("Error in handle_callback_failure for %s: %s", callback_name, e) async def _strip_base64_from_messages( self, @@ -946,7 +946,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac """ raw_messages: Any = payload.get("messages", []) messages: list[Any] = raw_messages if isinstance(raw_messages, list) else [] - verbose_logger.debug(f"[CustomLogger] Stripping base64 from {len(messages)} messages") + verbose_logger.debug("[CustomLogger] Stripping base64 from %s messages", len(messages)) if messages: payload["messages"] = self._process_messages(messages=messages, max_depth=max_depth) @@ -958,7 +958,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac if isinstance(content, list): total_items += len(content) - verbose_logger.debug(f"[CustomLogger] Completed base64 strip; retained {total_items} content items") + verbose_logger.debug("[CustomLogger] Completed base64 strip; retained %s content items", total_items) return payload def _strip_base64_from_messages_sync( @@ -978,7 +978,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac """ raw_messages: Any = payload.get("messages", []) messages: list[Any] = raw_messages if isinstance(raw_messages, list) else [] - verbose_logger.debug(f"[CustomLogger] Stripping base64 from {len(messages)} messages") + verbose_logger.debug("[CustomLogger] Stripping base64 from %s messages", len(messages)) if messages: payload["messages"] = self._process_messages(messages=messages, max_depth=max_depth) @@ -990,7 +990,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac if isinstance(content, list): total_items += len(content) - verbose_logger.debug(f"[CustomLogger] Completed base64 strip; retained {total_items} content items") + verbose_logger.debug("[CustomLogger] Completed base64 strip; retained %s content items", total_items) return payload def _redact_base64( @@ -1001,12 +1001,12 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac ) -> Any: """Recursively redact inline base64 from any nested structure with a max recursion depth limit.""" if depth > max_depth: - verbose_logger.warning(f"[CustomLogger] Max recursion depth {max_depth} reached while redacting base64") + verbose_logger.warning("[CustomLogger] Max recursion depth %s reached while redacting base64", max_depth) return "[MAX_DEPTH_REACHED]" if isinstance(value, str): if _BASE64_INLINE_PATTERN.search(value): - verbose_logger.debug(f"[CustomLogger] Redacted inline base64 string: {value[:40]}...") + verbose_logger.debug("[CustomLogger] Redacted inline base64 string: %s...", value[:40]) return _BASE64_INLINE_PATTERN.sub("[BASE64_REDACTED]", value) return value diff --git a/litellm/integrations/custom_secret_manager.py b/litellm/integrations/custom_secret_manager.py index 8cb7f02b798..e59842d409a 100644 --- a/litellm/integrations/custom_secret_manager.py +++ b/litellm/integrations/custom_secret_manager.py @@ -237,7 +237,7 @@ class CustomSecretManager(BaseSecretManager): Returns: True if the secret manager is healthy, False otherwise """ - verbose_logger.debug(f"Health check not implemented for {self.secret_manager_name}") + verbose_logger.debug("Health check not implemented for %s", self.secret_manager_name) return True def __repr__(self) -> str: diff --git a/litellm/integrations/datadog/datadog.py b/litellm/integrations/datadog/datadog.py index fa14e1fa459..ce6dda96820 100644 --- a/litellm/integrations/datadog/datadog.py +++ b/litellm/integrations/datadog/datadog.py @@ -171,7 +171,7 @@ class DataDogLogger( batch_size=_resolve_dd_batch_size(), ) except Exception as e: - verbose_logger.exception(f"Datadog: Got exception on init Datadog client {e}") + verbose_logger.exception("Datadog: Got exception on init Datadog client %s", e) raise e def _get_datadog_params(self) -> dict: @@ -210,7 +210,7 @@ class DataDogLogger( self.DD_API_KEY = dd_api_key or ( os.getenv("DD_API_KEY") if allow_env_credentials else None ) # Optional when using agent - verbose_logger.debug(f"Datadog: Using DD Agent at {self.intake_url}") + verbose_logger.debug("Datadog: Using DD Agent at %s", self.intake_url) def _configure_dd_direct_api( self, @@ -257,7 +257,7 @@ class DataDogLogger( await self._log_async_event(kwargs, response_obj, start_time, end_time) except Exception as e: - verbose_logger.exception(f"Datadog Layer Error - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Datadog Layer Error - %s\n%s", e, traceback.format_exc()) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): try: @@ -265,7 +265,7 @@ class DataDogLogger( await self._log_async_event(kwargs, response_obj, start_time, end_time) except Exception as e: - verbose_logger.exception(f"Datadog Layer Error - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Datadog Layer Error - %s\n%s", e, traceback.format_exc()) async def async_post_call_failure_hook( self, @@ -340,7 +340,7 @@ class DataDogLogger( if len(self.log_queue) >= self.batch_size: await self.flush_queue() except Exception as e: - verbose_logger.exception(f"Datadog: async_post_call_failure_hook - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Datadog: async_post_call_failure_hook - %s\n%s", e, traceback.format_exc()) return None async def async_send_batch(self): @@ -376,11 +376,11 @@ class DataDogLogger( self.log_queue = undelivered + self.log_queue if self.is_mock_mode: - verbose_logger.debug(f"[DATADOG MOCK] Batch of {len(batch_to_send)} events successfully mocked") + verbose_logger.debug("[DATADOG MOCK] Batch of %s events successfully mocked", len(batch_to_send)) except Exception as e: self.log_queue = batch_to_send + self.log_queue - verbose_logger.exception(f"Datadog Error sending batch API - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Datadog Error sending batch API - %s\n%s", e, traceback.format_exc()) async def _send_with_413_split(self, batch: list) -> list: """ @@ -411,7 +411,7 @@ class DataDogLogger( if isinstance(e, MaskedHTTPStatusError) and e.status_code == 413: response = e.response else: - verbose_logger.exception(f"Datadog Error sending batch API - {e}") + verbose_logger.exception("Datadog Error sending batch API - %s", e) return self._undelivered(chunk, pending) if response.status_code == 413: @@ -515,7 +515,7 @@ class DataDogLogger( ) except Exception as e: - verbose_logger.exception(f"Datadog Layer Error - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Datadog Layer Error - %s\n%s", e, traceback.format_exc()) async def _log_async_event(self, kwargs, response_obj, start_time, end_time): dd_payload = self.create_datadog_logging_payload( @@ -526,7 +526,7 @@ class DataDogLogger( ) self.log_queue.append(dd_payload) - verbose_logger.debug(f"Datadog, event added to queue. Will flush in {self.flush_interval} seconds...") + verbose_logger.debug("Datadog, event added to queue. Will flush in %s seconds...", self.flush_interval) if len(self.log_queue) >= self.batch_size: await self.flush_queue() @@ -653,7 +653,7 @@ class DataDogLogger( self.log_queue.append(_dd_payload) except Exception as e: - verbose_logger.exception(f"Datadog: Logger - Exception in async_service_failure_hook: {e}") + verbose_logger.exception("Datadog: Logger - Exception in async_service_failure_hook: %s", e) async def async_service_success_hook( self, @@ -692,7 +692,7 @@ class DataDogLogger( self.log_queue.append(_dd_payload) except Exception as e: - verbose_logger.exception(f"Datadog: Logger - Exception in async_service_failure_hook: {e}") + verbose_logger.exception("Datadog: Logger - Exception in async_service_failure_hook: %s", e) def _create_v0_logging_payload( self, diff --git a/litellm/integrations/datadog/datadog_cost_management.py b/litellm/integrations/datadog/datadog_cost_management.py index da45f94f02b..21a289877c2 100644 --- a/litellm/integrations/datadog/datadog_cost_management.py +++ b/litellm/integrations/datadog/datadog_cost_management.py @@ -84,7 +84,7 @@ class DatadogCostManagementLogger(CustomBatchLogger): await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"Datadog Cost Management: Error in async_log_success_event: {e}") + verbose_logger.exception("Datadog Cost Management: Error in async_log_success_event: %s", e) async def async_send_batch(self): if not self.log_queue: @@ -104,7 +104,7 @@ class DatadogCostManagementLogger(CustomBatchLogger): await self._upload_to_datadog(aggregated_entries) except Exception as e: self.log_queue = batch_to_send + self.log_queue - verbose_logger.exception(f"Datadog Cost Management: Error in async_send_batch: {e}") + verbose_logger.exception("Datadog Cost Management: Error in async_send_batch: %s", e) def _aggregate_costs(self, logs: list[StandardLoggingPayload]) -> list[DatadogFOCUSCostEntry]: """ @@ -159,7 +159,7 @@ class DatadogCostManagementLogger(CustomBatchLogger): aggregator[key]["BilledCost"] += cost except Exception as e: - verbose_logger.warning(f"Error processing log for cost aggregation: {e}") + verbose_logger.warning("Error processing log for cost aggregation: %s", e) continue return list(aggregator.values()) @@ -254,5 +254,5 @@ class DatadogCostManagementLogger(CustomBatchLogger): response.raise_for_status() verbose_logger.debug( - f"Datadog Cost Management: Uploaded {len(payload)} cost entries. Status: {response.status_code}" + "Datadog Cost Management: Uploaded %s cost entries. Status: %s", len(payload), response.status_code ) diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index 02e1affd361..8d7ed415315 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -89,7 +89,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): kwargs.update(dict_datadog_llm_obs_params) CustomBatchLogger.__init__(self, **kwargs, flush_lock=self.flush_lock) except Exception as e: - verbose_logger.exception(f"DataDogLLMObs: Error initializing - {e}") + verbose_logger.exception("DataDogLLMObs: Error initializing - %s", e) raise e def _configure_dd_agent(self, dd_agent_host: str): @@ -103,7 +103,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): agent_port = os.getenv("LITELLM_DD_LLM_OBS_PORT", "8126") self.DD_SITE = "localhost" # Not used for URL construction in agent mode self.intake_url = f"http://{dd_agent_host}:{agent_port}/api/intake/llm-obs/v1/trace/spans" - verbose_logger.debug(f"DataDogLLMObs: Using DD Agent at {self.intake_url}") + verbose_logger.debug("DataDogLLMObs: Using DD Agent at %s", self.intake_url) def _configure_dd_direct_api(self): """ @@ -137,34 +137,34 @@ class DataDogLLMObsLogger(CustomBatchLogger): async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): try: - verbose_logger.debug(f"DataDogLLMObs: Logging success event for model {kwargs.get('model', 'unknown')}") + verbose_logger.debug("DataDogLLMObs: Logging success event for model %s", kwargs.get("model", "unknown")) payload = self.create_llm_obs_payload(kwargs, start_time, end_time) - verbose_logger.debug(f"DataDogLLMObs: Payload: {payload}") + verbose_logger.debug("DataDogLLMObs: Payload: %s", payload) self.log_queue.append(payload) if len(self.log_queue) >= self.batch_size: await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"DataDogLLMObs: Error logging success event - {e}") + verbose_logger.exception("DataDogLLMObs: Error logging success event - %s", e) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): try: - verbose_logger.debug(f"DataDogLLMObs: Logging failure event for model {kwargs.get('model', 'unknown')}") + verbose_logger.debug("DataDogLLMObs: Logging failure event for model %s", kwargs.get("model", "unknown")) payload = self.create_llm_obs_payload(kwargs, start_time, end_time) - verbose_logger.debug(f"DataDogLLMObs: Payload: {payload}") + verbose_logger.debug("DataDogLLMObs: Payload: %s", payload) self.log_queue.append(payload) if len(self.log_queue) >= self.batch_size: await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"DataDogLLMObs: Error logging failure event - {e}") + verbose_logger.exception("DataDogLLMObs: Error logging failure event - %s", e) async def async_send_batch(self): try: if not self.log_queue: return - verbose_logger.debug(f"DataDogLLMObs: Flushing {len(self.log_queue)} events") + verbose_logger.debug("DataDogLLMObs: Flushing %s events", len(self.log_queue)) if self.is_mock_mode: verbose_logger.debug("[DATADOG MOCK] Mock mode enabled - API calls will be intercepted") @@ -207,14 +207,14 @@ class DataDogLLMObsLogger(CustomBatchLogger): ) if self.is_mock_mode: - verbose_logger.debug(f"[DATADOG MOCK] Batch of {len(self.log_queue)} events successfully mocked") + verbose_logger.debug("[DATADOG MOCK] Batch of %s events successfully mocked", len(self.log_queue)) else: - verbose_logger.debug(f"DataDogLLMObs: Successfully sent batch - status_code: {response.status_code}") + verbose_logger.debug("DataDogLLMObs: Successfully sent batch - status_code: %s", response.status_code) self.log_queue.clear() except httpx.HTTPStatusError as e: - verbose_logger.exception(f"DataDogLLMObs: Error sending batch - {e.response.text}") + verbose_logger.exception("DataDogLLMObs: Error sending batch - %s", e.response.text) except Exception as e: - verbose_logger.exception(f"DataDogLLMObs: Error sending batch - {e}") + verbose_logger.exception("DataDogLLMObs: Error sending batch - %s", e) def create_llm_obs_payload(self, kwargs: dict, start_time: datetime, end_time: datetime) -> LLMObsPayload: standard_logging_payload: StandardLoggingPayload | None = kwargs.get("standard_logging_object") @@ -613,7 +613,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): try: spend_metrics["user_api_key_spend"] = float(user_api_key_spend) except (ValueError, TypeError): - verbose_logger.debug(f"Invalid user_api_key_spend value: {user_api_key_spend}") + verbose_logger.debug("Invalid user_api_key_spend value: %s", user_api_key_spend) # API key budget reset datetime user_api_key_budget_reset_at = metadata.get("user_api_key_budget_reset_at") @@ -640,10 +640,10 @@ class DataDogLLMObsLogger(CustomBatchLogger): spend_metrics["user_api_key_budget_reset_at"] = iso_string # Debug logging to verify the conversion - verbose_logger.debug(f"Converted budget_reset_at to ISO format: {iso_string}") + verbose_logger.debug("Converted budget_reset_at to ISO format: %s", iso_string) except Exception as e: - verbose_logger.debug(f"Error processing budget reset datetime: {e}") - verbose_logger.debug(f"Original value: {user_api_key_budget_reset_at}") + verbose_logger.debug("Error processing budget reset datetime: %s", e) + verbose_logger.debug("Original value: %s", user_api_key_budget_reset_at) return spend_metrics @@ -707,7 +707,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): kv_pairs[f"tool_calls.{idx}.function.arguments"] = json.dumps(function_arguments) except (KeyError, TypeError, ValueError) as e: - verbose_logger.debug(f"DataDogLLMObs: Error processing tool call {idx}: {e}") + verbose_logger.debug("DataDogLLMObs: Error processing tool call %s: %s", idx, e) continue return kv_pairs @@ -747,6 +747,6 @@ class DataDogLLMObsLogger(CustomBatchLogger): tool_call_metadata[f"output_{key}"] = value except Exception as e: - verbose_logger.debug(f"DataDogLLMObs: Error extracting tool call metadata: {e}") + verbose_logger.debug("DataDogLLMObs: Error extracting tool call metadata: %s", e) return tool_call_metadata diff --git a/litellm/integrations/datadog/datadog_metrics.py b/litellm/integrations/datadog/datadog_metrics.py index 9fb86bfb125..c33c44e4249 100644 --- a/litellm/integrations/datadog/datadog_metrics.py +++ b/litellm/integrations/datadog/datadog_metrics.py @@ -180,7 +180,7 @@ class DatadogMetricsLogger(CustomBatchLogger): await self.flush_queue() except Exception as e: - verbose_logger.exception(f"Datadog Metrics: Error in async_log_success_event: {e}") + verbose_logger.exception("Datadog Metrics: Error in async_log_success_event: %s", e) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): try: @@ -202,7 +202,7 @@ class DatadogMetricsLogger(CustomBatchLogger): await self.flush_queue() except Exception as e: - verbose_logger.exception(f"Datadog Metrics: Error in async_log_failure_event: {e}") + verbose_logger.exception("Datadog Metrics: Error in async_log_failure_event: %s", e) async def async_send_batch(self): if not self.log_queue: @@ -214,7 +214,7 @@ class DatadogMetricsLogger(CustomBatchLogger): try: await self._upload_to_datadog(payload_data) except Exception as e: - verbose_logger.exception(f"Datadog Metrics: Error in async_send_batch: {e}") + verbose_logger.exception("Datadog Metrics: Error in async_send_batch: %s", e) raise async def _upload_to_datadog(self, payload: DatadogMetricsPayload): @@ -242,7 +242,7 @@ class DatadogMetricsLogger(CustomBatchLogger): response.raise_for_status() verbose_logger.debug( - f"Datadog Metrics: Uploaded {len(payload['series'])} metric points. Status: {response.status_code}" + "Datadog Metrics: Uploaded %s metric points. Status: %s", len(payload["series"]), response.status_code ) async def async_health_check(self) -> IntegrationHealthCheckStatus: diff --git a/litellm/integrations/deepeval/api.py b/litellm/integrations/deepeval/api.py index adca8928df4..512c74e035c 100644 --- a/litellm/integrations/deepeval/api.py +++ b/litellm/integrations/deepeval/api.py @@ -23,9 +23,9 @@ def log_retry_error(details): exception = details.get("exception") tries = details.get("tries") if exception: - logging.error(f"Confident AI Error: {exception}. Retrying: {tries} time(s)...") + logging.error("Confident AI Error: %s. Retrying: %s time(s)...", exception, tries) else: - logging.error(f"Retrying: {tries} time(s)...") + logging.error("Retrying: %s time(s)...", tries) class HttpMethods(Enum): diff --git a/litellm/integrations/gcs_bucket/gcs_bucket.py b/litellm/integrations/gcs_bucket/gcs_bucket.py index b5b3d4e81a3..e402e4962b7 100644 --- a/litellm/integrations/gcs_bucket/gcs_bucket.py +++ b/litellm/integrations/gcs_bucket/gcs_bucket.py @@ -76,7 +76,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): await self.log_queue.put(GCSLogQueueItem(payload=logging_payload, kwargs=kwargs, response_obj=response_obj)) except Exception as e: - verbose_logger.exception(f"GCS Bucket logging error: {e}") + verbose_logger.exception("GCS Bucket logging error: %s", e) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): try: @@ -95,7 +95,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): await self.log_queue.put(GCSLogQueueItem(payload=logging_payload, kwargs=kwargs, response_obj=response_obj)) except Exception as e: - verbose_logger.exception(f"GCS Bucket logging error: {e}") + verbose_logger.exception("GCS Bucket logging error: %s", e) def _drain_queue_batch(self) -> list[GCSLogQueueItem]: """ @@ -218,7 +218,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): except Exception as e: success_count = 0 error_count = len(items) - verbose_logger.exception(f"GCS Bucket error logging batch payload to GCS bucket: {e}") + verbose_logger.exception("GCS Bucket error logging batch payload to GCS bucket: %s", e) return (success_count, error_count) async def _send_individual_logs(self, items: list[GCSLogQueueItem]) -> None: @@ -255,7 +255,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): logging_payload=item["payload"], ) except Exception as e: - verbose_logger.exception(f"GCS Bucket error logging individual payload to GCS bucket: {e}") + verbose_logger.exception("GCS Bucket error logging individual payload to GCS bucket: %s", e) async def async_send_batch(self): """ @@ -336,7 +336,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): loaded_response = json.loads(response) return loaded_response except Exception as e: - verbose_logger.debug(f"Failed to fetch payload for date {date_str}: {e}") + verbose_logger.debug("Failed to fetch payload for date %s: %s", date_str, e) continue return None @@ -370,7 +370,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): """ while True: await asyncio.sleep(self.flush_interval) - verbose_logger.debug(f"GCS Bucket periodic flush after {self.flush_interval} seconds") + verbose_logger.debug("GCS Bucket periodic flush after %s seconds", self.flush_interval) await self.flush_queue() async def async_health_check(self) -> IntegrationHealthCheckStatus: diff --git a/litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py b/litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py index 86cf8617dd5..20e89c0647b 100644 --- a/litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py +++ b/litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py @@ -45,7 +45,7 @@ async def _mock_async_handler_get(self, url, params=None, headers=None, follow_r """Monkey-patched AsyncHTTPHandler.get that intercepts GCS calls.""" # Only mock GCS API calls if isinstance(url, str) and "storage.googleapis.com" in url: - verbose_logger.info(f"[GCS MOCK] GET to {url}") + verbose_logger.info("[GCS MOCK] GET to %s", url) await asyncio.sleep(_MOCK_LATENCY_SECONDS) # Return a minimal but valid StandardLoggingPayload JSON string as bytes # This matches what GCS returns when downloading with ?alt=media @@ -117,7 +117,7 @@ async def _mock_async_handler_delete( """Monkey-patched AsyncHTTPHandler.delete that intercepts GCS calls.""" # Only mock GCS API calls if isinstance(url, str) and "storage.googleapis.com" in url: - verbose_logger.info(f"[GCS MOCK] DELETE to {url}") + verbose_logger.info("[GCS MOCK] DELETE to %s", url) await asyncio.sleep(_MOCK_LATENCY_SECONDS) # DELETE returns 204 No Content with empty body (not JSON) return MockResponse( diff --git a/litellm/integrations/gcs_pubsub/pub_sub.py b/litellm/integrations/gcs_pubsub/pub_sub.py index b43e7626b77..d915c7341df 100644 --- a/litellm/integrations/gcs_pubsub/pub_sub.py +++ b/litellm/integrations/gcs_pubsub/pub_sub.py @@ -132,7 +132,7 @@ class GcsPubSubLogger(CustomBatchLogger): await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"PubSub Layer Error - {e}\n{traceback.format_exc()}") + verbose_logger.exception("PubSub Layer Error - %s\n%s", e, traceback.format_exc()) async def async_send_batch(self): """ @@ -142,13 +142,13 @@ class GcsPubSubLogger(CustomBatchLogger): if not self.log_queue: return - verbose_logger.debug(f"PubSub - about to flush {len(self.log_queue)} events") + verbose_logger.debug("PubSub - about to flush %s events", len(self.log_queue)) for message in self.log_queue: await self.publish_message(message) except Exception as e: - verbose_logger.exception(f"PubSub Error sending batch - {e}\n{traceback.format_exc()}") + verbose_logger.exception("PubSub Error sending batch - %s\n%s", e, traceback.format_exc()) finally: self.log_queue.clear() diff --git a/litellm/integrations/generic_api/generic_api_callback.py b/litellm/integrations/generic_api/generic_api_callback.py index c7f2661a5ad..3e02826ee5d 100644 --- a/litellm/integrations/generic_api/generic_api_callback.py +++ b/litellm/integrations/generic_api/generic_api_callback.py @@ -42,7 +42,7 @@ def load_compatible_callbacks() -> dict: with open(json_path, "r") as f: return json.load(f) except Exception as e: - verbose_logger.warning(f"Error loading generic_api_compatible_callbacks.json: {e}") + verbose_logger.warning("Error loading generic_api_compatible_callbacks.json: %s", e) return {} @@ -124,7 +124,7 @@ class GenericAPILogger(CustomBatchLogger): ######################################################### if callback_name: if is_callback_compatible(callback_name): - verbose_logger.debug(f"Loading configuration for callback: {callback_name}") + verbose_logger.debug("Loading configuration for callback: %s", callback_name) callback_config = get_callback_config(callback_name) # Use config from JSON if not explicitly provided @@ -145,7 +145,7 @@ class GenericAPILogger(CustomBatchLogger): log_format = callback_config["log_format"] else: verbose_logger.warning( - f"callback_name '{callback_name}' not found in generic_api_compatible_callbacks.json" + "callback_name '%s' not found in generic_api_compatible_callbacks.json", callback_name ) ######################################################### @@ -177,7 +177,12 @@ class GenericAPILogger(CustomBatchLogger): self.log_format: LOG_FORMAT_TYPES = log_format or "json_array" verbose_logger.debug( - f"in init GenericAPILogger, callback_name: {self.callback_name}, endpoint {self.endpoint}, headers {self.headers}, event_types: {self.event_types}, log_format: {self.log_format}" + "in init GenericAPILogger, callback_name: %s, endpoint %s, headers %s, event_types: %s, log_format: %s", + self.callback_name, + self.endpoint, + self.headers, + self.event_types, + self.log_format, ) ######################################################### @@ -214,7 +219,7 @@ class GenericAPILogger(CustomBatchLogger): key, value = item.split("=", 1) headers_dict[key.strip()] = value.strip() except Exception as e: - verbose_logger.warning(f"Error parsing headers from environment variables: {e}") + verbose_logger.warning("Error parsing headers from environment variables: %s", e) # 2. Update with litellm generic headers if available if litellm.generic_logger_headers: @@ -308,7 +313,7 @@ class GenericAPILogger(CustomBatchLogger): await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"Generic API Logger Error - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Generic API Logger Error - %s\n%s", e, traceback.format_exc()) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): """ @@ -339,7 +344,7 @@ class GenericAPILogger(CustomBatchLogger): await self.async_send_batch() except Exception as e: - verbose_logger.exception(f"Generic API Logger Error - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Generic API Logger Error - %s\n%s", e, traceback.format_exc()) async def async_send_batch(self): """ @@ -355,7 +360,7 @@ class GenericAPILogger(CustomBatchLogger): return verbose_logger.debug( - f"Generic API Logger - about to flush {len(self.log_queue)} events in '{self.log_format}' format" + "Generic API Logger - about to flush %s events in '%s' format", len(self.log_queue), self.log_format ) if self.log_format == "single": @@ -371,11 +376,13 @@ class GenericAPILogger(CustomBatchLogger): # Log results for idx, result in enumerate(responses): if isinstance(result, Exception): - verbose_logger.exception(f"Generic API Logger - Error sending log {idx}: {result}") + verbose_logger.exception("Generic API Logger - Error sending log %s: %s", idx, result) else: # result is a Response object verbose_logger.debug( - f"Generic API Logger - sent log {idx}, status: {result.status_code}" # type: ignore + "Generic API Logger - sent log %s, status: %s", + idx, + result.status_code, # type: ignore ) else: # Format the payload based on log_format @@ -390,12 +397,14 @@ class GenericAPILogger(CustomBatchLogger): response = await self._post_with_retries(data=data) verbose_logger.debug( - f"Generic API Logger - sent batch to {self.endpoint}, " - f"status: {response.status_code}, format: {self.log_format}" + "Generic API Logger - sent batch to %s, status: %s, format: %s", + self.endpoint, + response.status_code, + self.log_format, ) except Exception as e: - verbose_logger.exception(f"Generic API Logger Error sending batch - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Generic API Logger Error sending batch - %s\n%s", e, traceback.format_exc()) finally: self.log_queue.clear() @@ -405,7 +414,7 @@ class GenericAPILogger(CustomBatchLogger): Returns a dict of the payload to send to the Generic API Endpoint """ - verbose_logger.debug(f"GenericAPILogger Logging - Enters logging function for model {kwargs}") + verbose_logger.debug("GenericAPILogger Logging - Enters logging function for model %s", kwargs) # construct payload to send custom logger # follows the same params as langfuse.py diff --git a/litellm/integrations/gitlab/gitlab_prompt_manager.py b/litellm/integrations/gitlab/gitlab_prompt_manager.py index 54e0a3ad02e..9fda4ebcc19 100644 --- a/litellm/integrations/gitlab/gitlab_prompt_manager.py +++ b/litellm/integrations/gitlab/gitlab_prompt_manager.py @@ -379,7 +379,7 @@ class GitLabPromptManager(CustomPromptManagement): except Exception as e: import litellm - litellm._logging.verbose_proxy_logger.error(f"Error in GitLab prompt pre_call_hook: {e}") + litellm._logging.verbose_proxy_logger.error("Error in GitLab prompt pre_call_hook: %s", e) return messages, litellm_params def _parse_prompt_to_messages(self, prompt_content: str) -> list[AllMessageValues]: diff --git a/litellm/integrations/lago.py b/litellm/integrations/lago.py index 3186f1bf58b..000d3e2c79a 100644 --- a/litellm/integrations/lago.py +++ b/litellm/integrations/lago.py @@ -117,7 +117,7 @@ class LagoLogger(CustomLogger): } } - verbose_logger.debug(f"\033[91mLogged Lago Object:\n{returned_val}\033[0m\n") + verbose_logger.debug("\x1b[91mLogged Lago Object:\n%s\x1b[0m\n", returned_val) return returned_val def log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -149,7 +149,7 @@ class LagoLogger(CustomLogger): except Exception as e: error_response = getattr(e, "response", None) if error_response is not None and hasattr(error_response, "text"): - verbose_logger.debug(f"\nError Message: {error_response.text}") + verbose_logger.debug("\nError Message: %s", error_response.text) raise e async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -184,8 +184,8 @@ class LagoLogger(CustomLogger): response.raise_for_status() - verbose_logger.debug(f"Logged Lago Object: {response.text}") + verbose_logger.debug("Logged Lago Object: %s", response.text) except Exception as e: if response is not None and hasattr(response, "text"): - verbose_logger.debug(f"\nError Message: {response.text}") + verbose_logger.debug("\nError Message: %s", response.text) raise e diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 2dab1874c01..d10c7a699f5 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -199,7 +199,7 @@ class LangFuseLogger: ) langfuse_client = Langfuse(**parameters) litellm.initialized_langfuse_clients += 1 - verbose_logger.debug(f"Created langfuse client number {litellm.initialized_langfuse_clients}") + verbose_logger.debug("Created langfuse client number %s", litellm.initialized_langfuse_clients) return langfuse_client @staticmethod @@ -226,9 +226,9 @@ class LangFuseLogger: if metadata_param_key.startswith("langfuse_"): trace_param_key = metadata_param_key.replace("langfuse_", "", 1) if trace_param_key in metadata: - verbose_logger.warning(f"Overwriting Langfuse `{trace_param_key}` from request header") + verbose_logger.warning("Overwriting Langfuse `%s` from request header", trace_param_key) else: - verbose_logger.debug(f"Found Langfuse `{trace_param_key}` in request header") + verbose_logger.debug("Found Langfuse `%s` in request header", trace_param_key) metadata[trace_param_key] = proxy_headers.get(metadata_param_key) return metadata @@ -256,7 +256,7 @@ class LangFuseLogger: Logs a success or error event on Langfuse """ try: - verbose_logger.debug(f"Langfuse Logging - Enters logging function for model {kwargs}") + verbose_logger.debug("Langfuse Logging - Enters logging function for model %s", kwargs) # set default values for input/output for langfuse logging input = None @@ -295,7 +295,7 @@ class LangFuseLogger: level=level, status_message=status_message, ) - verbose_logger.debug(f"OUTPUT IN LANGFUSE: {output}; original: {response_obj}") + verbose_logger.debug("OUTPUT IN LANGFUSE: %s; original: %s", output, response_obj) trace_id = None generation_id = None if self._is_langfuse_v2(): @@ -325,12 +325,12 @@ class LangFuseLogger: input=input, response_obj=response_obj, ) - verbose_logger.debug(f"Langfuse Layer Logging - final response object: {response_obj}") + verbose_logger.debug("Langfuse Layer Logging - final response object: %s", response_obj) verbose_logger.info("Langfuse Layer Logging - logging success") return {"trace_id": trace_id, "generation_id": generation_id} except Exception as e: - verbose_logger.exception(f"Langfuse Layer Error(): Exception occured - {e}") + verbose_logger.exception("Langfuse Layer Error(): Exception occured - %s", e) return {"trace_id": None, "generation_id": None} def _get_langfuse_input_output_content( @@ -625,7 +625,7 @@ class LangFuseLogger: trace_params["metadata"] = {"metadata_passed_to_litellm": metadata} cost = kwargs.get("response_cost", None) - verbose_logger.debug(f"trace: {cost}") + verbose_logger.debug("trace: %s", cost) clean_metadata["litellm_response_cost"] = cost if standard_logging_object is not None: @@ -780,12 +780,13 @@ class LangFuseLogger: if hasattr(generation_client, "trace_id") and generation_client.trace_id: if generation_client.trace_id != trace_id: verbose_logger.warning( - f"Langfuse trace_id mismatch: set {trace_id}, but langfuse returned {generation_client.trace_id}. " - "Using our intended trace_id for consistency." + "Langfuse trace_id mismatch: set %s, but langfuse returned %s. Using our intended trace_id for consistency.", + trace_id, + generation_client.trace_id, ) return trace_id, generation_id except Exception: - verbose_logger.error(f"Langfuse Layer Error - {traceback.format_exc()}") + verbose_logger.error("Langfuse Layer Error - %s", traceback.format_exc()) return None, None @staticmethod @@ -902,7 +903,7 @@ class LangFuseLogger: # For other types, try to apply the function directly return masking_function(data) except Exception as e: - verbose_logger.warning(f"Failed to apply masking function: {e}. Returning original data.") + verbose_logger.warning("Failed to apply masking function: %s. Returning original data.", e) return data @staticmethod @@ -966,7 +967,7 @@ class LangFuseLogger: end_time=guardrail_entry.get("end_time", None), # type: ignore ) - verbose_logger.debug(f"Logged guardrail information as span: {span}") + verbose_logger.debug("Logged guardrail information as span: %s", span) span.end() @@ -1035,7 +1036,7 @@ def _add_prompt_to_generation_params( try: generation_params["prompt"] = langfuse_client.get_prompt(prompt_management_metadata["prompt_id"]) except Exception as e: - verbose_logger.debug(f"[Non-blocking] Langfuse Logger: Error getting prompt client for logging: {e}") + verbose_logger.debug("[Non-blocking] Langfuse Logger: Error getting prompt client for logging: %s", e) else: generation_params["prompt"] = user_prompt diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index 143362f3468..d7e9460a580 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -315,10 +315,10 @@ class LangfuseOtelLogger(OpenTelemetry): if langfuse_host: normalized_host = langfuse_host if langfuse_host.startswith("http") else f"https://{langfuse_host}" endpoint = f"{normalized_host.rstrip('/')}/api/public/otel" - verbose_logger.debug(f"Using Langfuse OTEL endpoint from host: {endpoint}") + verbose_logger.debug("Using Langfuse OTEL endpoint from host: %s", endpoint) else: endpoint = LANGFUSE_CLOUD_US_ENDPOINT - verbose_logger.debug(f"Using Langfuse US cloud endpoint: {endpoint}") + verbose_logger.debug("Using Langfuse US cloud endpoint: %s", endpoint) auth_header = LangfuseOtelLogger._get_langfuse_authorization_header( public_key=public_key, secret_key=secret_key diff --git a/litellm/integrations/langfuse/langfuse_prompt_management.py b/litellm/integrations/langfuse/langfuse_prompt_management.py index 56383b45a8c..f7fc63c0866 100644 --- a/litellm/integrations/langfuse/langfuse_prompt_management.py +++ b/litellm/integrations/langfuse/langfuse_prompt_management.py @@ -317,7 +317,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge except Exception as e: from litellm._logging import verbose_logger - verbose_logger.exception(f"Langfuse Layer Error - Exception occurred while logging success event: {e}") + verbose_logger.exception("Langfuse Layer Error - Exception occurred while logging success event: %s", e) self.handle_callback_failure(callback_name="langfuse") async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): @@ -347,5 +347,5 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge except Exception as e: from litellm._logging import verbose_logger - verbose_logger.exception(f"Langfuse Layer Error - Exception occurred while logging failure event: {e}") + verbose_logger.exception("Langfuse Layer Error - Exception occurred while logging failure event: %s", e) self.handle_callback_failure(callback_name="langfuse") diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 1f5d3179fb3..6dd0863cc41 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -194,7 +194,7 @@ class LangsmithLogger(CustomBatchLogger): fields = self._extract_metadata_fields(metadata, credentials) verbose_logger.debug( - f"Langsmith Logging - project_name: {fields['project_name']}, run_name {fields['run_name']}" + "Langsmith Logging - project_name: %s, run_name %s", fields["project_name"], fields["run_name"] ) payload: StandardLoggingPayload | None = kwargs.get("standard_logging_object", None) @@ -244,7 +244,7 @@ class LangsmithLogger(CustomBatchLogger): random_sample = random.random() if random_sample > sampling_rate: verbose_logger.info( - f"Skipping Langsmith logging. Sampling rate={sampling_rate}, random_sample={random_sample}" + "Skipping Langsmith logging. Sampling rate=%s, random_sample=%s", sampling_rate, random_sample ) return # Skip logging verbose_logger.debug( @@ -267,7 +267,7 @@ class LangsmithLogger(CustomBatchLogger): credentials=credentials, ) ) - verbose_logger.debug(f"Langsmith, event added to queue. Will flush in {self.flush_interval} seconds...") + verbose_logger.debug("Langsmith, event added to queue. Will flush in %s seconds...", self.flush_interval) if len(self.log_queue) >= self.batch_size: self._send_batch() @@ -282,7 +282,7 @@ class LangsmithLogger(CustomBatchLogger): random_sample = random.random() if random_sample > sampling_rate: verbose_logger.info( - f"Skipping Langsmith logging. Sampling rate={sampling_rate}, random_sample={random_sample}" + "Skipping Langsmith logging. Sampling rate=%s, random_sample=%s", sampling_rate, random_sample ) return # Skip logging verbose_logger.debug( @@ -321,7 +321,7 @@ class LangsmithLogger(CustomBatchLogger): random_sample = random.random() if random_sample > sampling_rate: verbose_logger.info( - f"Skipping Langsmith logging. Sampling rate={sampling_rate}, random_sample={random_sample}" + "Skipping Langsmith logging. Sampling rate=%s, random_sample=%s", sampling_rate, random_sample ) return # Skip logging verbose_logger.info("Langsmith Failure Event Logging!") @@ -422,16 +422,16 @@ class LangsmithLogger(CustomBatchLogger): response.raise_for_status() if response.status_code >= 300: - verbose_logger.error(f"Langsmith Error: {response.status_code} - {response.text}") + verbose_logger.error("Langsmith Error: %s - %s", response.status_code, response.text) else: if self.is_mock_mode: - verbose_logger.debug(f"[LANGSMITH MOCK] Batch of {len(elements_to_log)} runs successfully mocked") + verbose_logger.debug("[LANGSMITH MOCK] Batch of %s runs successfully mocked", len(elements_to_log)) else: - verbose_logger.debug(f"Batch of {len(self.log_queue)} runs successfully created") + verbose_logger.debug("Batch of %s runs successfully created", len(self.log_queue)) except httpx.HTTPStatusError as e: - verbose_logger.exception(f"Langsmith HTTP Error: {e.response.status_code} - {e.response.text}") + verbose_logger.exception("Langsmith HTTP Error: %s - %s", e.response.status_code, e.response.text) except Exception: - verbose_logger.exception(f"Langsmith Layer Error - {traceback.format_exc()}") + verbose_logger.exception("Langsmith Layer Error - %s", traceback.format_exc()) def _group_batches_by_credentials(self) -> dict[CredentialsKey, BatchGroup]: """Groups queue objects by credentials using a proper key structure""" diff --git a/litellm/integrations/literal_ai.py b/litellm/integrations/literal_ai.py index a54fdcf4dbc..a17a35f6a0e 100644 --- a/litellm/integrations/literal_ai.py +++ b/litellm/integrations/literal_ai.py @@ -94,9 +94,9 @@ class LiteralAILogger(CustomBatchLogger): ) if response.status_code >= 300: - verbose_logger.error(f"Literal AI Error: {response.status_code} - {response.text}") + verbose_logger.error("Literal AI Error: %s - %s", response.status_code, response.text) else: - verbose_logger.debug(f"Batch of {len(self.log_queue)} runs successfully created") + verbose_logger.debug("Batch of %s runs successfully created", len(self.log_queue)) except Exception: verbose_logger.exception("Literal AI Layer Error") @@ -152,11 +152,11 @@ class LiteralAILogger(CustomBatchLogger): headers=self.headers, ) if response.status_code >= 300: - verbose_logger.error(f"Literal AI Error: {response.status_code} - {response.text}") + verbose_logger.error("Literal AI Error: %s - %s", response.status_code, response.text) else: - verbose_logger.debug(f"Batch of {len(self.log_queue)} runs successfully created") + verbose_logger.debug("Batch of %s runs successfully created", len(self.log_queue)) except httpx.HTTPStatusError as e: - verbose_logger.exception(f"Literal AI HTTP Error: {e.response.status_code} - {e.response.text}") + verbose_logger.exception("Literal AI HTTP Error: %s - %s", e.response.status_code, e.response.text) except Exception: verbose_logger.exception("Literal AI Layer Error") diff --git a/litellm/integrations/logfire_logger.py b/litellm/integrations/logfire_logger.py index c94fb832ccc..814e2b88ba1 100644 --- a/litellm/integrations/logfire_logger.py +++ b/litellm/integrations/logfire_logger.py @@ -90,7 +90,7 @@ class LogfireLogger: try: import logfire - verbose_logger.debug(f"logfire Logging - Enters logging function for model {kwargs}") + verbose_logger.debug("logfire Logging - Enters logging function for model %s", kwargs) if not response_obj: response_obj = {} @@ -159,4 +159,4 @@ class LogfireLogger: print_verbose(f"Logfire Layer Logging - final response object: {response_obj}") except Exception as e: - verbose_logger.debug(f"Logfire Layer Error - {e}\n{traceback.format_exc()}") + verbose_logger.debug("Logfire Layer Error - %s\n%s", e, traceback.format_exc()) diff --git a/litellm/integrations/mlflow.py b/litellm/integrations/mlflow.py index 7ce5b3d3eea..f41de320843 100644 --- a/litellm/integrations/mlflow.py +++ b/litellm/integrations/mlflow.py @@ -99,7 +99,7 @@ class MlflowLogger(CustomLogger): ) except Exception as e: - verbose_logger.debug(f"MLflow Logging Error - {e}", stack_info=True) + verbose_logger.debug("MLflow Logging Error - %s", e, stack_info=True) def _handle_stream_event(self, kwargs, response_obj, start_time, end_time): """ diff --git a/litellm/integrations/mock_client_factory.py b/litellm/integrations/mock_client_factory.py index 2aab1792618..f1bc2db16ef 100644 --- a/litellm/integrations/mock_client_factory.py +++ b/litellm/integrations/mock_client_factory.py @@ -144,7 +144,7 @@ def create_mock_client_factory(config: MockClientConfig): ): """Monkey-patched AsyncHTTPHandler.post that intercepts API calls.""" if isinstance(url, str) and _is_mock_url(url): - verbose_logger.info(f"[{config.name} MOCK] POST to {url}") + verbose_logger.info("[%s MOCK] POST to %s", config.name, url) await asyncio.sleep(_MOCK_LATENCY_SECONDS) return MockResponse( status_code=config.default_status_code, @@ -172,7 +172,7 @@ def create_mock_client_factory(config: MockClientConfig): def _mock_sync_client_post(self, url, **kwargs): """Monkey-patched httpx.Client.post that intercepts API calls.""" if _is_mock_url(url): - verbose_logger.info(f"[{config.name} MOCK] POST to {url} (sync)") + verbose_logger.info("[%s MOCK] POST to %s (sync)", config.name, url) return MockResponse( status_code=config.default_status_code, json_data=config.default_json_data, @@ -198,7 +198,7 @@ def create_mock_client_factory(config: MockClientConfig): ): """Monkey-patched HTTPHandler.post that intercepts API calls.""" if isinstance(url, str) and _is_mock_url(url): - verbose_logger.info(f"[{config.name} MOCK] POST to {url}") + verbose_logger.info("[%s MOCK] POST to %s", config.name, url) import time time.sleep(_MOCK_LATENCY_SECONDS) @@ -236,29 +236,29 @@ def create_mock_client_factory(config: MockClientConfig): if _mocks_initialized: return - verbose_logger.debug(f"[{config.name} MOCK] Initializing {config.name} mock client...") + verbose_logger.debug("[%s MOCK] Initializing %s mock client...", config.name, config.name) if config.patch_async_handler and _original_async_handler_post is None: from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler _original_async_handler_post = AsyncHTTPHandler.post AsyncHTTPHandler.post = _mock_async_handler_post # type: ignore - verbose_logger.debug(f"[{config.name} MOCK] Patched AsyncHTTPHandler.post") + verbose_logger.debug("[%s MOCK] Patched AsyncHTTPHandler.post", config.name) if config.patch_sync_client and _original_sync_client_post is None: _original_sync_client_post = httpx.Client.post httpx.Client.post = _mock_sync_client_post # type: ignore - verbose_logger.debug(f"[{config.name} MOCK] Patched httpx.Client.post") + verbose_logger.debug("[%s MOCK] Patched httpx.Client.post", config.name) if config.patch_http_handler and _original_http_handler_post is None: from litellm.llms.custom_httpx.http_handler import HTTPHandler _original_http_handler_post = HTTPHandler.post HTTPHandler.post = _mock_http_handler_post # type: ignore - verbose_logger.debug(f"[{config.name} MOCK] Patched HTTPHandler.post") + verbose_logger.debug("[%s MOCK] Patched HTTPHandler.post", config.name) verbose_logger.debug(f"[{config.name} MOCK] Mock latency set to {_MOCK_LATENCY_SECONDS * 1000:.0f}ms") - verbose_logger.debug(f"[{config.name} MOCK] {config.name} mock client initialization complete") + verbose_logger.debug("[%s MOCK] %s mock client initialization complete", config.name, config.name) _mocks_initialized = True @@ -274,7 +274,7 @@ def create_mock_client_factory(config: MockClientConfig): result = bool(result) if result is not None else False if result: - verbose_logger.info(f"{config.name} Mock Mode: ENABLED - API calls will be mocked") + verbose_logger.info("%s Mock Mode: ENABLED - API calls will be mocked", config.name) return result diff --git a/litellm/integrations/newrelic/newrelic.py b/litellm/integrations/newrelic/newrelic.py index 5511cb06174..bf8ab29d384 100644 --- a/litellm/integrations/newrelic/newrelic.py +++ b/litellm/integrations/newrelic/newrelic.py @@ -116,11 +116,12 @@ class NewRelicLogger(CustomLogger): self.enabled = True verbose_logger.info( - f"New Relic AI Monitoring initialized for app: {self.app_name}, " - f"content recording: {self.record_content}" + "New Relic AI Monitoring initialized for app: %s, content recording: %s", + self.app_name, + self.record_content, ) except Exception as e: - verbose_logger.error(f"Failed to initialize New Relic agent: {e}. Integration will be disabled.") + verbose_logger.error("Failed to initialize New Relic agent: %s. Integration will be disabled.", e) self.enabled = False def _get_newrelic_params(self) -> dict: @@ -170,9 +171,10 @@ class NewRelicLogger(CustomLogger): if value in ("0", "false", "no", "off"): return False verbose_logger.warning( - f"{var_name}={raw!r} is not a recognised boolean " - f"(accepts true/false, 1/0, yes/no, on/off). " - f"Falling back to default ({default})." + "%s=%r is not a recognised boolean (accepts true/false, 1/0, yes/no, on/off). Falling back to default (%s).", + var_name, + raw, + default, ) return default @@ -188,7 +190,7 @@ class NewRelicLogger(CustomLogger): return version("litellm") except Exception as e: - verbose_logger.warning(f"Unable to determine litellm version: {e}") + verbose_logger.warning("Unable to determine litellm version: %s", e) return "unknown" def _emit_supportability_metric(self): @@ -216,12 +218,12 @@ class NewRelicLogger(CustomLogger): if app and app.enabled: app.record_custom_metric(metric_name, 1) - verbose_logger.info(f"Emitted New Relic supportability metric: {metric_name}") + verbose_logger.info("Emitted New Relic supportability metric: %s", metric_name) else: verbose_logger.info("New Relic application is not enabled; skipping metric recording.") except Exception as e: - verbose_logger.warning(f"Failed to emit supportability metric: {e}") + verbose_logger.warning("Failed to emit supportability metric: %s", e) def _check_and_emit_periodic_metric(self): """ @@ -294,14 +296,13 @@ class NewRelicLogger(CustomLogger): trace_id = slo_trace_id except Exception as e: - verbose_logger.warning(f"Unable to parse New Relic trace context from upstream sources: {e}") + verbose_logger.warning("Unable to parse New Relic trace context from upstream sources: %s", e) if not trace_id: trace_id = uuid.uuid4().hex verbose_logger.debug( - f"New Relic trace_id not available from distributed tracing headers or " - f"StandardLoggingPayload. Generated trace_id={trace_id} for AI monitoring " - f"event grouping." + "New Relic trace_id not available from distributed tracing headers or StandardLoggingPayload. Generated trace_id=%s for AI monitoring event grouping.", + trace_id, ) return trace_id @@ -638,7 +639,7 @@ class NewRelicLogger(CustomLogger): verbose_logger.warning("New Relic application is not enabled; skipping summary event recording.") except Exception as e: - verbose_logger.warning(f"Failed to record New Relic summary event: {e}") + verbose_logger.warning("Failed to record New Relic summary event: %s", e) self.handle_callback_failure("newrelic") def _record_message_events( @@ -699,7 +700,7 @@ class NewRelicLogger(CustomLogger): app.record_custom_event("LlmChatCompletionMessage", event_data) except Exception as e: - verbose_logger.warning(f"Failed to record New Relic message events: {e}") + verbose_logger.warning("Failed to record New Relic message events: %s", e) self.handle_callback_failure("newrelic") def _record_error_metric(self): @@ -714,7 +715,7 @@ class NewRelicLogger(CustomLogger): if app and app.enabled: app.record_custom_metric("LLM/LiteLLM/Error", 1) except Exception as e: - verbose_logger.warning(f"Failed to record New Relic error metric: {e}") + verbose_logger.warning("Failed to record New Relic error metric: %s", e) self.handle_callback_failure("newrelic") def _process_success( @@ -846,7 +847,7 @@ class NewRelicLogger(CustomLogger): try: self._process_success(kwargs, response_obj, start_time, end_time) except Exception as e: - verbose_logger.warning(f"Error in New Relic log_success_event: {e}") + verbose_logger.warning("Error in New Relic log_success_event: %s", e) self.handle_callback_failure("newrelic") async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -859,7 +860,7 @@ class NewRelicLogger(CustomLogger): try: self._process_success(kwargs, response_obj, start_time, end_time) except Exception as e: - verbose_logger.warning(f"Error in New Relic async_log_success_event: {e}") + verbose_logger.warning("Error in New Relic async_log_success_event: %s", e) self.handle_callback_failure("newrelic") def log_failure_event(self, kwargs, response_obj, start_time, end_time): @@ -872,7 +873,7 @@ class NewRelicLogger(CustomLogger): self._record_error_metric() except Exception as e: - verbose_logger.warning(f"Error in New Relic log_failure_event: {e}") + verbose_logger.warning("Error in New Relic log_failure_event: %s", e) self.handle_callback_failure("newrelic") async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): @@ -885,5 +886,5 @@ class NewRelicLogger(CustomLogger): self._record_error_metric() except Exception as e: - verbose_logger.warning(f"Error in New Relic async_log_failure_event: {e}") + verbose_logger.warning("Error in New Relic async_log_failure_event: %s", e) self.handle_callback_failure("newrelic") diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 5342894a04f..94902460c81 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -2688,7 +2688,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): ) except json.JSONDecodeError: verbose_logger.debug( - f"litellm.integrations.opentelemetry.py::set_raw_request_attributes() - raw_response not json string - {_raw_response}" + "litellm.integrations.opentelemetry.py::set_raw_request_attributes() - raw_response not json string - %s", + _raw_response, ) self.safe_set_attribute( diff --git a/litellm/integrations/opik/opik.py b/litellm/integrations/opik/opik.py index e4d40a1af8f..afb6fef9d71 100644 --- a/litellm/integrations/opik/opik.py +++ b/litellm/integrations/opik/opik.py @@ -81,7 +81,7 @@ class OpikLogger(CustomBatchLogger): self.flush_lock: asyncio.Lock | None = asyncio.Lock() except Exception as e: verbose_logger.exception( - f"OpikLogger - Asynchronous processing not initialized as we are not running in an async context {e}" + "OpikLogger - Asynchronous processing not initialized as we are not running in an async context %s", e ) self.flush_lock = None @@ -154,14 +154,14 @@ class OpikLogger(CustomBatchLogger): self.log_queue.append(span_payload.__dict__) verbose_logger.debug( - f"OpikLogger added event to log_queue - Will flush in {self.flush_interval} seconds..." + "OpikLogger added event to log_queue - Will flush in %s seconds...", self.flush_interval ) if len(self.log_queue) >= self.batch_size: verbose_logger.debug("OpikLogger - Flushing batch") await self.flush_queue() except Exception as e: - verbose_logger.exception(f"OpikLogger failed to log success event - {e}\n{traceback.format_exc()}") + verbose_logger.exception("OpikLogger failed to log success event - %s\n%s", e, traceback.format_exc()) def _sync_send(self, url: str, headers: dict[str, str], batch: dict[str, Any]) -> None: try: @@ -174,7 +174,7 @@ class OpikLogger(CustomBatchLogger): if response.status_code != 204: raise Exception(f"Response from opik API status_code: {response.status_code}, text: {response.text}") except Exception as e: - verbose_logger.exception(f"OpikLogger failed to send batch - {e}\n{traceback.format_exc()}") + verbose_logger.exception("OpikLogger failed to send batch - %s\n%s", e, traceback.format_exc()) def log_success_event( self, @@ -245,7 +245,7 @@ class OpikLogger(CustomBatchLogger): batch={"spans": [span_payload.__dict__]}, ) except Exception as e: - verbose_logger.exception(f"OpikLogger failed to log success event - {e}\n{traceback.format_exc()}") + verbose_logger.exception("OpikLogger failed to log success event - %s\n%s", e, traceback.format_exc()) async def _submit_batch(self, url: str, headers: dict[str, str], batch: dict[str, Any]) -> None: try: @@ -257,11 +257,11 @@ class OpikLogger(CustomBatchLogger): response.raise_for_status() if response.status_code >= 300: - verbose_logger.error(f"OpikLogger - Error: {response.status_code} - {response.text}") + verbose_logger.error("OpikLogger - Error: %s - %s", response.status_code, response.text) else: - verbose_logger.info(f"OpikLogger - {len(self.log_queue)} Opik events submitted") + verbose_logger.info("OpikLogger - %s Opik events submitted", len(self.log_queue)) except Exception as e: - verbose_logger.exception(f"OpikLogger failed to send batch - {e}") + verbose_logger.exception("OpikLogger failed to send batch - %s", e) def _create_opik_headers(self) -> dict[str, str]: headers: dict[str, str] = {} @@ -283,7 +283,7 @@ class OpikLogger(CustomBatchLogger): # Send trace batch if len(traces) > 0: await self._submit_batch(url=self.trace_url, headers=self.headers, batch={"traces": traces}) - verbose_logger.info(f"Sent {len(traces)} traces") + verbose_logger.info("Sent %s traces", len(traces)) if len(spans) > 0: await self._submit_batch(url=self.span_url, headers=self.headers, batch={"spans": spans}) - verbose_logger.info(f"Sent {len(spans)} spans") + verbose_logger.info("Sent %s spans", len(spans)) diff --git a/litellm/integrations/opik/opik_payload_builder/extractors.py b/litellm/integrations/opik/opik_payload_builder/extractors.py index f95bd110cb3..ccc59cde751 100644 --- a/litellm/integrations/opik/opik_payload_builder/extractors.py +++ b/litellm/integrations/opik/opik_payload_builder/extractors.py @@ -66,7 +66,7 @@ def extract_opik_metadata( if requester_opik: opik_meta.update(requester_opik) - _logging.verbose_logger.debug(f"litellm_opik_metadata - {json.dumps(opik_meta, default=str)}") + _logging.verbose_logger.debug("litellm_opik_metadata - %s", json.dumps(opik_meta, default=str)) return opik_meta @@ -92,7 +92,7 @@ def extract_span_identifiers( try: return current_span_data.trace_id, current_span_data.id except AttributeError: - _logging.verbose_logger.warning(f"Unexpected current_span_data format: {type(current_span_data)}") + _logging.verbose_logger.warning("Unexpected current_span_data format: %s", type(current_span_data)) return None, None @@ -152,7 +152,7 @@ def apply_proxy_header_overrides( if isinstance(parsed_tags, list): tags.extend(parsed_tags) except (json.JSONDecodeError, TypeError): - _logging.verbose_logger.warning(f"Failed to parse tags from header: {value}") + _logging.verbose_logger.warning("Failed to parse tags from header: %s", value) return project_name, tags, thread_id diff --git a/litellm/integrations/opik/opik_payload_builder/payload_builders.py b/litellm/integrations/opik/opik_payload_builder/payload_builders.py index 517d5431b70..19e1dd1042c 100644 --- a/litellm/integrations/opik/opik_payload_builder/payload_builders.py +++ b/litellm/integrations/opik/opik_payload_builder/payload_builders.py @@ -61,7 +61,7 @@ def build_span_payload( created = response_obj.get("created", 0) span_name = f"{model}_{obj_type}_{created}" - _logging.verbose_logger.debug(f"OpikLogger creating span with id {span_id} for trace {trace_id}") + _logging.verbose_logger.debug("OpikLogger creating span with id %s for trace %s", span_id, trace_id) return types.SpanPayload( id=span_id, diff --git a/litellm/integrations/posthog.py b/litellm/integrations/posthog.py index 216edc44d3f..e6620535d81 100644 --- a/litellm/integrations/posthog.py +++ b/litellm/integrations/posthog.py @@ -72,7 +72,7 @@ class PostHogLogger(CustomBatchLogger): super().__init__(**kwargs, flush_lock=None, batch_size=POSTHOG_MAX_BATCH_SIZE) except Exception as e: - verbose_logger.exception(f"PostHog: Got exception on init PostHog client {e}") + verbose_logger.exception("PostHog: Got exception on init PostHog client %s", e) raise e def log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -107,7 +107,7 @@ class PostHogLogger(CustomBatchLogger): verbose_logger.debug("PostHog: Sync event successfully sent") except Exception as e: - verbose_logger.exception(f"PostHog Sync Layer Error - {e}") + verbose_logger.exception("PostHog Sync Layer Error - %s", e) async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): try: @@ -115,7 +115,7 @@ class PostHogLogger(CustomBatchLogger): self._ensure_async_setup() # Lazy initialization await self._log_async_event(kwargs, response_obj, start_time, end_time) except Exception as e: - verbose_logger.exception(f"PostHog Layer Error - {e}") + verbose_logger.exception("PostHog Layer Error - %s", e) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): try: @@ -123,7 +123,7 @@ class PostHogLogger(CustomBatchLogger): self._ensure_async_setup() # Lazy initialization await self._log_async_event(kwargs, response_obj, start_time, end_time) except Exception as e: - verbose_logger.exception(f"PostHog Layer Error - {e}") + verbose_logger.exception("PostHog Layer Error - %s", e) async def _log_async_event(self, kwargs, response_obj=None, start_time=0.0, end_time=0.0): # Note: response_obj, start_time, end_time not used - all data comes from kwargs @@ -132,7 +132,7 @@ class PostHogLogger(CustomBatchLogger): # Store event with its credentials for batch sending self.log_queue.append({"event": event_payload, "api_key": api_key, "api_url": api_url}) - verbose_logger.debug(f"PostHog, event added to queue. Will flush in {self.flush_interval} seconds...") + verbose_logger.debug("PostHog, event added to queue. Will flush in %s seconds...", self.flush_interval) if len(self.log_queue) >= self.batch_size: await self.flush_queue() @@ -328,7 +328,7 @@ class PostHogLogger(CustomBatchLogger): if not self.log_queue: return - verbose_logger.debug(f"PostHog: Sending batch of {len(self.log_queue)} events") + verbose_logger.debug("PostHog: Sending batch of %s events", len(self.log_queue)) if self.is_mock_mode: verbose_logger.debug("[POSTHOG MOCK] Mock mode enabled - API calls will be intercepted") @@ -363,11 +363,11 @@ class PostHogLogger(CustomBatchLogger): ) if self.is_mock_mode: - verbose_logger.debug(f"[POSTHOG MOCK] Batch of {len(self.log_queue)} events successfully mocked") + verbose_logger.debug("[POSTHOG MOCK] Batch of %s events successfully mocked", len(self.log_queue)) else: - verbose_logger.debug(f"PostHog: Batch of {len(self.log_queue)} events successfully sent") + verbose_logger.debug("PostHog: Batch of %s events successfully sent", len(self.log_queue)) except Exception as e: - verbose_logger.exception(f"PostHog Error sending batch API - {e}") + verbose_logger.exception("PostHog Error sending batch API - %s", e) def _ensure_async_setup(self): if not self._async_initialized: @@ -377,7 +377,7 @@ class PostHogLogger(CustomBatchLogger): self._async_initialized = True verbose_logger.debug("PostHog: Async components initialized") except Exception as e: - verbose_logger.error(f"PostHog: Failed to initialize async components: {e}") + verbose_logger.error("PostHog: Failed to initialize async components: %s", e) raise def _extract_metadata(self, kwargs: dict[str, Any]) -> dict[str, Any]: @@ -408,7 +408,7 @@ class PostHogLogger(CustomBatchLogger): if not self.log_queue: return - verbose_logger.debug(f"PostHog: Flushing {len(self.log_queue)} remaining events on exit") + verbose_logger.debug("PostHog: Flushing %s remaining events on exit", len(self.log_queue)) try: # Group events by credentials (same logic as async_send_batch) @@ -436,13 +436,13 @@ class PostHogLogger(CustomBatchLogger): response.raise_for_status() if response.status_code != 200: - verbose_logger.error(f"PostHog: Failed to flush on exit - status {response.status_code}") + verbose_logger.error("PostHog: Failed to flush on exit - status %s", response.status_code) if self.is_mock_mode: - verbose_logger.debug(f"[POSTHOG MOCK] Successfully flushed {len(self.log_queue)} events on exit") + verbose_logger.debug("[POSTHOG MOCK] Successfully flushed %s events on exit", len(self.log_queue)) else: - verbose_logger.debug(f"PostHog: Successfully flushed {len(self.log_queue)} events on exit") + verbose_logger.debug("PostHog: Successfully flushed %s events on exit", len(self.log_queue)) self.log_queue.clear() except Exception as e: - verbose_logger.error(f"PostHog: Error flushing events on exit: {e}") + verbose_logger.error("PostHog: Error flushing events on exit: %s", e) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index b7705a40e0c..78f4e213f50 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -697,7 +697,7 @@ class PrometheusLogger(CustomLogger): if not config: return {} - verbose_logger.debug(f"prometheus config: {config}") + verbose_logger.debug("prometheus config: %s", config) # Parse and validate all configuration groups parsed_configs = [] @@ -963,7 +963,10 @@ class PrometheusLogger(CustomLogger): except ImportError: # Fallback to simple logging if rich is not available verbose_logger.error( - f"Invalid labels for metric '{metric_name}': {invalid_labels}. Valid labels: {sorted(valid_labels)}" + "Invalid labels for metric '%s': %s. Valid labels: %s", + metric_name, + invalid_labels, + sorted(valid_labels), ) def _pretty_print_invalid_metric_error(self, invalid_metric_name: str, valid_metrics: tuple) -> None: @@ -1003,7 +1006,9 @@ class PrometheusLogger(CustomLogger): except ImportError: # Fallback to simple logging if rich is not available - verbose_logger.error(f"Invalid metric name: {invalid_metric_name}. Valid metrics: {sorted(valid_metrics)}") + verbose_logger.error( + "Invalid metric name: %s. Valid metrics: %s", invalid_metric_name, sorted(valid_metrics) + ) ######################################################### # End of pretty print functions @@ -1078,9 +1083,10 @@ class PrometheusLogger(CustomLogger): except ImportError: # Fallback to simple logging if rich is not available verbose_logger.info( - f"Enabled metrics: {sorted(self.enabled_metrics) if hasattr(self, 'enabled_metrics') else 'All metrics'}" + "Enabled metrics: %s", + sorted(self.enabled_metrics) if hasattr(self, "enabled_metrics") else "All metrics", ) - verbose_logger.info(f"Label filters: {label_filters}") + verbose_logger.info("Label filters: %s", label_filters) def _is_metric_enabled(self, metric_name: str) -> bool: """Check if a metric is enabled based on configuration""" @@ -1866,7 +1872,9 @@ class PrometheusLogger(CustomLogger): for i, r in enumerate(results): if isinstance(r, Exception): verbose_logger.debug( - f"[Non-Blocking] Prometheus: Budget metric lookup {['key', 'team', 'user', 'org'][i]} failed: {r}" + "[Non-Blocking] Prometheus: Budget metric lookup %s failed: %s", + ["key", "team", "user", "org"][i], + r, ) def _increment_top_level_request_and_spend_metrics( @@ -2132,7 +2140,7 @@ class PrometheusLogger(CustomLogger): response_cost=0, ) except Exception as e: - verbose_logger.exception(f"prometheus Layer Error(): Exception occured - {e}") + verbose_logger.exception("prometheus Layer Error(): Exception occured - %s", e) def _extract_status_code( self, @@ -2262,8 +2270,9 @@ class PrometheusLogger(CustomLogger): if self._is_invalid_api_key_request(status_code, exception=exception): verbose_logger.debug( - "Skipping Prometheus metrics for invalid API key request: " - f"status_code={status_code}, exception={type(exception).__name__ if exception else None}" + "Skipping Prometheus metrics for invalid API key request: status_code=%s, exception=%s", + status_code, + type(exception).__name__ if exception else None, ) return True @@ -2383,7 +2392,7 @@ class PrometheusLogger(CustomLogger): ) except Exception as e: - verbose_logger.exception(f"prometheus Layer Error(): Exception occured - {e}") + verbose_logger.exception("prometheus Layer Error(): Exception occured - %s", e) async def async_post_call_success_hook(self, data: dict, user_api_key_dict: UserAPIKeyAuth, response): """ @@ -2608,7 +2617,7 @@ class PrometheusLogger(CustomLogger): ) except Exception as e: - verbose_logger.debug(f"Prometheus Error: set_llm_deployment_failure_metrics. Exception occured - {e}") + verbose_logger.debug("Prometheus Error: set_llm_deployment_failure_metrics. Exception occured - %s", e) def _set_deployment_tpm_rpm_limit_metrics( self, @@ -2722,7 +2731,7 @@ class PrometheusLogger(CustomLogger): ) self.litellm_remaining_requests_metric.labels(**_labels).set(remaining_requests) except Exception as e: - verbose_logger.exception(f"Prometheus Error: _async_set_router_remaining_metrics. Exception occured - {e}") + verbose_logger.exception("Prometheus Error: _async_set_router_remaining_metrics. Exception occured - %s", e) def set_llm_deployment_success_metrics( self, @@ -2865,7 +2874,7 @@ class PrometheusLogger(CustomLogger): self.litellm_deployment_latency_per_output_token.labels(**_labels).observe(latency_per_token) except Exception as e: - verbose_logger.exception(f"Prometheus Error: set_llm_deployment_success_metrics. Exception occured - {e}") + verbose_logger.exception("Prometheus Error: set_llm_deployment_success_metrics. Exception occured - %s", e) return def _record_guardrail_metrics( @@ -2910,7 +2919,7 @@ class PrometheusLogger(CustomLogger): hook_type=hook_type, ).inc() except Exception as e: - verbose_logger.debug(f"Error recording guardrail metrics: {e}") + verbose_logger.debug("Error recording guardrail metrics: %s", e) ######################################## # Managed Batch Metric Recording Methods @@ -2933,7 +2942,7 @@ class PrometheusLogger(CustomLogger): api_key_alias=api_key_alias, ).inc() except Exception as e: - verbose_logger.warning(f"Error recording batch created metric: {e}") + verbose_logger.warning("Error recording batch created metric: %s", e) def record_managed_file_size( self, @@ -2954,7 +2963,7 @@ class PrometheusLogger(CustomLogger): user=user or "", ).set(size_bytes) except Exception as e: - verbose_logger.warning(f"Error recording file size metric: {e}") + verbose_logger.warning("Error recording file size metric: %s", e) def record_managed_batch_duration( self, @@ -2968,7 +2977,7 @@ class PrometheusLogger(CustomLogger): api_provider=api_provider or "", ).observe(duration_seconds) except Exception as e: - verbose_logger.warning(f"Error recording batch duration metric: {e}") + verbose_logger.warning("Error recording batch duration metric: %s", e) def record_managed_file_created( self, @@ -2987,14 +2996,14 @@ class PrometheusLogger(CustomLogger): api_key_alias=api_key_alias, ).inc() except Exception as e: - verbose_logger.warning(f"Error recording file created metric: {e}") + verbose_logger.warning("Error recording file created metric: %s", e) def record_managed_file_deleted(self, result: str): """Record a managed file deletion attempt. result is 'success' or 'blocked'.""" try: self.litellm_managed_file_deleted_total.labels(result=result).inc() except Exception as e: - verbose_logger.warning(f"Error recording file deleted metric: {e}") + verbose_logger.warning("Error recording file deleted metric: %s", e) def record_check_batch_cost_run( self, @@ -3021,7 +3030,7 @@ class PrometheusLogger(CustomLogger): api_provider=api_provider or "", ).inc() except Exception as e: - verbose_logger.warning(f"Error recording check batch cost metrics: {e}") + verbose_logger.warning("Error recording check batch cost metrics: %s", e) def record_check_batch_cost_error(self, error_type: str): try: @@ -3029,7 +3038,7 @@ class PrometheusLogger(CustomLogger): error_type=error_type, ).inc() except Exception as e: - verbose_logger.warning(f"Error recording check batch cost error metric: {e}") + verbose_logger.warning("Error recording check batch cost error metric: %s", e) @staticmethod def _get_exception_class_name(exception: Exception) -> str: @@ -3313,7 +3322,7 @@ class PrometheusLogger(CustomLogger): await set_metrics_function(data) except Exception as e: - verbose_logger.exception(f"Error initializing {data_type} budget metrics: {e}") + verbose_logger.exception("Error initializing %s budget metrics: %s", data_type, e) async def _initialize_team_budget_metrics(self): """ @@ -3493,18 +3502,18 @@ class PrometheusLogger(CustomLogger): # Get total user count total_users = await UserRepository(prisma_client).table.count() self.litellm_total_users_metric.set(total_users) - verbose_logger.debug(f"Prometheus: set litellm_total_users to {total_users}") + verbose_logger.debug("Prometheus: set litellm_total_users to %s", total_users) billable_users = await UserRepository(prisma_client).count_billable_users() self.litellm_active_users_metric.set(billable_users) - verbose_logger.debug(f"Prometheus: set litellm_active_users to {billable_users}") + verbose_logger.debug("Prometheus: set litellm_active_users to %s", billable_users) # Get total team count total_teams = await TeamRepository(prisma_client).table.count() self.litellm_teams_count_metric.set(total_teams) - verbose_logger.debug(f"Prometheus: set litellm_teams_count to {total_teams}") + verbose_logger.debug("Prometheus: set litellm_teams_count to %s", total_teams) except Exception as e: - verbose_logger.exception(f"Error initializing user/team count metrics: {e}") + verbose_logger.exception("Error initializing user/team count metrics: %s", e) async def _set_key_list_budget_metrics(self, keys: list[str | UserAPIKeyAuth]): """Helper function to set budget metrics for a list of keys""" @@ -3595,7 +3604,7 @@ class PrometheusLogger(CustomLogger): user_api_key_cache=user_api_key_cache, ) except Exception as e: - verbose_logger.debug(f"[Non-Blocking] Prometheus: Error getting team info: {e}") + verbose_logger.debug("[Non-Blocking] Prometheus: Error getting team info: %s", e) return team_object if team_info: @@ -3693,7 +3702,7 @@ class PrometheusLogger(CustomLogger): include_budget_table=True, ) except Exception as e: - verbose_logger.debug(f"[Non-Blocking] Prometheus: Error getting org info: {e}") + verbose_logger.debug("[Non-Blocking] Prometheus: Error getting org info: %s", e) return if org_info is None: @@ -3850,7 +3859,7 @@ class PrometheusLogger(CustomLogger): if key_object: user_api_key_dict.budget_reset_at = key_object.budget_reset_at except Exception as e: - verbose_logger.debug(f"[Non-Blocking] Prometheus: Error getting key info: {e}") + verbose_logger.debug("[Non-Blocking] Prometheus: Error getting key info: %s", e) return user_api_key_dict @@ -3915,7 +3924,7 @@ class PrometheusLogger(CustomLogger): check_db_only=False, ) except Exception as e: - verbose_logger.debug(f"[Non-Blocking] Prometheus: Error getting user info: {e}") + verbose_logger.debug("[Non-Blocking] Prometheus: Error getting user info: %s", e) return user_object if user_info: diff --git a/litellm/integrations/prometheus_services.py b/litellm/integrations/prometheus_services.py index 002d61265a4..446287441d7 100644 --- a/litellm/integrations/prometheus_services.py +++ b/litellm/integrations/prometheus_services.py @@ -92,7 +92,7 @@ class PrometheusServicesLogger: metrics = DEFAULT_SERVICE_CONFIGS.get(service, {}).get("metrics", []) if not metrics: - verbose_logger.debug(f"No metrics found for service {service}") + verbose_logger.debug("No metrics found for service %s", service) return DEFAULT_METRICS return metrics diff --git a/litellm/integrations/rubrik.py b/litellm/integrations/rubrik.py index 9942776bc00..e7fa1ca99b5 100644 --- a/litellm/integrations/rubrik.py +++ b/litellm/integrations/rubrik.py @@ -163,9 +163,9 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): parsed_rate = float(rbrk_sampling_rate.strip()) self.sampling_rate = max(0.0, min(1.0, parsed_rate)) if parsed_rate != self.sampling_rate: - verbose_logger.warning(f"RUBRIK_SAMPLING_RATE={parsed_rate} clamped to {self.sampling_rate}") + verbose_logger.warning("RUBRIK_SAMPLING_RATE=%s clamped to %s", parsed_rate, self.sampling_rate) except ValueError: - verbose_logger.warning(f"Invalid RUBRIK_SAMPLING_RATE: {rbrk_sampling_rate!r}, using 1.0") + verbose_logger.warning("Invalid RUBRIK_SAMPLING_RATE: %r, using 1.0", rbrk_sampling_rate) def _parse_batch_size(self) -> None: _batch_size = os.getenv("RUBRIK_BATCH_SIZE") @@ -173,11 +173,11 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): try: parsed_size = int(_batch_size) if parsed_size <= 0: - verbose_logger.warning(f"RUBRIK_BATCH_SIZE={_batch_size!r} must be > 0, using default") + verbose_logger.warning("RUBRIK_BATCH_SIZE=%r must be > 0, using default", _batch_size) else: self.batch_size = parsed_size except ValueError: - verbose_logger.warning(f"Invalid RUBRIK_BATCH_SIZE: {_batch_size!r}, using default") + verbose_logger.warning("Invalid RUBRIK_BATCH_SIZE: %r, using default", _batch_size) def _setup_clients(self, webhook_url: str) -> None: self.response_moderation_endpoint = f"{webhook_url}{_WEBHOOK_PATH_RESPONSE_MODERATION}" @@ -279,7 +279,9 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): return inputs except Exception as e: verbose_logger.error( - f"{label} hook failed: {e}. Returning original inputs unchanged.", + "%s hook failed: %s. Returning original inputs unchanged.", + label, + e, exc_info=True, ) return inputs @@ -388,8 +390,9 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): if logging_obj is None: verbose_logger.error( "Rubrik: moderation block fired with logging_obj=None for " - f"litellm_call_id={request_data.get('litellm_call_id')}; " - "cannot suppress success event or attach failure payload." + "litellm_call_id=%s; " + "cannot suppress success event or attach failure payload.", + request_data.get("litellm_call_id"), ) request_data["_rubrik_logging_obj"] = None return @@ -650,14 +653,15 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): payload["messages"] = (system_scaffold, messages) except Exception as e: verbose_logger.warning( - f"Rubrik: failed to prepend system prompt: {e}", + "Rubrik: failed to prepend system prompt: %s", + e, exc_info=True, ) async def _prepare_log_payload(self, kwargs: Mapping[str, Any], event_type: str) -> StandardLoggingPayload | None: """Shared logic for success logging (sampled).""" if random.random() > self.sampling_rate: - verbose_logger.debug(f"Skipping Rubrik {event_type} logging (sampling_rate={self.sampling_rate})") + verbose_logger.debug("Skipping Rubrik %s logging (sampling_rate=%s)", event_type, self.sampling_rate) return None # Deep-copy so mutations don't affect other callbacks sharing this object @@ -701,7 +705,9 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): await self._append_and_maybe_flush(payload) except Exception as e: verbose_logger.error( - f"Rubrik {event_type} logging hook failed: {e}. Skipping logging for this event.", + "Rubrik %s logging hook failed: %s. Skipping logging for this event.", + event_type, + e, exc_info=True, ) @@ -710,7 +716,8 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): # skip here to avoid double-logging the pre-block response. if kwargs.get("_rubrik_blocked"): verbose_logger.debug( - f"Rubrik: skipping success event for blocked request litellm_call_id={kwargs.get('litellm_call_id')}" + "Rubrik: skipping success event for blocked request litellm_call_id=%s", + kwargs.get("litellm_call_id"), ) return await self._enqueue_log_event(kwargs, "success") @@ -755,11 +762,14 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): # way we cannot build the payload. verbose_logger.warning( "Rubrik: block exception without stashed logging_obj. " - f"litellm_call_id={request_data.get('litellm_call_id')}, " - f"model={request_data.get('model')}, " - f"user_id={user_api_key_dict.user_id}, " - f"raising_guardrail=" - f"{getattr(original_exception, 'guardrail_name', None)}" + "litellm_call_id=%s, " + "model=%s, " + "user_id=%s, " + "raising_guardrail=%s", + request_data.get("litellm_call_id"), + request_data.get("model"), + user_api_key_dict.user_id, + getattr(original_exception, "guardrail_name", None), ) return @@ -786,8 +796,9 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): payload = self._prepare_block_failure_payload(logging_obj, exception, user_api_key_dict) except (AttributeError, ImportError, KeyError, TypeError) as e: verbose_logger.error( - f"Rubrik: failed to build blocked-tool payload for " - f"litellm_call_id={call_id}: {e}. Event will NOT be logged.", + "Rubrik: failed to build blocked-tool payload for litellm_call_id=%s: %s. Event will NOT be logged.", + call_id, + e, exc_info=True, ) return @@ -796,7 +807,9 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): await self._append_and_maybe_flush(payload) except Exception as e: verbose_logger.error( - f"Rubrik: failed to enqueue blocked-tool event for litellm_call_id={call_id}: {e}.", + "Rubrik: failed to enqueue blocked-tool event for litellm_call_id=%s: %s.", + call_id, + e, exc_info=True, ) @@ -853,8 +866,9 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): else: verbose_logger.debug( "Rubrik: standard_logging_object not yet on model_call_details " - f"for litellm_call_id={call_details.get('litellm_call_id')}; " - "using best-effort fallback payload." + "for litellm_call_id=%s; " + "using best-effort fallback payload.", + call_details.get("litellm_call_id"), ) payload = self._build_fallback_payload(call_details, user_api_key_dict) @@ -932,7 +946,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): ) response.raise_for_status() except httpx.HTTPStatusError as e: - verbose_logger.exception(f"Rubrik HTTP Error: {e.response.status_code} - {e.response.text}") + verbose_logger.exception("Rubrik HTTP Error: %s - %s", e.response.status_code, e.response.text) raise except Exception: verbose_logger.exception("Rubrik Layer Error") @@ -989,7 +1003,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): Exception: If the service is unavailable or returns an error. TypeError: If the response JSON is not a dict. """ - verbose_logger.debug(f"Sending request to {service_name}: {endpoint}") + verbose_logger.debug("Sending request to %s: %s", service_name, endpoint) http_response = await self.moderation_client.post( endpoint, json=payload, diff --git a/litellm/integrations/s3.py b/litellm/integrations/s3.py index c35cc88107f..a4bd488221c 100644 --- a/litellm/integrations/s3.py +++ b/litellm/integrations/s3.py @@ -31,7 +31,7 @@ class S3Logger: import boto3 try: - verbose_logger.debug(f"in init s3 logger - s3_callback_params {litellm.s3_callback_params}") + verbose_logger.debug("in init s3 logger - s3_callback_params %s", litellm.s3_callback_params) s3_use_team_prefix = False @@ -62,7 +62,7 @@ class S3Logger: self.s3_server_side_encryption, self.s3_sse_kms_key_id = resolve_sse_params( s3_server_side_encryption, s3_sse_kms_key_id ) - verbose_logger.debug(f"s3 logger using endpoint url {s3_endpoint_url}") + verbose_logger.debug("s3 logger using endpoint url %s", s3_endpoint_url) # Create an S3 client with custom endpoint URL self.s3_client = boto3.client( "s3", @@ -86,7 +86,7 @@ class S3Logger: def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose): try: - verbose_logger.debug(f"s3 Logging - Enters logging function for model {kwargs}") + verbose_logger.debug("s3 Logging - Enters logging function for model %s", kwargs) # construct payload to send to s3 # follows the same params as langfuse.py @@ -168,14 +168,14 @@ class S3Logger: print_verbose(f"s3 Layer Logging - final response object: {response_obj}") return response except Exception as e: - verbose_logger.exception(f"s3 Layer Error - {e}") + verbose_logger.exception("s3 Layer Error - %s", e) def _validated_sse_value(name: str, value: str | None) -> str | None: if value is None or isinstance(value, str): return value verbose_logger.warning( - f"s3 logging: ignoring {name} because it has invalid type {type(value).__name__}; expected a string" + "s3 logging: ignoring %s because it has invalid type %s; expected a string", name, type(value).__name__ ) return None @@ -191,8 +191,8 @@ def resolve_sse_params( return None, None if valid_key_id and not algorithm.startswith("aws:kms"): verbose_logger.warning( - f"s3 logging: ignoring s3_sse_kms_key_id because s3_server_side_encryption is {algorithm}; " - "set it to aws:kms to encrypt with the KMS key" + "s3 logging: ignoring s3_sse_kms_key_id because s3_server_side_encryption is %s; set it to aws:kms to encrypt with the KMS key", + algorithm, ) return algorithm, None return algorithm, valid_key_id diff --git a/litellm/integrations/s3_v2.py b/litellm/integrations/s3_v2.py index 44c6e42f9f0..c197b65696f 100644 --- a/litellm/integrations/s3_v2.py +++ b/litellm/integrations/s3_v2.py @@ -64,12 +64,12 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): _masker = SensitiveDataMasker() if s3_callback_params_override is not None: verbose_logger.debug( - f"in init s3 logger (audit override) - {_masker.mask_dict(dict(s3_callback_params_override))}" + "in init s3 logger (audit override) - %s", _masker.mask_dict(dict(s3_callback_params_override)) ) else: verbose_logger.debug( - f"in init s3 logger - s3_callback_params " - f"{_masker.mask_dict(dict(litellm.s3_callback_params or {}))}" + "in init s3 logger - s3_callback_params %s", + _masker.mask_dict(dict(litellm.s3_callback_params or {})), ) # Initialize S3 params first to get the correct s3_verify value @@ -98,11 +98,11 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): s3_server_side_encryption=s3_server_side_encryption, s3_sse_kms_key_id=s3_sse_kms_key_id, ) - verbose_logger.debug(f"s3 logger using endpoint url {s3_endpoint_url}") + verbose_logger.debug("s3 logger using endpoint url %s", s3_endpoint_url) # IMPORTANT # Create httpx client AFTER _init_s3_params so we have the correct s3_verify value - verbose_logger.debug(f"s3_v2 logger creating async httpx client with s3_verify={self.s3_verify}") + verbose_logger.debug("s3_v2 logger creating async httpx client with s3_verify=%s", self.s3_verify) self.async_httpx_client = get_async_httpx_client( llm_provider=httpxSpecialProvider.LoggingCallback, params={"ssl_verify": self.s3_verify}, @@ -111,7 +111,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): asyncio.create_task(self.periodic_flush()) self.flush_lock = asyncio.Lock() - verbose_logger.debug(f"s3 flush interval: {s3_flush_interval}, s3 batch size: {s3_batch_size}") + verbose_logger.debug("s3 flush interval: %s, s3 batch size: %s", s3_flush_interval, s3_batch_size) # Call CustomLogger's __init__ CustomBatchLogger.__init__( self, @@ -259,7 +259,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): async def _async_log_event_base(self, kwargs, response_obj, start_time, end_time): try: - verbose_logger.debug(f"s3 Logging - Enters logging function for model {kwargs}") + verbose_logger.debug("s3 Logging - Enters logging function for model %s", kwargs) s3_batch_logging_element = self.create_s3_batch_logging_element( start_time=start_time, @@ -284,7 +284,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): self.batch_size, ) except Exception as e: - verbose_logger.exception(f"s3 Layer Error - {e}") + verbose_logger.exception("s3 Layer Error - %s", e) self.handle_callback_failure(callback_name="S3Logger") async def async_upload_data_to_s3(self, batch_logging_element: s3BatchLoggingElement): @@ -313,8 +313,8 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): aws_sts_endpoint=self.s3_aws_sts_endpoint, ) - verbose_logger.debug(f"s3_v2 logger - uploading data to s3 - {batch_logging_element.s3_object_key}") - verbose_logger.debug(f"s3_v2 logger - s3_verify setting: {self.s3_verify}") + verbose_logger.debug("s3_v2 logger - uploading data to s3 - %s", batch_logging_element.s3_object_key) + verbose_logger.debug("s3_v2 logger - s3_verify setting: %s", self.s3_verify) # Prepare the URL url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}" @@ -374,16 +374,19 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): if response.status_code in (500, 503) and attempt < max_retries - 1: wait_time = 2**attempt # 1s, 2s verbose_logger.warning( - f"S3 upload returned {response.status_code}, retrying in {wait_time}s " - f"(attempt {attempt + 1}/{max_retries}) " - f"key={batch_logging_element.s3_object_key}" + "S3 upload returned %s, retrying in %ss (attempt %s/%s) key=%s", + response.status_code, + wait_time, + attempt + 1, + max_retries, + batch_logging_element.s3_object_key, ) await asyncio.sleep(wait_time) continue response.raise_for_status() break except Exception as e: - verbose_logger.exception(f"Error uploading to s3: {e}") + verbose_logger.exception("Error uploading to s3: %s", e) self.handle_callback_failure(callback_name="S3Logger") async def async_send_batch(self): @@ -395,7 +398,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): Raises: Does not raise an exception, will only verbose_logger.exception() """ - verbose_logger.debug(f"s3_v2 logger - sending batch of {len(self.log_queue)}") + verbose_logger.debug("s3_v2 logger - sending batch of %s", len(self.log_queue)) if not self.log_queue: return @@ -447,7 +450,10 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): s3_file_name = litellm.utils.get_logging_id(start_time, standard_logging_payload) or "" verbose_logger.debug( - f"Creating s3 file with prefix_components={prefix_components},prefix_path={prefix_path} and {s3_file_name}" + "Creating s3 file with prefix_components=%s,prefix_path=%s and %s", + prefix_components, + prefix_path, + s3_file_name, ) s3_object_key = get_s3_object_key( s3_path=cast(str | None, self.s3_path) or "", @@ -455,7 +461,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): start_time=start_time, s3_file_name=s3_file_name, ) - verbose_logger.debug(f"s3_object_key={s3_object_key}") + verbose_logger.debug("s3_object_key=%s", s3_object_key) s3_object_download_filename = ( f"time-{start_time.strftime('%Y-%m-%dT%H-%M-%S-%f')}_{standard_logging_payload['id']}.json" @@ -479,7 +485,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): except ImportError: raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") try: - verbose_logger.debug(f"s3_v2 logger - uploading data to s3 - {batch_logging_element.s3_object_key}") + verbose_logger.debug("s3_v2 logger - uploading data to s3 - %s", batch_logging_element.s3_object_key) credentials: Credentials = self.get_credentials( aws_access_key_id=self.s3_aws_access_key_id, aws_secret_access_key=self.s3_aws_secret_access_key, @@ -548,16 +554,19 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): if response.status_code in (500, 503) and attempt < max_retries - 1: wait_time = 2**attempt # 1s, 2s verbose_logger.warning( - f"S3 upload returned {response.status_code}, retrying in {wait_time}s " - f"(attempt {attempt + 1}/{max_retries}) " - f"key={batch_logging_element.s3_object_key}" + "S3 upload returned %s, retrying in %ss (attempt %s/%s) key=%s", + response.status_code, + wait_time, + attempt + 1, + max_retries, + batch_logging_element.s3_object_key, ) time.sleep(wait_time) continue response.raise_for_status() break except Exception as e: - verbose_logger.exception(f"Error uploading to s3: {e}") + verbose_logger.exception("Error uploading to s3: %s", e) self.handle_callback_failure(callback_name="S3Logger") async def _download_object_from_s3(self, s3_object_key: str) -> dict | None: @@ -596,7 +605,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): aws_sts_endpoint=self.s3_aws_sts_endpoint, ) - verbose_logger.debug(f"s3_v2 logger - downloading data from s3 - {s3_object_key}") + verbose_logger.debug("s3_v2 logger - downloading data from s3 - %s", s3_object_key) # Prepare the URL url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{s3_object_key}" @@ -642,7 +651,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): return response.json() except Exception as e: - verbose_logger.exception(f"Error downloading from S3: {e}") + verbose_logger.exception("Error downloading from S3: %s", e) return None async def get_proxy_server_request_from_cold_storage_with_object_key( @@ -666,5 +675,5 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): downloaded_object = await self._download_object_from_s3(object_key) return downloaded_object except Exception as e: - verbose_logger.exception(f"Error retrieving object {object_key} from cold storage: {e}") + verbose_logger.exception("Error retrieving object %s from cold storage: %s", object_key, e) return None diff --git a/litellm/integrations/sqs.py b/litellm/integrations/sqs.py index 56618b62368..267bc0def22 100644 --- a/litellm/integrations/sqs.py +++ b/litellm/integrations/sqs.py @@ -68,7 +68,7 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): **kwargs, ) -> None: try: - verbose_logger.debug(f"in init sqs logger - sqs_callback_params {litellm.aws_sqs_callback_params}") + verbose_logger.debug("in init sqs logger - sqs_callback_params %s", litellm.aws_sqs_callback_params) self.async_httpx_client = get_async_httpx_client( llm_provider=httpxSpecialProvider.LoggingCallback, @@ -100,7 +100,7 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): asyncio.create_task(self.periodic_flush()) self.flush_lock = asyncio.Lock() - verbose_logger.debug(f"sqs flush interval: {sqs_flush_interval}, sqs batch size: {sqs_batch_size}") + verbose_logger.debug("sqs flush interval: %s, sqs batch size: %s", sqs_flush_interval, sqs_batch_size) CustomBatchLogger.__init__( self, @@ -215,7 +215,7 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): self.batch_size, ) except Exception as e: - verbose_logger.exception(f"sqs Layer Error - {e}") + verbose_logger.exception("sqs Layer Error - %s", e) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): try: @@ -233,10 +233,10 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): ) except Exception as e: - verbose_logger.exception(f"Datadog Layer Error - {e}\n{traceback.format_exc()}") + verbose_logger.exception("Datadog Layer Error - %s\n%s", e, traceback.format_exc()) async def async_send_batch(self) -> None: - verbose_logger.debug(f"sqs logger - sending batch of {len(self.log_queue)}") + verbose_logger.debug("sqs logger - sending batch of %s", len(self.log_queue)) if not self.log_queue: return @@ -305,7 +305,7 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): ) response.raise_for_status() except Exception as e: - verbose_logger.exception(f"Error sending to SQS: {e}") + verbose_logger.exception("Error sending to SQS: %s", e) async def async_health_check(self) -> IntegrationHealthCheckStatus: """ diff --git a/litellm/integrations/traceloop.py b/litellm/integrations/traceloop.py index 77f20972f7a..f5d28fd369f 100644 --- a/litellm/integrations/traceloop.py +++ b/litellm/integrations/traceloop.py @@ -15,7 +15,9 @@ class TraceloopLogger: from traceloop.sdk.tracing.tracing import TracerWrapper except ModuleNotFoundError as e: verbose_logger.error( - f"Traceloop not installed, try running 'pip install traceloop-sdk' to fix this error: {e}\n{traceback.format_exc()}" + "Traceloop not installed, try running 'pip install traceloop-sdk' to fix this error: %s\n%s", + e, + traceback.format_exc(), ) raise e diff --git a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py index 6eac7a27e73..2e533b488b6 100644 --- a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py +++ b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py @@ -124,7 +124,7 @@ class VectorStorePreCallHook(CustomLogger): }, ) - verbose_logger.debug(f"search_response: {search_response}") + verbose_logger.debug("search_response: %s", search_response) # Store search results for later use in citations all_search_results.append(search_response) @@ -137,7 +137,7 @@ class VectorStorePreCallHook(CustomLogger): # Get the number of results for logging num_results = 0 num_results = len(search_response.get("data", []) or []) - verbose_logger.debug(f"Vector store search completed. Added context from {num_results} results") + verbose_logger.debug("Vector store search completed. Added context from %s results", num_results) # Store search results as-is (already in OpenAI-compatible format) if litellm_logging_obj and all_search_results: @@ -146,7 +146,7 @@ class VectorStorePreCallHook(CustomLogger): return model, modified_messages, non_default_params except Exception as e: - verbose_logger.exception(f"Error in VectorStorePreCallHook: {e}") + verbose_logger.exception("Error in VectorStorePreCallHook: %s", e) # Return original parameters on error return model, messages, non_default_params @@ -243,14 +243,14 @@ class VectorStorePreCallHook(CustomLogger): verbose_logger.debug("No litellm_logging_obj in request_data") return None - verbose_logger.debug(f"model_call_details keys: {list(litellm_logging_obj.model_call_details.keys())}") + verbose_logger.debug("model_call_details keys: %s", list(litellm_logging_obj.model_call_details.keys())) # Get search results from model_call_details (already in OpenAI format) search_results: list[VectorStoreSearchResponse] | None = litellm_logging_obj.model_call_details.get( "search_results" ) - verbose_logger.debug(f"Search results found: {search_results is not None}") + verbose_logger.debug("Search results found: %s", search_results is not None) if not search_results: verbose_logger.debug("No search results found") @@ -269,13 +269,13 @@ class VectorStorePreCallHook(CustomLogger): # Set the provider_specific_fields setattr(choice.message, "provider_specific_fields", provider_fields) - verbose_logger.debug(f"Added {len(search_results)} search results to response") + verbose_logger.debug("Added %s search results to response", len(search_results)) # Return modified response return response except Exception as e: - verbose_logger.exception(f"Error adding search results to response: {e}") + verbose_logger.exception("Error adding search results to response: %s", e) # Don't fail the request if search results fail to be added return None @@ -297,7 +297,7 @@ class VectorStorePreCallHook(CustomLogger): # Get search results from model_call_details (already in OpenAI format) search_results: list[VectorStoreSearchResponse] | None = request_data.get("search_results") - verbose_logger.debug(f"Search results found for streaming chunk: {search_results is not None}") + verbose_logger.debug("Search results found for streaming chunk: %s", search_results is not None) if not search_results: verbose_logger.debug("No search results found for streaming chunk") @@ -316,12 +316,12 @@ class VectorStorePreCallHook(CustomLogger): # Set the provider_specific_fields choice.delta.provider_specific_fields = provider_fields - verbose_logger.debug(f"Added {len(search_results)} search results to streaming chunk") + verbose_logger.debug("Added %s search results to streaming chunk", len(search_results)) # Return modified chunk return response_chunk except Exception as e: - verbose_logger.exception(f"Error adding search results to streaming chunk: {e}") + verbose_logger.exception("Error adding search results to streaming chunk: %s", e) # Don't fail the request if search results fail to be added return response_chunk diff --git a/litellm/integrations/weave/weave_otel.py b/litellm/integrations/weave/weave_otel.py index 321dda2983d..ffa30582771 100644 --- a/litellm/integrations/weave/weave_otel.py +++ b/litellm/integrations/weave/weave_otel.py @@ -148,10 +148,10 @@ def get_weave_otel_config() -> WeaveOtelConfig: host = "https://" + host # Self-managed instances use a different path endpoint = host.rstrip("/") + WEAVE_OTEL_ENDPOINT - verbose_logger.debug(f"Using Weave OTEL endpoint from host: {endpoint}") + verbose_logger.debug("Using Weave OTEL endpoint from host: %s", endpoint) else: endpoint = WEAVE_BASE_URL + WEAVE_OTEL_ENDPOINT - verbose_logger.debug(f"Using Weave cloud endpoint: {endpoint}") + verbose_logger.debug("Using Weave cloud endpoint: %s", endpoint) # Weave uses Basic auth with format: api: auth_header = _get_weave_authorization_header(api_key=api_key) diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 718f7b8fcd7..effc53d1c08 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -155,8 +155,8 @@ class WebSearchInterceptionLogger(CustomLogger): ) if anthropic_config is not None and anthropic_config.handles_web_search_natively(): verbose_logger.debug( - f"WebSearchInterception: Skipping short-circuit for {provider_str} " - "(provider handles web search natively via the agentic loop)" + "WebSearchInterception: Skipping short-circuit for %s (provider handles web search natively via the agentic loop)", + provider_str, ) return None except (ValueError, Exception): @@ -176,7 +176,7 @@ class WebSearchInterceptionLogger(CustomLogger): return None verbose_logger.debug( - f"WebSearchInterception: Short-circuit search detected (provider={provider_str}, query='{query}')" + "WebSearchInterception: Short-circuit search detected (provider=%s, query='%s')", provider_str, query ) # Native clients (Claude Desktop / Cowork / Anthropic SDK) make a @@ -198,7 +198,7 @@ class WebSearchInterceptionLogger(CustomLogger): else: search_result_text, structured = await self._execute_search(query, kwargs=kwargs) except Exception as e: - verbose_logger.error(f"WebSearchInterception: Short-circuit search failed: {e}") + verbose_logger.error("WebSearchInterception: Short-circuit search failed: %s", e) search_result_text, structured = f"Search failed: {e}", None content: list[dict[str, object]] = [] @@ -235,9 +235,9 @@ class WebSearchInterceptionLogger(CustomLogger): } verbose_logger.debug( - "WebSearchInterception: Short-circuit search completed, " - f"returning synthetic response ({len(search_result_text)} chars, " - f"native_blocks={native_tool is not None})" + "WebSearchInterception: Short-circuit search completed, returning synthetic response (%s chars, native_blocks=%s)", + len(search_result_text), + native_tool is not None, ) return response @@ -294,8 +294,10 @@ class WebSearchInterceptionLogger(CustomLogger): converted_tool = get_litellm_web_search_tool_openai() converted_tools.append(converted_tool) verbose_logger.debug( - f"WebSearchInterception: Converted {tool.get('name', 'unknown')} " - f"(type={tool.get('type', 'none')}) to {LITELLM_WEB_SEARCH_TOOL_NAME}" + "WebSearchInterception: Converted %s (type=%s) to %s", + tool.get("name", "unknown"), + tool.get("type", "none"), + LITELLM_WEB_SEARCH_TOOL_NAME, ) else: # Keep other tools as-is @@ -419,14 +421,14 @@ class WebSearchInterceptionLogger(CustomLogger): custom_llm_provider = kwargs.get("litellm_params", {}).get("custom_llm_provider", "") verbose_logger.debug( - f"WebSearchInterception: Pre-request hook called" - f" - custom_llm_provider={custom_llm_provider}" - f" - enabled_providers={self.enabled_providers or 'ALL'}" + "WebSearchInterception: Pre-request hook called - custom_llm_provider=%s - enabled_providers=%s", + custom_llm_provider, + self.enabled_providers or "ALL", ) if self.enabled_providers is not None and custom_llm_provider not in self.enabled_providers: verbose_logger.debug( - f"WebSearchInterception: Skipping - provider {custom_llm_provider} not in {self.enabled_providers}" + "WebSearchInterception: Skipping - provider %s not in %s", custom_llm_provider, self.enabled_providers ) return None @@ -440,7 +442,7 @@ class WebSearchInterceptionLogger(CustomLogger): if not has_websearch: return None - verbose_logger.debug(f"WebSearchInterception: Pre-request hook triggered for provider={custom_llm_provider}") + verbose_logger.debug("WebSearchInterception: Pre-request hook triggered for provider=%s", custom_llm_provider) # If the client sent an Anthropic-native web_search_* tool, mark the # request so the agentic loop emits native web_search_tool_result @@ -457,15 +459,17 @@ class WebSearchInterceptionLogger(CustomLogger): standard_tool = get_litellm_web_search_tool() converted_tools.append(standard_tool) verbose_logger.debug( - f"WebSearchInterception: Converted {tool.get('name', 'unknown')} " - f"(type={tool.get('type', 'none')}) to {LITELLM_WEB_SEARCH_TOOL_NAME}" + "WebSearchInterception: Converted %s (type=%s) to %s", + tool.get("name", "unknown"), + tool.get("type", "none"), + LITELLM_WEB_SEARCH_TOOL_NAME, ) else: converted_tools.append(tool) kwargs["tools"] = converted_tools verbose_logger.debug( - f"WebSearchInterception: Tools after conversion: {[t.get('name') for t in converted_tools]}" + "WebSearchInterception: Tools after conversion: %s", [t.get("name") for t in converted_tools] ) if "tool_choice" in kwargs: @@ -511,15 +515,17 @@ class WebSearchInterceptionLogger(CustomLogger): kwargs=kwargs, ) - verbose_logger.debug(f"WebSearchInterception: Hook called! provider={custom_llm_provider}, stream={stream}") - verbose_logger.debug(f"WebSearchInterception: Response type: {type(response)}") + verbose_logger.debug("WebSearchInterception: Hook called! provider=%s, stream=%s", custom_llm_provider, stream) + verbose_logger.debug("WebSearchInterception: Response type: %s", type(response)) # Check if provider should be intercepted # Note: custom_llm_provider is already normalized by get_llm_provider() # (e.g., "bedrock/invoke/..." -> "bedrock") if self.enabled_providers is not None and custom_llm_provider not in self.enabled_providers: verbose_logger.debug( - f"WebSearchInterception: Skipping provider {custom_llm_provider} (not in enabled list: {self.enabled_providers})" + "WebSearchInterception: Skipping provider %s (not in enabled list: %s)", + custom_llm_provider, + self.enabled_providers, ) return False, {} @@ -541,7 +547,7 @@ class WebSearchInterceptionLogger(CustomLogger): return False, {} verbose_logger.debug( - f"WebSearchInterception: Detected {len(tool_calls)} WebSearch tool call(s), executing agentic loop" + "WebSearchInterception: Detected %s WebSearch tool call(s), executing agentic loop", len(tool_calls) ) # Extract thinking blocks from response content. @@ -576,7 +582,7 @@ class WebSearchInterceptionLogger(CustomLogger): if thinking_blocks: verbose_logger.debug( - f"WebSearchInterception: Extracted {len(thinking_blocks)} thinking block(s) from response" + "WebSearchInterception: Extracted %s thinking block(s) from response", len(thinking_blocks) ) # Return tools dict with tool calls and thinking blocks @@ -606,14 +612,16 @@ class WebSearchInterceptionLogger(CustomLogger): """ verbose_logger.debug( - f"WebSearchInterception: Chat completion hook called! provider={custom_llm_provider}, stream={stream}" + "WebSearchInterception: Chat completion hook called! provider=%s, stream=%s", custom_llm_provider, stream ) - verbose_logger.debug(f"WebSearchInterception: Response type: {type(response)}") + verbose_logger.debug("WebSearchInterception: Response type: %s", type(response)) # Check if provider should be intercepted if self.enabled_providers is not None and custom_llm_provider not in self.enabled_providers: verbose_logger.debug( - f"WebSearchInterception: Skipping provider {custom_llm_provider} (not in enabled list: {self.enabled_providers})" + "WebSearchInterception: Skipping provider %s (not in enabled list: %s)", + custom_llm_provider, + self.enabled_providers, ) return False, {} @@ -635,7 +643,7 @@ class WebSearchInterceptionLogger(CustomLogger): return False, {} verbose_logger.debug( - f"WebSearchInterception: Detected {len(tool_calls)} WebSearch tool call(s), executing agentic loop" + "WebSearchInterception: Detected %s WebSearch tool call(s), executing agentic loop", len(tool_calls) ) # Return tools dict with tool calls @@ -659,12 +667,14 @@ class WebSearchInterceptionLogger(CustomLogger): ) -> tuple[bool, dict]: """Check if WebSearch interception is needed for the Responses API.""" verbose_logger.debug( - f"WebSearchInterception: Responses hook called! provider={custom_llm_provider}, stream={stream}" + "WebSearchInterception: Responses hook called! provider=%s, stream=%s", custom_llm_provider, stream ) if self.enabled_providers is not None and custom_llm_provider not in self.enabled_providers: verbose_logger.debug( - f"WebSearchInterception: Skipping provider {custom_llm_provider} (not in enabled list: {self.enabled_providers})" + "WebSearchInterception: Skipping provider %s (not in enabled list: %s)", + custom_llm_provider, + self.enabled_providers, ) return False, {} @@ -684,7 +694,7 @@ class WebSearchInterceptionLogger(CustomLogger): return False, {} verbose_logger.debug( - f"WebSearchInterception: Detected {len(tool_calls)} WebSearch function_call(s), executing agentic loop" + "WebSearchInterception: Detected %s WebSearch function_call(s), executing agentic loop", len(tool_calls) ) tools_dict = { @@ -716,7 +726,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls = tools["tool_calls"] thinking_blocks = tools.get("thinking_blocks", []) - verbose_logger.debug(f"WebSearchInterception: Executing agentic loop for {len(tool_calls)} search(es)") + verbose_logger.debug("WebSearchInterception: Executing agentic loop for %s search(es)", len(tool_calls)) return await self._execute_agentic_loop( model=model, @@ -853,7 +863,8 @@ class WebSearchInterceptionLogger(CustomLogger): # Object refused write — fall through and leave the response # untouched rather than crash the request. verbose_logger.debug( - f"WebSearchInterception: could not inject native blocks into response of type {type(response).__name__}" + "WebSearchInterception: could not inject native blocks into response of type %s", + type(response).__name__, ) return response @@ -878,7 +889,7 @@ class WebSearchInterceptionLogger(CustomLogger): response_format = tools.get("response_format", "openai") verbose_logger.debug( - f"WebSearchInterception: Executing chat completion agentic loop for {len(tool_calls)} search(es)" + "WebSearchInterception: Executing chat completion agentic loop for %s search(es)", len(tool_calls) ) return await self._execute_chat_completion_agentic_loop( @@ -962,7 +973,7 @@ class WebSearchInterceptionLogger(CustomLogger): for tool_call in tool_calls ] - verbose_logger.debug(f"WebSearchInterception: Executing {len(search_tasks)} responses search(es) in parallel") + verbose_logger.debug("WebSearchInterception: Executing %s responses search(es) in parallel", len(search_tasks)) search_results = await asyncio.gather(*search_tasks, return_exceptions=True) search_texts = [self._extract_search_text(result) for result in search_results] @@ -1038,12 +1049,12 @@ class WebSearchInterceptionLogger(CustomLogger): @staticmethod def _extract_search_text(result: object) -> str: if isinstance(result, Exception): - verbose_logger.error(f"WebSearchInterception: Responses search failed with error: {result}") + verbose_logger.error("WebSearchInterception: Responses search failed with error: %s", result) return f"Search failed: {result}" if isinstance(result, tuple) and len(result) == 2: text_value, _ = result return text_value if isinstance(text_value, str) else str(text_value) - verbose_logger.debug(f"WebSearchInterception: Unexpected search result type {type(result)}") + verbose_logger.debug("WebSearchInterception: Unexpected search result type %s", type(result)) return str(result) @staticmethod @@ -1176,15 +1187,15 @@ class WebSearchInterceptionLogger(CustomLogger): for tool_call in tool_calls: query = tool_call["input"].get("query") if query: - verbose_logger.debug(f"WebSearchInterception: Queuing search for query='{query}'") + verbose_logger.debug("WebSearchInterception: Queuing search for query='%s'", query) search_tasks.append(self._execute_search(query, kwargs=kwargs)) else: - verbose_logger.debug(f"WebSearchInterception: Tool call {tool_call['id']} has no query") + verbose_logger.debug("WebSearchInterception: Tool call %s has no query", tool_call["id"]) # Add empty result for tools without query search_tasks.append(self._create_empty_search_result()) # Execute searches in parallel - verbose_logger.debug(f"WebSearchInterception: Executing {len(search_tasks)} search(es) in parallel") + verbose_logger.debug("WebSearchInterception: Executing %s search(es) in parallel", len(search_tasks)) search_results = await asyncio.gather(*search_tasks, return_exceptions=True) # Split the gathered (text, structured) tuples into two parallel lists. @@ -1194,7 +1205,7 @@ class WebSearchInterceptionLogger(CustomLogger): structured_results: list[SearchResponse | None] = [] for i, result in enumerate(search_results): if isinstance(result, Exception): - verbose_logger.error(f"WebSearchInterception: Search {i} failed with error: {result}") + verbose_logger.error("WebSearchInterception: Search %s failed with error: %s", i, result) final_search_results.append(f"Search failed: {result}") structured_results.append(None) elif isinstance(result, tuple) and len(result) == 2: @@ -1204,7 +1215,7 @@ class WebSearchInterceptionLogger(CustomLogger): else: # Defensive: legacy callers / unexpected shape — preserve text, # drop structure. - verbose_logger.debug(f"WebSearchInterception: Unexpected result type {type(result)} at index {i}") + verbose_logger.debug("WebSearchInterception: Unexpected result type %s at index %s", type(result), i) final_search_results.append(str(result)) structured_results.append(None) @@ -1224,7 +1235,7 @@ class WebSearchInterceptionLogger(CustomLogger): max_tokens = self._resolve_max_tokens(anthropic_messages_optional_request_params, kwargs) - verbose_logger.debug(f"WebSearchInterception: Using max_tokens={max_tokens} for follow-up request") + verbose_logger.debug("WebSearchInterception: Using max_tokens=%s for follow-up request", max_tokens) optional_params_without_max_tokens = { k: v for k, v in anthropic_messages_optional_request_params.items() if k != "max_tokens" @@ -1286,12 +1297,12 @@ class WebSearchInterceptionLogger(CustomLogger): if not search_provider: search_provider = "perplexity" verbose_logger.debug( - "WebSearchInterception: No search tools configured in router, " - f"using default provider '{search_provider}'" + "WebSearchInterception: No search tools configured in router, using default provider '%s'", + search_provider, ) verbose_logger.debug( - f"WebSearchInterception: Executing search for '{query}' using provider '{search_provider}'" + "WebSearchInterception: Executing search for '%s' using provider '%s'", query, search_provider ) search_kwargs = { key: value @@ -1304,11 +1315,11 @@ class WebSearchInterceptionLogger(CustomLogger): search_result_text = WebSearchTransformation.format_search_response(result) verbose_logger.debug( - f"WebSearchInterception: Search completed for '{query}', got {len(search_result_text)} chars" + "WebSearchInterception: Search completed for '%s', got %s chars", query, len(search_result_text) ) return search_result_text, result except Exception as e: - verbose_logger.error(f"WebSearchInterception: Search failed for '{query}': {e}") + verbose_logger.error("WebSearchInterception: Search failed for '%s': %s", query, e) raise async def _authorize_search_tool( @@ -1392,21 +1403,25 @@ class WebSearchInterceptionLogger(CustomLogger): if matching_tools: search_provider = (matching_tools[0].get("litellm_params", {}) or {}).get("search_provider") verbose_logger.debug( - f"WebSearchInterception: Found search tool '{self.search_tool_name}' " - f"from {source} with provider '{search_provider}'" + "WebSearchInterception: Found search tool '%s' from %s with provider '%s'", + self.search_tool_name, + source, + search_provider, ) return matching_tools[0] verbose_logger.debug( - f"WebSearchInterception: Search tool '{self.search_tool_name}' not found in {source}, " - "falling back to first available or perplexity" + "WebSearchInterception: Search tool '%s' not found in %s, falling back to first available or perplexity", + self.search_tool_name, + source, ) if search_tools: first_tool = search_tools[0] search_provider = (first_tool.get("litellm_params", {}) or {}).get("search_provider") verbose_logger.debug( - f"WebSearchInterception: Using first available search tool from {source} " - f"with provider '{search_provider}'" + "WebSearchInterception: Using first available search tool from %s with provider '%s'", + source, + search_provider, ) return first_tool @@ -1470,15 +1485,15 @@ class WebSearchInterceptionLogger(CustomLogger): query = args.get("query") if query: - verbose_logger.debug(f"WebSearchInterception: Queuing search for query='{query}'") + verbose_logger.debug("WebSearchInterception: Queuing search for query='%s'", query) search_tasks.append(self._execute_search(query, kwargs=kwargs)) else: - verbose_logger.debug(f"WebSearchInterception: Tool call {tool_call.get('id')} has no query") + verbose_logger.debug("WebSearchInterception: Tool call %s has no query", tool_call.get("id")) # Add empty result for tools without query search_tasks.append(self._create_empty_search_result()) # Execute searches in parallel - verbose_logger.debug(f"WebSearchInterception: Executing {len(search_tasks)} search(es) in parallel") + verbose_logger.debug("WebSearchInterception: Executing %s search(es) in parallel", len(search_tasks)) search_results = await asyncio.gather(*search_tasks, return_exceptions=True) # Chat-completion path only needs text — OpenAI tool_result format @@ -1486,13 +1501,13 @@ class WebSearchInterceptionLogger(CustomLogger): final_search_results: list[str] = [] for i, result in enumerate(search_results): if isinstance(result, Exception): - verbose_logger.error(f"WebSearchInterception: Search {i} failed with error: {result}") + verbose_logger.error("WebSearchInterception: Search %s failed with error: %s", i, result) final_search_results.append(f"Search failed: {result}") elif isinstance(result, tuple) and len(result) == 2: text_value, _ = result final_search_results.append(cast(str, text_value) if isinstance(text_value, str) else str(text_value)) else: - verbose_logger.debug(f"WebSearchInterception: Unexpected result type {type(result)} at index {i}") + verbose_logger.debug("WebSearchInterception: Unexpected result type %s at index %s", type(result), i) final_search_results.append(str(result)) # Build assistant and tool messages using transformation @@ -1517,7 +1532,7 @@ class WebSearchInterceptionLogger(CustomLogger): ] verbose_logger.debug("WebSearchInterception: Making follow-up chat completion request with search results") - verbose_logger.debug(f"WebSearchInterception: Follow-up messages count: {len(follow_up_messages)}") + verbose_logger.debug("WebSearchInterception: Follow-up messages count: %s", len(follow_up_messages)) # Remove internal parameters that shouldn't be passed to follow-up request internal_params = { diff --git a/litellm/integrations/websearch_interception/transformation.py b/litellm/integrations/websearch_interception/transformation.py index 9dd0c155142..3b9683366be 100644 --- a/litellm/integrations/websearch_interception/transformation.py +++ b/litellm/integrations/websearch_interception/transformation.py @@ -103,7 +103,7 @@ class WebSearchTransformation: parsed_input = json.loads(arguments) if arguments else {} except json.JSONDecodeError: verbose_logger.warning( - f"WebSearchInterception: Failed to parse function_call arguments: {arguments}" + "WebSearchInterception: Failed to parse function_call arguments: %s", arguments ) parsed_input = {} elif isinstance(arguments, dict): @@ -122,7 +122,7 @@ class WebSearchTransformation: "input": parsed_input, } ) - verbose_logger.debug(f"WebSearchInterception: Found {item_name} function_call with call_id={call_id}") + verbose_logger.debug("WebSearchInterception: Found %s function_call with call_id=%s", item_name, call_id) return len(tool_calls) > 0, tool_calls @@ -178,7 +178,7 @@ class WebSearchTransformation: "input": block_input, } tool_calls.append(tool_call) - verbose_logger.debug(f"WebSearchInterception: Found {block_name} tool_use with id={tool_call['id']}") + verbose_logger.debug("WebSearchInterception: Found %s tool_use with id=%s", block_name, tool_call["id"]) return len(tool_calls) > 0, tool_calls @@ -255,7 +255,7 @@ class WebSearchTransformation: arguments = json.loads(function_arguments) except json.JSONDecodeError: verbose_logger.warning( - f"WebSearchInterception: Failed to parse function arguments: {function_arguments}" + "WebSearchInterception: Failed to parse function arguments: %s", function_arguments ) arguments = {} else: @@ -273,7 +273,7 @@ class WebSearchTransformation: "input": arguments, # For compatibility with Anthropic format } tool_calls.append(tool_call_dict) - verbose_logger.debug(f"WebSearchInterception: Found {function_name} tool_call with id={tool_id}") + verbose_logger.debug("WebSearchInterception: Found %s tool_call with id=%s", function_name, tool_id) return len(tool_calls) > 0, tool_calls diff --git a/litellm/integrations/weights_biases.py b/litellm/integrations/weights_biases.py index 0fe2a70ab66..0c1d28c23ef 100644 --- a/litellm/integrations/weights_biases.py +++ b/litellm/integrations/weights_biases.py @@ -42,9 +42,9 @@ try: elif response["object"] == "chat.completion": return self._resolve_chat_completion(request, response, time_elapsed) else: - logger.debug(f"Unknown OpenAI response object: {response['object']}") + logger.debug("Unknown OpenAI response object: %s", response["object"]) except Exception as e: - logger.warning(f"Failed to resolve request/response: {e}") + logger.warning("Failed to resolve request/response: %s", e) return None @staticmethod diff --git a/litellm/interactions/streaming_iterator.py b/litellm/interactions/streaming_iterator.py index f8ac4f25ebc..0fa6a5a7f75 100644 --- a/litellm/interactions/streaming_iterator.py +++ b/litellm/interactions/streaming_iterator.py @@ -109,7 +109,7 @@ class BaseInteractionsAPIStreamingIterator: return None except json.JSONDecodeError: # If we can't parse the chunk, continue - verbose_logger.debug(f"Failed to parse streaming chunk: {stripped_chunk[:200]}...") + verbose_logger.debug("Failed to parse streaming chunk: %s...", stripped_chunk[:200]) return None def _handle_logging_completed_response(self): diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 101cbae23f9..75e9524e3f1 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -536,7 +536,7 @@ def _map_anthropic_exception( llm_provider="anthropic", ) if hasattr(original_exception, "status_code"): - verbose_logger.debug(f"status_code: {original_exception.status_code}") + verbose_logger.debug("status_code: %s", original_exception.status_code) if original_exception.status_code == 401: raise AuthenticationError( message=f"AnthropicException - {error_str}", @@ -1752,7 +1752,7 @@ def _map_aleph_alpha_exception( response=getattr(original_exception, "response", None), ) elif hasattr(original_exception, "status_code"): - verbose_logger.debug(f"status code: {original_exception.status_code}") + verbose_logger.debug("status code: %s", original_exception.status_code) if original_exception.status_code == 401: raise AuthenticationError( message=f"AlephAlphaException - {original_exception.message}", @@ -2526,7 +2526,9 @@ def exception_logging( model_call_details["exception"] = exception model_call_details["additional_args"] = additional_args # User Logging -> if you pass in a custom logging function or want to use sentry breadcrumbs - verbose_logger.debug(f"Logging Details: logger_fn - {logger_fn} | callable(logger_fn) - {callable(logger_fn)}") + verbose_logger.debug( + "Logging Details: logger_fn - %s | callable(logger_fn) - %s", logger_fn, callable(logger_fn) + ) if logger_fn and callable(logger_fn): try: logger_fn( @@ -2534,11 +2536,11 @@ def exception_logging( ) # Expectation: any logger function passed in by the user should accept a dict object except Exception: verbose_logger.debug( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {traceback.format_exc()}" + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", traceback.format_exc() ) except Exception: verbose_logger.debug( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {traceback.format_exc()}" + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", traceback.format_exc() ) diff --git a/litellm/litellm_core_utils/fallback_utils.py b/litellm/litellm_core_utils/fallback_utils.py index 4e7ce828a58..a40970a1234 100644 --- a/litellm/litellm_core_utils/fallback_utils.py +++ b/litellm/litellm_core_utils/fallback_utils.py @@ -70,7 +70,7 @@ async def async_completion_with_fallbacks(**kwargs): ) except Exception as e: - verbose_logger.exception(f"Fallback attempt failed for model {model}: {e}") + verbose_logger.exception("Fallback attempt failed for model %s: %s", model, e) most_recent_exception_str = str(e) continue diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index f38b2859259..6b05591fb85 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -202,7 +202,7 @@ try: EnterpriseStandardLoggingPayloadSetup ) except Exception as e: - verbose_logger.debug(f"[Non-Blocking] Unable to import GenericAPILogger - LiteLLM Enterprise Feature - {e}") + verbose_logger.debug("[Non-Blocking] Unable to import GenericAPILogger - LiteLLM Enterprise Feature - %s", e) GenericAPILogger = CustomLogger # type: ignore ResendEmailLogger = CustomLogger # type: ignore SendGridEmailLogger = CustomLogger # type: ignore @@ -546,7 +546,7 @@ class Logging(LiteLLMLoggingBaseClass): self.litellm_request_debug = litellm_params.get("litellm_request_debug", False) self.logger_fn = litellm_params.get("logger_fn", None) if _is_debugging_on() or self.litellm_request_debug: - verbose_logger.debug(f"self.optional_params: {self.optional_params}") + verbose_logger.debug("self.optional_params: %s", self.optional_params) self.model_call_details.update( { @@ -981,7 +981,7 @@ class Logging(LiteLLMLoggingBaseClass): ) # Expectation: any logger function passed in by the user should accept a dict object except Exception as e: verbose_logger.exception( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {e}" + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e ) self.model_call_details["api_call_start_time"] = datetime.datetime.now() @@ -1001,7 +1001,7 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.debug("reaches supabase for logging!") model = self.model_call_details["model"] messages = self.model_call_details["input"] - verbose_logger.debug(f"supabaseClient: {supabaseClient}") + verbose_logger.debug("supabaseClient: %s", supabaseClient) supabaseClient.input_log_event( model=model, messages=messages, @@ -1041,15 +1041,15 @@ class Logging(LiteLLMLoggingBaseClass): callback_func=callback, ) except Exception as e: - verbose_logger.exception(f"litellm.Logging.pre_call(): Exception occured - {e}") + verbose_logger.exception("litellm.Logging.pre_call(): Exception occured - %s", e) verbose_logger.debug( - f"LiteLLM.Logging: is sentry capture exception initialized {capture_exception}" + "LiteLLM.Logging: is sentry capture exception initialized %s", capture_exception ) if capture_exception: # log this error to sentry for debugging capture_exception(e) except Exception as e: - verbose_logger.exception(f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {e}") - verbose_logger.error(f"LiteLLM.Logging: is sentry capture exception initialized {capture_exception}") + verbose_logger.exception("LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e) + verbose_logger.error("LiteLLM.Logging: is sentry capture exception initialized %s", capture_exception) if capture_exception: # log this error to sentry for debugging capture_exception(e) @@ -1091,10 +1091,10 @@ class Logging(LiteLLMLoggingBaseClass): ) if self.litellm_request_debug: verbose_logger.warning( - f"\033[92m{curl_command}\033[0m\n" + "\x1b[92m%s\x1b[0m\n", curl_command ) # .warning ensures this shows up in all environments else: - verbose_logger.debug(f"\033[92m{curl_command}\033[0m\n") + verbose_logger.debug("\x1b[92m%s\x1b[0m\n", curl_command) def _get_request_body(self, data: dict) -> str: return str(data) @@ -1164,7 +1164,7 @@ class Logging(LiteLLMLoggingBaseClass): ) # Expectation: any logger function passed in by the user should accept a dict object except Exception as e: verbose_logger.exception( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {e}" + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e ) original_response = redact_message_input_output_from_logging( model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), @@ -1201,15 +1201,16 @@ class Logging(LiteLLMLoggingBaseClass): ) except Exception as e: verbose_logger.exception( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while post-call logging with integrations {e}" + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while post-call logging with integrations %s", + e, ) verbose_logger.debug( - f"LiteLLM.Logging: is sentry capture exception initialized {capture_exception}" + "LiteLLM.Logging: is sentry capture exception initialized %s", capture_exception ) if capture_exception: # log this error to sentry for debugging capture_exception(e) except Exception as e: - verbose_logger.exception(f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {e}") + verbose_logger.exception("LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e) async def async_post_mcp_tool_call_hook( self, @@ -1249,7 +1250,7 @@ class Logging(LiteLLMLoggingBaseClass): if response is not None: response_obj = self._parse_post_mcp_call_hook_response(response=response) except Exception as e: - verbose_logger.exception(f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {e}") + verbose_logger.exception("LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e) return response_obj def _parse_post_mcp_call_hook_response(self, response: MCPPostCallResponseObject | None) -> Any: @@ -1439,14 +1440,14 @@ class Logging(LiteLLMLoggingBaseClass): error_str=str(e), traceback_str=_get_traceback_str_for_error(str(e)), ) - verbose_logger.debug(f"response_cost_failure_debug_information: {debug_info}") + verbose_logger.debug("response_cost_failure_debug_information: %s", debug_info) self.model_call_details["response_cost_failure_debug_information"] = debug_info return None try: response_cost = litellm.response_cost_calculator(**response_cost_calculator_kwargs) - verbose_logger.debug(f"response_cost: {response_cost}") + verbose_logger.debug("response_cost: %s", response_cost) additional_response_cost: object = self.model_call_details.get("additional_response_cost") if isinstance(additional_response_cost, (int, float)) and additional_response_cost > 0: return (response_cost or 0.0) + additional_response_cost @@ -1462,7 +1463,7 @@ class Logging(LiteLLMLoggingBaseClass): call_type=response_cost_calculator_kwargs["call_type"], custom_pricing=response_cost_calculator_kwargs["custom_pricing"], ) - verbose_logger.debug(f"response_cost_failure_debug_information: {debug_info}") + verbose_logger.debug("response_cost_failure_debug_information: %s", debug_info) self.model_call_details["response_cost_failure_debug_information"] = debug_info return None @@ -1497,7 +1498,7 @@ class Logging(LiteLLMLoggingBaseClass): raw_response=httpx.Response(status_code=200, headers={}), ) except Exception as e: # noqa: BLE001 - cost normalization must never break the response path - verbose_logger.debug(f"generate_content response cost normalization failed: {e}") + verbose_logger.debug("generate_content response cost normalization failed: %s", e) return None async def _response_cost_calculator_async( @@ -1666,7 +1667,7 @@ class Logging(LiteLLMLoggingBaseClass): # proxy cost tracking cal backs should run if not (isinstance(callback, CustomLogger) and "_PROXY_" in callback.__class__.__name__): - verbose_logger.debug(f"no-log request, skipping logging for {event_hook} event") + verbose_logger.debug("no-log request, skipping logging for %s event", event_hook) return False # Check for dynamically disabled callbacks via headers @@ -1676,7 +1677,7 @@ class Logging(LiteLLMLoggingBaseClass): standard_callback_dynamic_params=self.standard_callback_dynamic_params, ): verbose_logger.debug( - f"Callback {callback} disabled via x-litellm-disable-callbacks header for {event_hook} event" + "Callback %s disabled via x-litellm-disable-callbacks header for %s event", callback, event_hook ) return False @@ -1989,7 +1990,7 @@ class Logging(LiteLLMLoggingBaseClass): await self.async_success_handler(result=complete_streaming_response) def success_handler(self, result=None, start_time=None, end_time=None, cache_hit=None, **kwargs): - verbose_logger.debug(f"Logging Details LiteLLM-Success Call: Cache_hit={cache_hit}") + verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit) if not self.should_run_logging(event_type="sync_success"): # prevent double logging return start_time, end_time, result = self._success_handler_helper_fn( @@ -2210,7 +2211,8 @@ class Logging(LiteLLMLoggingBaseClass): # this only logs streaming once, complete_streaming_response exists i.e when stream ends if self.stream: verbose_logger.debug( - f"is complete_streaming_response in kwargs: {kwargs.get('complete_streaming_response', None)}" + "is complete_streaming_response in kwargs: %s", + kwargs.get("complete_streaming_response", None), ) if complete_streaming_response is None: continue @@ -2247,7 +2249,8 @@ class Logging(LiteLLMLoggingBaseClass): # this only logs streaming once, complete_streaming_response exists i.e when stream ends if self.stream: verbose_logger.debug( - f"is complete_streaming_response in kwargs: {kwargs.get('complete_streaming_response', None)}" + "is complete_streaming_response in kwargs: %s", + kwargs.get("complete_streaming_response", None), ) if complete_streaming_response is None: continue @@ -2389,7 +2392,8 @@ class Logging(LiteLLMLoggingBaseClass): pass except Exception as e: verbose_logger.exception( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging {e}", + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging %s", + e, ) async def async_success_handler(self, result=None, start_time=None, end_time=None, cache_hit=None, **kwargs): @@ -2483,10 +2487,10 @@ class Logging(LiteLLMLoggingBaseClass): result=complete_streaming_response ) - verbose_logger.debug(f"Model={self.model}; cost={self.model_call_details['response_cost']}") + verbose_logger.debug("Model=%s; cost=%s", self.model, self.model_call_details["response_cost"]) except litellm.NotFoundError: verbose_logger.warning( - f"Model={self.model} not found in completion cost map. Setting 'response_cost' to None" + "Model=%s not found in completion cost map. Setting 'response_cost' to None", self.model ) self.model_call_details["response_cost"] = None @@ -2681,7 +2685,8 @@ class Logging(LiteLLMLoggingBaseClass): ) except Exception: verbose_logger.error( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging {traceback.format_exc()}" + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging %s", + traceback.format_exc(), ) self._handle_callback_failure(callback=callback) @@ -2705,7 +2710,7 @@ class Logging(LiteLLMLoggingBaseClass): break # Only increment once except Exception as e: - verbose_logger.debug(f"Error in _handle_callback_failure: {e}") + verbose_logger.debug("Error in _handle_callback_failure: %s", e) def _failure_handler_helper_fn(self, exception, traceback_exception, start_time=None, end_time=None): if start_time is None: @@ -2784,7 +2789,7 @@ class Logging(LiteLLMLoggingBaseClass): ) # type: ignore def failure_handler(self, exception, traceback_exception, start_time=None, end_time=None): - verbose_logger.debug(f"Logging Details LiteLLM-Failure Call: {litellm.failure_callback}") + verbose_logger.debug("Logging Details LiteLLM-Failure Call: %s", litellm.failure_callback) if not self.should_run_logging(event_type="sync_failure"): # prevent double logging return litellm_params = self.model_call_details.get("litellm_params", {}) @@ -2949,7 +2954,7 @@ class Logging(LiteLLMLoggingBaseClass): capture_exception(e) except Exception as e: verbose_logger.exception( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure logging {e}" + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure logging %s", e ) async def async_failure_handler(self, exception, traceback_exception, start_time=None, end_time=None): @@ -3005,8 +3010,9 @@ class Logging(LiteLLMLoggingBaseClass): ) except Exception as e: verbose_logger.exception( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure \ - logging {e}\nCallback={callback}" + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure logging %s\nCallback=%s", + e, + callback, ) # Track callback logging failures in Prometheus self._handle_callback_failure(callback=callback) @@ -3134,7 +3140,7 @@ class Logging(LiteLLMLoggingBaseClass): """ filtered = [cb for cb in callbacks if not self._is_internal_litellm_proxy_callback(cb)] - verbose_logger.debug(f"Filtered callbacks: {filtered}") + verbose_logger.debug("Filtered callbacks: %s", filtered) return filtered def _get_callback_name(self, cb) -> str: @@ -4149,7 +4155,7 @@ def _init_custom_logger_compatible_class( return newrelic_logger # type: ignore return None except Exception as e: - verbose_logger.exception(f"[Non-Blocking Error] Error initializing custom logger: {e}") + verbose_logger.exception("[Non-Blocking Error] Error initializing custom logger: %s", e) return None return None @@ -4433,7 +4439,7 @@ def get_custom_logger_compatible_class( return None except Exception as e: - verbose_logger.exception(f"[Non-Blocking Error] Error getting custom logger: {e}") + verbose_logger.exception("[Non-Blocking Error] Error getting custom logger: %s", e) return None @@ -4783,7 +4789,8 @@ class StandardLoggingPayloadSetup: ) except Exception: verbose_logger.debug( # keep in debug otherwise it will trigger on every call - f"Model={model_cost_name} is not mapped in model cost map. Defaulting to None model_cost_information for standard_logging_payload" + "Model=%s is not mapped in model cost map. Defaulting to None model_cost_information for standard_logging_payload", + model_cost_name, ) model_cost_information = StandardLoggingModelInformation( model_map_key=model_cost_name, model_map_value=None @@ -5437,7 +5444,7 @@ def get_standard_logging_object_payload( return payload except Exception as e: - verbose_logger.exception(f"Error creating standard logging object - {e}") + verbose_logger.exception("Error creating standard logging object - %s", e) return None diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 2eaac7cb1ca..51c4e0200c3 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -150,7 +150,8 @@ def _generic_cost_per_character( prompt_cost = prompt_characters * custom_prompt_cost except Exception as e: verbose_logger.exception( - f"litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - {e}\nDefaulting to None" + "litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - %s\nDefaulting to None", + e, ) prompt_cost = None @@ -165,7 +166,8 @@ def _generic_cost_per_character( completion_cost = completion_characters * custom_completion_cost except Exception as e: verbose_logger.exception( - f"litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - {e}\nDefaulting to None" + "litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - %s\nDefaulting to None", + e, ) completion_cost = None @@ -388,7 +390,8 @@ def _get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: floa return float(cost_per_unit) except ValueError: verbose_logger.exception( - f"litellm.litellm_core_utils.llm_cost_calc.utils.py::calculate_cost_per_component(): Exception occured - {cost_per_unit}\nDefaulting to 0.0" + "litellm.litellm_core_utils.llm_cost_calc.utils.py::calculate_cost_per_component(): Exception occured - %s\nDefaulting to 0.0", + cost_per_unit, ) # If the service tier key doesn't exist or is None, try to fall back to the standard key @@ -408,7 +411,8 @@ def _get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: floa return float(fallback_cost) except ValueError: verbose_logger.exception( - f"litellm.litellm_core_utils.llm_cost_calc.utils.py::_get_cost_per_unit(): Exception occured - {fallback_cost}\nDefaulting to 0.0" + "litellm.litellm_core_utils.llm_cost_calc.utils.py::_get_cost_per_unit(): Exception occured - %s\nDefaulting to 0.0", + fallback_cost, ) break # Only try the first matching suffix diff --git a/litellm/litellm_core_utils/llm_response_utils/get_api_base.py b/litellm/litellm_core_utils/llm_response_utils/get_api_base.py index 1982e40448d..8eb8815b584 100644 --- a/litellm/litellm_core_utils/llm_response_utils/get_api_base.py +++ b/litellm/litellm_core_utils/llm_response_utils/get_api_base.py @@ -53,7 +53,7 @@ def get_api_base(model: str, optional_params: dict | LiteLLM_Params) -> str | No api_key=_optional_params.api_key, ) except Exception as e: - verbose_logger.debug(f"Error occurred in getting api base - {e}") + verbose_logger.debug("Error occurred in getting api base - %s", e) custom_llm_provider = None dynamic_api_base = None diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py index be732adfbe1..8bbe9ff4d25 100644 --- a/litellm/litellm_core_utils/logging_callback_manager.py +++ b/litellm/litellm_core_utils/logging_callback_manager.py @@ -146,7 +146,7 @@ class LoggingCallbackManager: if callback not in parent_list: parent_list.append(callback) else: - verbose_logger.debug(f"Callback {callback} already exists in {parent_list}, not adding again..") + verbose_logger.debug("Callback %s already exists in %s, not adding again..", callback, parent_list) def _check_callback_list_size(self, parent_list: list[CustomLogger | Callable | str]) -> bool: """ @@ -155,7 +155,9 @@ class LoggingCallbackManager: """ if len(parent_list) >= MAX_CALLBACKS: verbose_logger.warning( - f"Cannot add callback - would exceed MAX_CALLBACKS limit of {MAX_CALLBACKS}. Current callbacks: {len(parent_list)}" + "Cannot add callback - would exceed MAX_CALLBACKS limit of %s. Current callbacks: %s", + MAX_CALLBACKS, + len(parent_list), ) return False return True @@ -281,7 +283,7 @@ class LoggingCallbackManager: parent_list.append(callback) else: verbose_logger.debug( - f"Callback function {callback.__name__} already exists in {parent_list}, not adding again.." + "Callback function %s already exists in %s, not adding again..", callback.__name__, parent_list ) def _add_custom_logger_to_list( @@ -301,7 +303,10 @@ class LoggingCallbackManager: and self._get_custom_logger_key(existing_logger) == custom_logger_key ): verbose_logger.debug( - f"Custom logger of type {custom_logger_type_name}, key: {custom_logger_key} already exists in {parent_list}, not adding again.." + "Custom logger of type %s, key: %s already exists in %s, not adding again..", + custom_logger_type_name, + custom_logger_key, + parent_list, ) return parent_list.append(custom_logger) diff --git a/litellm/litellm_core_utils/logging_utils.py b/litellm/litellm_core_utils/logging_utils.py index 9340554b6d9..99a9ae4861a 100644 --- a/litellm/litellm_core_utils/logging_utils.py +++ b/litellm/litellm_core_utils/logging_utils.py @@ -178,7 +178,7 @@ def _get_parent_otel_span_from_logging_obj( return _get_parent_otel_span_from_kwargs(logging_obj.model_call_details) except Exception as e: - verbose_logger.exception(f"Error in _get_parent_otel_span_from_logging_obj: {e}") + verbose_logger.exception("Error in _get_parent_otel_span_from_logging_obj: %s", e) return None @@ -265,7 +265,7 @@ def _set_duration_in_model_call_details( else: verbose_logger.debug("`logging_obj` not found - unable to track `llm_api_duration_ms") except Exception as e: - verbose_logger.warning(f"Error setting `llm_api_duration_ms`: {e}") + verbose_logger.warning("Error setting `llm_api_duration_ms`: %s", e) def track_llm_api_timing(): @@ -321,7 +321,7 @@ def track_llm_api_timing(): ) ) except Exception as e: - verbose_logger.debug(f"Error in service logging: {e}") + verbose_logger.debug("Error in service logging: %s", e) @functools.wraps(func) def sync_wrapper(*args, **kwargs): @@ -366,7 +366,7 @@ def track_llm_api_timing(): parent_otel_span=parent_otel_span, ) except Exception as e: - verbose_logger.debug(f"Error in service logging: {e}") + verbose_logger.debug("Error in service logging: %s", e) # Check if the function is async or sync if inspect.iscoroutinefunction(func): diff --git a/litellm/litellm_core_utils/logging_worker.py b/litellm/litellm_core_utils/logging_worker.py index b0d1de32c3c..ff588243621 100644 --- a/litellm/litellm_core_utils/logging_worker.py +++ b/litellm/litellm_core_utils/logging_worker.py @@ -100,7 +100,7 @@ class LoggingWorker: timeout=self.timeout, ) except Exception as e: - verbose_logger.exception(f"LoggingWorker error: {e}") + verbose_logger.exception("LoggingWorker error: %s", e) finally: self._queue.task_done() finally: @@ -297,7 +297,7 @@ class LoggingWorker: if extracted_tasks: await self._process_extracted_tasks(extracted_tasks) except Exception as e: - verbose_logger.exception(f"LoggingWorker error during aggressive clear: {e}") + verbose_logger.exception("LoggingWorker error during aggressive clear: %s", e) finally: # Always reset the flag even if an error occurs self._aggressive_clear_in_progress = False @@ -383,7 +383,7 @@ class LoggingWorker: for _ in range(MAX_ITERATIONS_TO_CLEAR_QUEUE): # Check if we've exceeded the maximum time if asyncio.get_event_loop().time() - start_time >= MAX_TIME_TO_CLEAR_QUEUE: - verbose_logger.warning(f"clear_queue exceeded max_time of {MAX_TIME_TO_CLEAR_QUEUE}s, stopping early") + verbose_logger.warning("clear_queue exceeded max_time of %ss, stopping early", MAX_TIME_TO_CLEAR_QUEUE) break try: diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 8aa4f60b7c5..838261dedc8 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1413,7 +1413,7 @@ def convert_to_gemini_tool_call_result( inline_data_list.append(BlobType(data=mime_rest[1], mime_type=clean_mime)) content_str = "" except Exception as e: - verbose_logger.warning(f"Failed to parse data URL in tool response: {e}") + verbose_logger.warning("Failed to parse data URL in tool response: %s", e) elif isinstance(message["content"], list): content_list = message["content"] for content in content_list: @@ -1432,7 +1432,7 @@ def convert_to_gemini_tool_call_result( ) ) except Exception as e: - verbose_logger.warning(f"Failed to process Anthropic image block in tool response: {e}") + verbose_logger.warning("Failed to process Anthropic image block in tool response: %s", e) elif content_type in ("input_image", "image_url"): # Extract image for inline_data (for Computer Use screenshots and tool results) image_url_data = content.get("image_url", "") @@ -1449,7 +1449,7 @@ def convert_to_gemini_tool_call_result( ) ) except Exception as e: - verbose_logger.warning(f"Failed to process image in tool response: {e}") + verbose_logger.warning("Failed to process image in tool response: %s", e) elif content_type in ("file", "input_file"): # Extract file for inline_data (for tool results with PDF, audio, video, etc.) file_data = content.get("file_data", "") @@ -1474,7 +1474,7 @@ def convert_to_gemini_tool_call_result( ) ) except Exception as e: - verbose_logger.warning(f"Failed to process file in tool response: {e}") + verbose_logger.warning("Failed to process file in tool response: %s", e) name: str | None = message.get("name", "") # type: ignore # Recover name from last message with tool calls @@ -1997,7 +1997,7 @@ def _sanitize_empty_text_content( message = cast(AllMessageValues, dict(message)) # Make a copy message["content"] = _EMPTY_TEXT_PLACEHOLDER verbose_logger.debug( - f"_sanitize_empty_text_content: Replaced empty text content in {message.get('role')} message" + "_sanitize_empty_text_content: Replaced empty text content in %s message", message.get("role") ) return message @@ -2022,7 +2022,7 @@ def _sanitize_empty_text_content( message = cast(AllMessageValues, dict(message)) # Make a copy message["content"] = new_blocks # type: ignore verbose_logger.debug( - f"_sanitize_empty_text_content: Replaced empty text block(s) in {message.get('role')} message" + "_sanitize_empty_text_content: Replaced empty text block(s) in %s message", message.get("role") ) return message @@ -2086,7 +2086,8 @@ def _add_missing_tool_results( if missing_tool_call_ids: verbose_logger.debug( - f"_add_missing_tool_results: Found {len(missing_tool_call_ids)} orphaned tool calls. Adding dummy tool results." + "_add_missing_tool_results: Found %s orphaned tool calls. Adding dummy tool results.", + len(missing_tool_call_ids), ) result_messages.append(current_message) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index c401741f9dc..fd109d7cf3c 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -178,7 +178,7 @@ class RealTimeStreaming: # Catch-all base object so unknown/new event names never raise. typed_obj = OpenAIRealtimeStreamResponseBaseObject(**message_obj) # type: ignore except Exception as e: - verbose_logger.debug(f"Error parsing message for logging: {e}") + verbose_logger.debug("Error parsing message for logging: %s", e) self.messages.append(message_obj) # type: ignore[arg-type] return self.messages.append(typed_obj) @@ -213,7 +213,7 @@ class RealTimeStreaming: if tools and isinstance(tools, list): self.session_tools = tools # GA: session.type is required; log it for traceability but no action needed - verbose_logger.debug(f"Realtime session.type: {session.get('type')}") + verbose_logger.debug("Realtime session.type: %s", session.get("type")) if session.get("type") == "transcription": self._is_transcription_session = True except (json.JSONDecodeError, AttributeError, TypeError): @@ -981,7 +981,7 @@ class RealTimeStreaming: try: await self._handle_provider_config_message(raw_response) except Exception as e: - verbose_logger.exception(f"Error processing backend message, skipping: {e}") + verbose_logger.exception("Error processing backend message, skipping: %s", e) continue else: event = self._parse_backend_event(raw_response) @@ -1008,9 +1008,9 @@ class RealTimeStreaming: await self.websocket.send_text(json.dumps(translated)) except websockets.exceptions.ConnectionClosed as e: # type: ignore - verbose_logger.exception(f"Connection closed in backend to client send messages - {e}") + verbose_logger.exception("Connection closed in backend to client send messages - %s", e) except Exception as e: - verbose_logger.exception(f"Error in backend to client send messages: {e}") + verbose_logger.exception("Error in backend to client send messages: %s", e) finally: await self.log_messages() @@ -1404,7 +1404,7 @@ class RealTimeStreaming: self._guardrail_turn_detection_update_sent = True except Exception as e: - verbose_logger.debug(f"Error in client ack messages: {e}") + verbose_logger.debug("Error in client ack messages: %s", e) async def bidirectional_forward(self): forward_task = asyncio.create_task(self.backend_to_client_send_messages()) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 25155068baa..5cdb5877915 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -618,7 +618,7 @@ class CustomStreamWrapper: else: return "" except Exception as e: - verbose_logger.exception(f"litellm.CustomStreamWrapper.handle_baseten_chunk(): Exception occured - {e}") + verbose_logger.exception("litellm.CustomStreamWrapper.handle_baseten_chunk(): Exception occured - %s", e) return "" def handle_triton_stream(self, chunk): @@ -1430,7 +1430,7 @@ class CustomStreamWrapper: model_response.choices[0].delta = Delta(**_json_delta) except Exception as e: verbose_logger.exception( - f"litellm.CustomStreamWrapper.chunk_creator(): Exception occured - {e}" + "litellm.CustomStreamWrapper.chunk_creator(): Exception occured - %s", e ) model_response.choices[0].delta = Delta() elif self._has_any_special_delta_attributes(delta): @@ -1538,7 +1538,7 @@ class CustomStreamWrapper: except Exception as e: from litellm._logging import verbose_logger - verbose_logger.exception(f"Error in post-call streaming deployment hook: {e}") + verbose_logger.exception("Error in post-call streaming deployment hook: %s", e) return chunk def _add_mcp_list_tools_to_first_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream: @@ -1578,7 +1578,7 @@ class CustomStreamWrapper: except Exception as e: from litellm._logging import verbose_logger - verbose_logger.exception(f"Error adding MCP list tools to first chunk: {e}") + verbose_logger.exception("Error adding MCP list tools to first chunk: %s", e) return chunk @@ -1615,7 +1615,7 @@ class CustomStreamWrapper: except Exception as e: from litellm._logging import verbose_logger - verbose_logger.exception(f"Error adding MCP metadata to final chunk: {e}") + verbose_logger.exception("Error adding MCP metadata to final chunk: %s", e) return chunk diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index ff94965f628..d65f70d5a7a 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -80,17 +80,20 @@ def get_modified_max_tokens( ) # give at least a 10 token buffer. token counting can be imprecise. input_tokens += int(token_buffer) - verbose_logger.debug(f"max_output_tokens: {max_output_tokens}, user_max_tokens: {user_max_tokens}") + verbose_logger.debug("max_output_tokens: %s, user_max_tokens: %s", max_output_tokens, user_max_tokens) ## CASE 1: model input + output can't exceed X - happens when max input = max output, e.g. gpt-3.5-turbo if _model_info["max_input_tokens"] == max_output_tokens: - verbose_logger.debug(f"input_tokens: {input_tokens}, max_output_tokens: {max_output_tokens}") + verbose_logger.debug("input_tokens: %s, max_output_tokens: %s", input_tokens, max_output_tokens) if input_tokens > max_output_tokens: pass # allow call to fail normally - don't set max_tokens to negative. elif ( user_max_tokens + input_tokens > max_output_tokens ): # we can still modify to keep it positive but below the limit verbose_logger.debug( - f"MODIFYING MAX TOKENS - user_max_tokens={user_max_tokens}, input_tokens={input_tokens}, max_output_tokens={max_output_tokens}" + "MODIFYING MAX TOKENS - user_max_tokens=%s, input_tokens=%s, max_output_tokens=%s", + user_max_tokens, + input_tokens, + max_output_tokens, ) user_max_tokens = int(max_output_tokens - input_tokens) ## CASE 2: user_max_tokens> model max output tokens @@ -98,13 +101,17 @@ def get_modified_max_tokens( user_max_tokens = max_output_tokens verbose_logger.debug( - f"litellm.litellm_core_utils.token_counter.py::get_modified_max_tokens() - user_max_tokens: {user_max_tokens}" + "litellm.litellm_core_utils.token_counter.py::get_modified_max_tokens() - user_max_tokens: %s", + user_max_tokens, ) return user_max_tokens except Exception as e: verbose_logger.debug( - f"litellm.litellm_core_utils.token_counter.py::get_modified_max_tokens() - Error while checking max token limit: {e}\nmodel={model}, base_model={base_model}" + "litellm.litellm_core_utils.token_counter.py::get_modified_max_tokens() - Error while checking max token limit: %s\nmodel=%s, base_model=%s", + e, + model, + base_model, ) return user_max_tokens @@ -280,7 +287,7 @@ def calculate_img_tokens( int: The number of tokens for the image. """ if use_default_image_token_count: - verbose_logger.debug(f"Using default image token count: {DEFAULT_IMAGE_TOKEN_COUNT}") + verbose_logger.debug("Using default image token count: %s", DEFAULT_IMAGE_TOKEN_COUNT) return DEFAULT_IMAGE_TOKEN_COUNT if mode == "low" or mode == "auto": return base_tokens @@ -367,7 +374,7 @@ def token_counter( if litellm.disable_token_counter is True: return 0 - verbose_logger.debug(f"messages in token_counter: {messages}, text in token_counter: {text}") + verbose_logger.debug("messages in token_counter: %s, text in token_counter: %s", messages, text) if text is not None and messages is not None: raise ValueError("text and messages cannot both be set") if use_default_image_token_count is None: diff --git a/litellm/llms/__init__.py b/litellm/llms/__init__.py index 60715ac9bbf..a35fe5b2093 100644 --- a/litellm/llms/__init__.py +++ b/litellm/llms/__init__.py @@ -92,7 +92,7 @@ def discover_guardrail_translation_mappings() -> dict[CallTypes, type["BaseTrans try: # Import the module - verbose_logger.debug(f"Discovering guardrail translations in: {module_path}") + verbose_logger.debug("Discovering guardrail translations in: %s", module_path) module = importlib.import_module(module_path) @@ -102,14 +102,14 @@ def discover_guardrail_translation_mappings() -> dict[CallTypes, type["BaseTrans if isinstance(mappings, dict): discovered_mappings.update(mappings) verbose_logger.debug( - f"Found guardrail_translation_mappings in {module_path}: {list(mappings.keys())}" + "Found guardrail_translation_mappings in %s: %s", module_path, list(mappings.keys()) ) except ImportError as e: - verbose_logger.error(f"Could not import {module_path}: {e}") + verbose_logger.error("Could not import %s: %s", module_path, e) continue except Exception as e: - verbose_logger.error(f"Error processing {module_path}: {e}") + verbose_logger.error("Error processing %s: %s", module_path, e) continue try: @@ -126,11 +126,13 @@ def discover_guardrail_translation_mappings() -> dict[CallTypes, type["BaseTrans verbose_logger.debug("MCP guardrail translation mappings not available; skipping") verbose_logger.debug( - f"Discovered {len(discovered_mappings)} guardrail translation mappings: {list(discovered_mappings.keys())}" + "Discovered %s guardrail translation mappings: %s", + len(discovered_mappings), + list(discovered_mappings.keys()), ) except Exception as e: - verbose_logger.error(f"Error discovering guardrail translation mappings: {e}") + verbose_logger.error("Error discovering guardrail translation mappings: %s", e) return discovered_mappings diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 0fb7d7802a0..c752d02f662 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -825,10 +825,10 @@ class AnthropicMessagesHandler(BaseTranslation): if delta.get("type") == "text_delta": text += delta.get("text", "") except json.JSONDecodeError: - verbose_proxy_logger.warning(f"Failed to parse JSON from SSE data: {data_line}") + verbose_proxy_logger.warning("Failed to parse JSON from SSE data: %s", data_line) except Exception as e: - verbose_proxy_logger.error(f"Error extracting text from SSE: {e}") + verbose_proxy_logger.error("Error extracting text from SSE: %s", e) return text @@ -889,10 +889,10 @@ class AnthropicMessagesHandler(BaseTranslation): if stop_reason is not None: return True except json.JSONDecodeError: - verbose_proxy_logger.warning(f"Failed to parse JSON from SSE data: {data_line}") + verbose_proxy_logger.warning("Failed to parse JSON from SSE data: %s", data_line) except Exception as e: - verbose_proxy_logger.error(f"Error checking streaming end in SSE: {e}") + verbose_proxy_logger.error("Error checking streaming end in SSE: %s", e) # Handle already-parsed dict format elif isinstance(response, dict): diff --git a/litellm/llms/anthropic/count_tokens/handler.py b/litellm/llms/anthropic/count_tokens/handler.py index 0c3d0e931a2..3762b2b2f2f 100644 --- a/litellm/llms/anthropic/count_tokens/handler.py +++ b/litellm/llms/anthropic/count_tokens/handler.py @@ -54,7 +54,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): # Validate the request self.validate_request(model, messages) - verbose_logger.debug(f"Processing Anthropic CountTokens request for model: {model}") + verbose_logger.debug("Processing Anthropic CountTokens request for model: %s", model) # Transform request to Anthropic format request_body = self.transform_request_to_count_tokens( @@ -64,12 +64,12 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): system=system, ) - verbose_logger.debug(f"Transformed request: {request_body}") + verbose_logger.debug("Transformed request: %s", request_body) # Get endpoint URL endpoint_url = api_base or self.get_anthropic_count_tokens_endpoint() - verbose_logger.debug(f"Making request to: {endpoint_url}") + verbose_logger.debug("Making request to: %s", endpoint_url) # Get required headers headers = self.get_required_headers(api_key) @@ -87,11 +87,11 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): timeout=request_timeout, ) - verbose_logger.debug(f"Response status: {response.status_code}") + verbose_logger.debug("Response status: %s", response.status_code) if response.status_code != 200: error_text = response.text - verbose_logger.error(f"Anthropic API error: {error_text}") + verbose_logger.error("Anthropic API error: %s", error_text) raise AnthropicError( status_code=response.status_code, message=error_text, @@ -99,7 +99,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): anthropic_response = response.json() - verbose_logger.debug(f"Anthropic response: {anthropic_response}") + verbose_logger.debug("Anthropic response: %s", anthropic_response) # Return Anthropic response directly - no transformation needed return anthropic_response @@ -109,13 +109,13 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): raise except httpx.HTTPStatusError as e: # HTTP errors - preserve the actual status code - verbose_logger.error(f"HTTP error in CountTokens handler: {e}") + verbose_logger.error("HTTP error in CountTokens handler: %s", e) raise AnthropicError( status_code=e.response.status_code, message=e.response.text, ) except Exception as e: - verbose_logger.error(f"Error in CountTokens handler: {e}") + verbose_logger.error("Error in CountTokens handler: %s", e) raise AnthropicError( status_code=500, message=f"CountTokens processing error: {e}", diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index 8cc9d2ec0a9..5a2b6044528 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -81,7 +81,7 @@ class AnthropicTokenCounter(BaseTokenCounter): original_response=result, ) except AnthropicError as e: - verbose_logger.warning(f"Anthropic CountTokens API error: status={e.status_code}, message={e.message}") + verbose_logger.warning("Anthropic CountTokens API error: status=%s, message=%s", e.status_code, e.message) return TokenCountResponse( total_tokens=0, request_model=request_model, @@ -92,7 +92,7 @@ class AnthropicTokenCounter(BaseTokenCounter): status_code=e.status_code, ) except Exception as e: - verbose_logger.warning(f"Error calling Anthropic CountTokens API: {e}") + verbose_logger.warning("Error calling Anthropic CountTokens API: %s", e) return TokenCountResponse( total_tokens=0, request_model=request_model, diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index bb61043742d..5de40cc34b5 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -669,7 +669,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): return {"type": "message_stop"} raise StopIteration except Exception as e: - verbose_logger.error(f"Anthropic Adapter - {e}\n{traceback.format_exc()}") + verbose_logger.error("Anthropic Adapter - %s\n%s", e, traceback.format_exc()) raise StopIteration async def __anext__(self): diff --git a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/clear_tool_uses.py b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/clear_tool_uses.py index abfc45859ef..b432b122504 100644 --- a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/clear_tool_uses.py +++ b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/clear_tool_uses.py @@ -68,8 +68,8 @@ def _trigger_met( messages=messages, tools=cast(Any, tools), ) - verbose_logger.debug(f"context_management polyfill: current_tokens: {current_tokens}") - verbose_logger.debug(f"context_management polyfill: threshold: {threshold}") + verbose_logger.debug("context_management polyfill: current_tokens: %s", current_tokens) + verbose_logger.debug("context_management polyfill: threshold: %s", threshold) return current_tokens > threshold, current_tokens diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py index 887240bb1cc..74ee291bf9f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py @@ -164,8 +164,9 @@ async def anthropic_messages_with_mcp( response = await litellm.anthropic_messages(messages=list(working_messages), stream=False, **base_call_args) else: verbose_logger.warning( - f"MCP tool loop hit its {MAX_MCP_TOOL_USE_ITERATIONS} iteration cap for model {model}; " - "returning the last response" + "MCP tool loop hit its %s iteration cap for model %s; returning the last response", + MAX_MCP_TOOL_USE_ITERATIONS, + model, ) if stream: diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index f698c78604d..251c4e0f61a 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -300,7 +300,7 @@ class AnthropicResponsesStreamWrapper: except StopAsyncIteration: pass except Exception as e: - verbose_logger.error(f"AnthropicResponsesStreamWrapper error: {e}\n{traceback.format_exc()}") + verbose_logger.error("AnthropicResponsesStreamWrapper error: %s\n%s", e, traceback.format_exc()) # Drain any remaining queued chunks if self._chunk_queue: diff --git a/litellm/llms/anthropic/files/handler.py b/litellm/llms/anthropic/files/handler.py index b911347b2ff..c05392e0d7c 100644 --- a/litellm/llms/anthropic/files/handler.py +++ b/litellm/llms/anthropic/files/handler.py @@ -270,7 +270,7 @@ class AnthropicFilesHandler: transformed_content += "\n" # Add trailing newline for JSONL format return transformed_content.encode("utf-8") except Exception as e: - verbose_logger.error(f"Error transforming Anthropic batch results to OpenAI format: {e}") + verbose_logger.error("Error transforming Anthropic batch results to OpenAI format: %s", e) # Return original content if transformation fails return anthropic_content @@ -330,7 +330,7 @@ class AnthropicFilesHandler: return openai_body except Exception as e: - verbose_logger.error(f"Error transforming Anthropic message to OpenAI format: {e}") + verbose_logger.error("Error transforming Anthropic message to OpenAI format: %s", e) # Return a basic error response if transformation fails error_response: OpenAIChatCompletionResponse = { "id": anthropic_message.get("id", ""), diff --git a/litellm/llms/azure/chat/o_series_transformation.py b/litellm/llms/azure/chat/o_series_transformation.py index 2b8c2e88e51..86944c3b2c2 100644 --- a/litellm/llms/azure/chat/o_series_transformation.py +++ b/litellm/llms/azure/chat/o_series_transformation.py @@ -91,7 +91,7 @@ class AzureOpenAIO1Config(OpenAIOSeriesConfig): ): # allow user to override default with model_info={"supports_native_streaming": true} return False except Exception as e: - verbose_logger.debug(f"Error getting model info in AzureOpenAIO1Config: {e}") + verbose_logger.debug("Error getting model info in AzureOpenAIO1Config: %s", e) return True def is_o_series_model(self, model: str) -> bool: diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index 8db422e00ff..45f683d5962 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -333,7 +333,8 @@ def get_azure_ad_token( verbose_logger.debug("Azure AD Token Provider could not be used.") except Exception as e: verbose_logger.error( - f"Error calling Azure AD token provider: {e}. Follow docs - https://docs.litellm.ai/docs/providers/azure/#azure-ad-token-refresh---defaultazurecredential" + "Error calling Azure AD token provider: %s. Follow docs - https://docs.litellm.ai/docs/providers/azure/#azure-ad-token-refresh---defaultazurecredential", + e, ) raise e @@ -351,7 +352,7 @@ def get_azure_ad_token( try: token = azure_ad_token_provider() if not isinstance(token, str): - verbose_logger.error(f"Azure AD token provider returned non-string value: {type(token)}") + verbose_logger.error("Azure AD token provider returned non-string value: %s", type(token)) raise TypeError(f"Azure AD token must be a string, got {type(token)}") else: azure_ad_token = token @@ -359,7 +360,7 @@ def get_azure_ad_token( # Re-raise TypeError directly raise except Exception as e: - verbose_logger.error(f"Error calling Azure AD token provider: {e}") + verbose_logger.error("Error calling Azure AD token provider: %s", e) raise RuntimeError(f"Failed to get Azure AD token: {e}") from e return azure_ad_token @@ -393,7 +394,7 @@ class BaseAzureLLM(BaseOpenAILLM): verbose_logger.debug("Successfully obtained Azure AD token provider using DefaultAzureCredential") return azure_ad_token_provider except Exception as e: - verbose_logger.debug(f"DefaultAzureCredential failed: {e}") + verbose_logger.debug("DefaultAzureCredential failed: %s", e) return None def get_azure_openai_client( @@ -481,7 +482,7 @@ class BaseAzureLLM(BaseOpenAILLM): if "http_client" in azure_client_params: v1_params["http_client"] = azure_client_params["http_client"] - verbose_logger.debug(f"Using Azure v1 API with base_url: {v1_params['base_url']}") + verbose_logger.debug("Using Azure v1 API with base_url: %s", v1_params["base_url"]) if _is_async is True: openai_client = AsyncOpenAI(**v1_params) # type: ignore @@ -582,7 +583,7 @@ class BaseAzureLLM(BaseOpenAILLM): # only show first 5 chars of api_key _api_key = _api_key[:8] + "*" * 15 verbose_logger.debug( - f"Initializing Azure OpenAI Client for {model_name}, Api Base: {api_base}, Api Key:{_api_key}" + "Initializing Azure OpenAI Client for %s, Api Base: %s, Api Key:%s", model_name, api_base, _api_key ) azure_client_params = { "api_key": api_key, diff --git a/litellm/llms/azure/cost_calculation.py b/litellm/llms/azure/cost_calculation.py index 6fddb8523e7..ab4307f9172 100644 --- a/litellm/llms/azure/cost_calculation.py +++ b/litellm/llms/azure/cost_calculation.py @@ -35,7 +35,10 @@ def cost_per_token( and response_time_ms is not None ): verbose_logger.debug( - f"For model={model} - output_cost_per_second: {model_info.get('output_cost_per_second')}; response time: {response_time_ms}" + "For model=%s - output_cost_per_second: %s; response time: %s", + model, + model_info.get("output_cost_per_second"), + response_time_ms, ) ## COST PER SECOND ## prompt_cost = 0.0 diff --git a/litellm/llms/azure/image_generation/__init__.py b/litellm/llms/azure/image_generation/__init__.py index 64636bc689d..a2a905f2287 100644 --- a/litellm/llms/azure/image_generation/__init__.py +++ b/litellm/llms/azure/image_generation/__init__.py @@ -29,6 +29,6 @@ def get_azure_image_generation_config(model: str) -> BaseImageGenerationConfig: return AzureFoundryMAIImageGenerationConfig() else: verbose_logger.debug( - f"Using AzureGPTImageGenerationConfig for model: {model}. This follows the gpt-image model format." + "Using AzureGPTImageGenerationConfig for model: %s. This follows the gpt-image model format.", model ) return AzureGPTImageGenerationConfig() diff --git a/litellm/llms/azure/responses/o_series_transformation.py b/litellm/llms/azure/responses/o_series_transformation.py index 7a88c42cb14..121ceda7fa1 100644 --- a/litellm/llms/azure/responses/o_series_transformation.py +++ b/litellm/llms/azure/responses/o_series_transformation.py @@ -68,7 +68,7 @@ class AzureOpenAIOSeriesResponsesAPIConfig(AzureOpenAIResponsesAPIConfig): # If drop_params is enabled, remove temperature parameter for O-series models if drop_params and "temperature" in mapped_params: verbose_logger.debug( - f"Dropping unsupported parameter 'temperature' for Azure OpenAI O-series responses API model {model}" + "Dropping unsupported parameter 'temperature' for Azure OpenAI O-series responses API model %s", model ) mapped_params.pop("temperature", None) diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 860b1b1dd5c..51b63b6299c 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -74,7 +74,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): return dict_reasoning_item except Exception as e: - verbose_logger.debug(f"Failed to create ResponseReasoningItem, falling back to manual filtering: {e}") + verbose_logger.debug("Failed to create ResponseReasoningItem, falling back to manual filtering: %s", e) # Fallback: manually filter out known None fields filtered_item = { k: v @@ -252,7 +252,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): delete_url = self._construct_url_for_response_id_in_path(api_base=api_base, response_id=response_id) data: dict = {} - verbose_logger.debug(f"delete response url={delete_url}") + verbose_logger.debug("delete response url=%s", delete_url) return delete_url, data ######################################################### @@ -273,7 +273,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): """ get_url = self._construct_url_for_response_id_in_path(api_base=api_base, response_id=response_id) data: dict = {} - verbose_logger.debug(f"get response url={get_url}") + verbose_logger.debug("get response url=%s", get_url) return get_url, data def transform_list_input_items_request( @@ -302,7 +302,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): params["limit"] = limit if order is not None: params["order"] = order - verbose_logger.debug(f"list input items url={url}") + verbose_logger.debug("list input items url=%s", url) return url, params ######################################################### @@ -329,7 +329,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): ) data: dict = {} - verbose_logger.debug(f"cancel response url={cancel_url}") + verbose_logger.debug("cancel response url=%s", cancel_url) return cancel_url, data def transform_cancel_response_api_response( diff --git a/litellm/llms/azure_ai/agents/handler.py b/litellm/llms/azure_ai/agents/handler.py index 7023dbca0b8..81532aed208 100644 --- a/litellm/llms/azure_ai/agents/handler.py +++ b/litellm/llms/azure_ai/agents/handler.py @@ -193,7 +193,7 @@ class AzureAIAgentsHandler: ), ) except Exception as e: - verbose_logger.warning(f"Failed to calculate token usage: {e}") + verbose_logger.warning("Failed to calculate token usage: %s", e) return model_response @@ -226,7 +226,7 @@ class AzureAIAgentsHandler: thread_id = optional_params.get("thread_id") api_base = api_base.rstrip("/") - verbose_logger.debug(f"Azure AI Agents completion - api_base: {api_base}, agent_id: {agent_id}") + verbose_logger.debug("Azure AI Agents completion - api_base: %s, agent_id: %s", api_base, agent_id) return headers, api_version, agent_id, thread_id, api_base @@ -305,11 +305,11 @@ class AzureAIAgentsHandler: # Step 1: Create thread if not provided if not thread_id: - verbose_logger.debug(f"Creating thread at: {self._build_thread_url(api_base, api_version)}") + verbose_logger.debug("Creating thread at: %s", self._build_thread_url(api_base, api_version)) response = make_request("POST", self._build_thread_url(api_base, api_version), {}) self._check_response(response, [200, 201], "Failed to create thread") thread_id = response.json()["id"] - verbose_logger.debug(f"Created thread: {thread_id}") + verbose_logger.debug("Created thread: %s", thread_id) # At this point thread_id is guaranteed to be a string assert thread_id is not None @@ -329,7 +329,7 @@ class AzureAIAgentsHandler: response = make_request("POST", self._build_runs_url(api_base, thread_id, api_version), run_payload) self._check_response(response, [200, 201], "Failed to create run") run_id = response.json()["id"] - verbose_logger.debug(f"Created run: {run_id}") + verbose_logger.debug("Created run: %s", run_id) # Step 4: Poll for completion status_url = self._build_run_status_url(api_base, thread_id, run_id, api_version) @@ -338,7 +338,7 @@ class AzureAIAgentsHandler: self._check_response(response, [200], "Failed to get run status") status = response.json().get("status") - verbose_logger.debug(f"Run status: {status}") + verbose_logger.debug("Run status: %s", status) if status == "completed": break @@ -428,11 +428,11 @@ class AzureAIAgentsHandler: # Step 1: Create thread if not provided if not thread_id: - verbose_logger.debug(f"Creating thread at: {self._build_thread_url(api_base, api_version)}") + verbose_logger.debug("Creating thread at: %s", self._build_thread_url(api_base, api_version)) response = await make_request("POST", self._build_thread_url(api_base, api_version), {}) self._check_response(response, [200, 201], "Failed to create thread") thread_id = response.json()["id"] - verbose_logger.debug(f"Created thread: {thread_id}") + verbose_logger.debug("Created thread: %s", thread_id) # At this point thread_id is guaranteed to be a string assert thread_id is not None @@ -452,7 +452,7 @@ class AzureAIAgentsHandler: response = await make_request("POST", self._build_runs_url(api_base, thread_id, api_version), run_payload) self._check_response(response, [200, 201], "Failed to create run") run_id = response.json()["id"] - verbose_logger.debug(f"Created run: {run_id}") + verbose_logger.debug("Created run: %s", run_id) # Step 4: Poll for completion status_url = self._build_run_status_url(api_base, thread_id, run_id, api_version) @@ -461,7 +461,7 @@ class AzureAIAgentsHandler: self._check_response(response, [200], "Failed to get run status") status = response.json().get("status") - verbose_logger.debug(f"Run status: {status}") + verbose_logger.debug("Run status: %s", status) if status == "completed": break @@ -526,7 +526,7 @@ class AzureAIAgentsHandler: payload["instructions"] = optional_params["instructions"] url = self._build_create_thread_and_run_url(api_base, api_version) - verbose_logger.debug(f"Azure AI Agents streaming - URL: {url}") + verbose_logger.debug("Azure AI Agents streaming - URL: %s", url) # Use LiteLLM's async HTTP client for streaming client = get_async_httpx_client( @@ -607,7 +607,7 @@ class AzureAIAgentsHandler: # Extract thread_id from thread.created event if current_event == "thread.created" and "id" in data: thread_id = data["id"] - verbose_logger.debug(f"Stream created thread: {thread_id}") + verbose_logger.debug("Stream created thread: %s", thread_id) # Extract annotations from completed message if current_event == "thread.message.completed": diff --git a/litellm/llms/azure_ai/agents/transformation.py b/litellm/llms/azure_ai/agents/transformation.py index fb18b0cb651..7b6c8f0eefb 100644 --- a/litellm/llms/azure_ai/agents/transformation.py +++ b/litellm/llms/azure_ai/agents/transformation.py @@ -229,7 +229,7 @@ class AzureAIAgentsConfig(BaseConfig): if "instructions" in optional_params: payload["instructions"] = optional_params["instructions"] - verbose_logger.debug(f"Azure AI Agents request payload: {payload}") + verbose_logger.debug("Azure AI Agents request payload: %s", payload) return payload def validate_environment( diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py index 65d8c0182ee..fca2244265b 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py @@ -56,7 +56,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): # Validate the request self.validate_request(model, messages) - verbose_logger.debug(f"Processing Azure AI Anthropic CountTokens request for model: {model}") + verbose_logger.debug("Processing Azure AI Anthropic CountTokens request for model: %s", model) # Transform request to Anthropic format request_body = self.transform_request_to_count_tokens( @@ -66,12 +66,12 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): system=system, ) - verbose_logger.debug(f"Transformed request: {request_body}") + verbose_logger.debug("Transformed request: %s", request_body) # Get endpoint URL endpoint_url = self.get_count_tokens_endpoint(api_base) - verbose_logger.debug(f"Making request to: {endpoint_url}") + verbose_logger.debug("Making request to: %s", endpoint_url) # Get required headers with Azure authentication headers = self.get_required_headers( @@ -92,11 +92,11 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): timeout=request_timeout, ) - verbose_logger.debug(f"Response status: {response.status_code}") + verbose_logger.debug("Response status: %s", response.status_code) if response.status_code != 200: error_text = response.text - verbose_logger.error(f"Azure AI Anthropic API error: {error_text}") + verbose_logger.error("Azure AI Anthropic API error: %s", error_text) raise AnthropicError( status_code=response.status_code, message=error_text, @@ -104,7 +104,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): azure_response = response.json() - verbose_logger.debug(f"Azure AI Anthropic response: {azure_response}") + verbose_logger.debug("Azure AI Anthropic response: %s", azure_response) # Return Anthropic-compatible response directly - no transformation needed return azure_response @@ -114,13 +114,13 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): raise except httpx.HTTPStatusError as e: # HTTP errors - preserve the actual status code - verbose_logger.error(f"HTTP error in CountTokens handler: {e}") + verbose_logger.error("HTTP error in CountTokens handler: %s", e) raise AnthropicError( status_code=e.response.status_code, message=e.response.text, ) except Exception as e: - verbose_logger.error(f"Error in CountTokens handler: {e}") + verbose_logger.error("Error in CountTokens handler: %s", e) raise AnthropicError( status_code=500, message=f"CountTokens processing error: {e}", diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py index 129d7bb7aa9..277c07584ed 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py @@ -95,7 +95,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): ) except AnthropicError as e: verbose_logger.warning( - f"Azure AI Anthropic CountTokens API error: status={e.status_code}, message={e.message}" + "Azure AI Anthropic CountTokens API error: status=%s, message=%s", e.status_code, e.message ) return TokenCountResponse( total_tokens=0, @@ -107,7 +107,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): status_code=e.status_code, ) except Exception as e: - verbose_logger.warning(f"Error calling Azure AI Anthropic CountTokens API: {e}") + verbose_logger.warning("Error calling Azure AI Anthropic CountTokens API: %s", e) return TokenCountResponse( total_tokens=0, request_model=request_model, diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 707ddc9e12b..083bed024f7 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -206,7 +206,7 @@ class AzureAIStudioConfig(OpenAIConfig): dynamic_api_key = api_key or get_secret_str("AZURE_AI_API_KEY") if self._is_azure_openai_model(model=model, api_base=api_base): - verbose_logger.debug(f"Model={model} is Azure OpenAI model. Setting custom_llm_provider='azure'.") + verbose_logger.debug("Model=%s is Azure OpenAI model. Setting custom_llm_provider='azure'.", model) custom_llm_provider = "azure" return api_base, dynamic_api_key, custom_llm_provider diff --git a/litellm/llms/azure_ai/cost_calculator.py b/litellm/llms/azure_ai/cost_calculator.py index 6cc0cb20e27..3642fb37e86 100644 --- a/litellm/llms/azure_ai/cost_calculator.py +++ b/litellm/llms/azure_ai/cost_calculator.py @@ -107,7 +107,7 @@ def cost_per_token( # Re-raise for non-router models - they should have pricing defined raise verbose_logger.debug( - f"Azure AI Model Router: model '{model}' not in cost map, calculating routing flat cost only. Error: {e}" + "Azure AI Model Router: model '%s' not in cost map, calculating routing flat cost only. Error: %s", model, e ) # Add flat cost for Azure Model Router diff --git a/litellm/llms/azure_ai/image_generation/__init__.py b/litellm/llms/azure_ai/image_generation/__init__.py index fd511654665..742922ad295 100644 --- a/litellm/llms/azure_ai/image_generation/__init__.py +++ b/litellm/llms/azure_ai/image_generation/__init__.py @@ -32,6 +32,6 @@ def get_azure_ai_image_generation_config(model: str) -> BaseImageGenerationConfi return AzureFoundryFluxImageGenerationConfig() else: verbose_logger.debug( - f"Using AzureGPTImageGenerationConfig for model: {model}. This follows the gpt-image-1 model format." + "Using AzureGPTImageGenerationConfig for model: %s. This follows the gpt-image-1 model format.", model ) return AzureFoundryGPTImageGenerationConfig() diff --git a/litellm/llms/azure_ai/ocr/common_utils.py b/litellm/llms/azure_ai/ocr/common_utils.py index 14b77338fd7..fbeb8b8f4bd 100644 --- a/litellm/llms/azure_ai/ocr/common_utils.py +++ b/litellm/llms/azure_ai/ocr/common_utils.py @@ -53,9 +53,9 @@ def get_azure_ai_ocr_config(model: str) -> Optional["BaseOCRConfig"]: # Check for Azure Document Intelligence models if is_azure_document_intelligence_model(model): - verbose_logger.debug(f"Routing {model} to Azure Document Intelligence OCR config") + verbose_logger.debug("Routing %s to Azure Document Intelligence OCR config", model) return AzureDocumentIntelligenceOCRConfig() # Default to Mistral-based OCR for other azure_ai models - verbose_logger.debug(f"Routing {model} to Azure AI (Mistral) OCR config") + verbose_logger.debug("Routing %s to Azure AI (Mistral) OCR config", model) return AzureAIOCRConfig() diff --git a/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py b/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py index 66ce84cea0f..503b58a44b7 100644 --- a/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py +++ b/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py @@ -353,7 +353,7 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): Returns: OCRRequestData with JSON data """ - verbose_logger.debug(f"Azure Document Intelligence transform_ocr_request - model: {model}") + verbose_logger.debug("Azure Document Intelligence transform_ocr_request - model: %s", model) if not isinstance(document, dict): raise ValueError(f"Expected document dict, got {type(document)}") @@ -455,7 +455,7 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): Retry-after duration in seconds (default: 2) """ retry_after = int(response.headers.get("retry-after", "2")) - verbose_logger.debug(f"Retry polling after: {retry_after} seconds") + verbose_logger.debug("Retry polling after: %s seconds", retry_after) return retry_after @staticmethod @@ -476,7 +476,7 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): result = response.json() status = result.get("status") - verbose_logger.debug(f"Azure DI operation status: {status}") + verbose_logger.debug("Azure DI operation status: %s", status) if status == "succeeded": return "succeeded" @@ -519,7 +519,7 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): client = _get_httpx_client() start_time = time.time() - verbose_logger.debug(f"Polling Azure DI operation: {operation_url}") + verbose_logger.debug("Polling Azure DI operation: %s", operation_url) while True: self._check_timeout(start_time=start_time, timeout_secs=timeout_secs) @@ -560,7 +560,7 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): client = get_async_httpx_client(llm_provider=litellm.LlmProviders.AZURE_AI) start_time = time.time() - verbose_logger.debug(f"Polling Azure DI operation (async): {operation_url}") + verbose_logger.debug("Polling Azure DI operation (async): %s", operation_url) while True: self._check_timeout(start_time=start_time, timeout_secs=timeout_secs) @@ -603,7 +603,7 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): """ operation = AzureDocumentIntelligenceOperation.model_validate(raw_response.json()) - verbose_logger.debug(f"Azure Document Intelligence response status: {operation.status}") + verbose_logger.debug("Azure Document Intelligence response status: %s", operation.status) if operation.status != "succeeded": raise ValueError(f"Azure Document Intelligence analysis failed with status: {operation.status}") diff --git a/litellm/llms/azure_ai/ocr/transformation.py b/litellm/llms/azure_ai/ocr/transformation.py index d757a7f1378..36f7159c0d4 100644 --- a/litellm/llms/azure_ai/ocr/transformation.py +++ b/litellm/llms/azure_ai/ocr/transformation.py @@ -117,13 +117,13 @@ class AzureAIOCRConfig(MistralOCRConfig): Returns: Base64 data URI string """ - verbose_logger.debug(f"Azure AI OCR: Converting URL to base64 data URI (sync): {url}") + verbose_logger.debug("Azure AI OCR: Converting URL to base64 data URI (sync): %s", url) # Fetch and convert to base64 data URI # convert_url_to_base64 already returns a full data URI like "data:image/jpeg;base64,..." data_uri = convert_url_to_base64(url=url) - verbose_logger.debug(f"Azure AI OCR: Converted URL to data URI (length: {len(data_uri)})") + verbose_logger.debug("Azure AI OCR: Converted URL to data URI (length: %s)", len(data_uri)) return data_uri @@ -140,13 +140,13 @@ class AzureAIOCRConfig(MistralOCRConfig): Returns: Base64 data URI string """ - verbose_logger.debug(f"Azure AI OCR: Converting URL to base64 data URI (async): {url}") + verbose_logger.debug("Azure AI OCR: Converting URL to base64 data URI (async): %s", url) # Fetch and convert to base64 data URI asynchronously # async_convert_url_to_base64 already returns a full data URI like "data:image/jpeg;base64,..." data_uri = await async_convert_url_to_base64(url=url) - verbose_logger.debug(f"Azure AI OCR: Converted URL to data URI (length: {len(data_uri)})") + verbose_logger.debug("Azure AI OCR: Converted URL to data URI (length: %s)", len(data_uri)) return data_uri @@ -174,7 +174,7 @@ class AzureAIOCRConfig(MistralOCRConfig): Returns: OCRRequestData with JSON data """ - verbose_logger.debug(f"Azure AI OCR transform_ocr_request (sync) - model: {model}") + verbose_logger.debug("Azure AI OCR transform_ocr_request (sync) - model: %s", model) if not isinstance(document, dict): raise ValueError(f"Expected document dict, got {type(document)}") @@ -231,7 +231,7 @@ class AzureAIOCRConfig(MistralOCRConfig): Returns: OCRRequestData with JSON data """ - verbose_logger.debug(f"Azure AI OCR async_transform_ocr_request - model: {model}") + verbose_logger.debug("Azure AI OCR async_transform_ocr_request - model: %s", model) if not isinstance(document, dict): raise ValueError(f"Expected document dict, got {type(document)}") diff --git a/litellm/llms/base_llm/files/azure_blob_storage_backend.py b/litellm/llms/base_llm/files/azure_blob_storage_backend.py index 33255657287..86229a16a63 100644 --- a/litellm/llms/base_llm/files/azure_blob_storage_backend.py +++ b/litellm/llms/base_llm/files/azure_blob_storage_backend.py @@ -129,11 +129,11 @@ class AzureBlobStorageBackend(BaseFileStorageBackend, AzureBlobStorageLogger): full_path=full_path, ) - verbose_logger.debug(f"Successfully uploaded file to Azure Blob Storage: {storage_url}") + verbose_logger.debug("Successfully uploaded file to Azure Blob Storage: %s", storage_url) return storage_url except Exception as e: - verbose_logger.exception(f"Error uploading file to Azure Blob Storage: {e}") + verbose_logger.exception("Error uploading file to Azure Blob Storage: %s", e) raise async def _upload_file_with_account_key(self, file_content: bytes, full_path: str) -> str: @@ -145,7 +145,7 @@ class AzureBlobStorageBackend(BaseFileStorageBackend, AzureBlobStorageLogger): # Create filesystem (container) if it doesn't exist if not await file_system_client.exists(): await file_system_client.create_file_system() - verbose_logger.debug(f"Created filesystem: {self.azure_storage_file_system}") + verbose_logger.debug("Created filesystem: %s", self.azure_storage_file_system) # Extract directory and filename (similar to logger's pattern) path_parts = full_path.split("/") @@ -157,7 +157,7 @@ class AzureBlobStorageBackend(BaseFileStorageBackend, AzureBlobStorageLogger): directory_client = file_system_client.get_directory_client(directory_path) if not await directory_client.exists(): await directory_client.create_directory() - verbose_logger.debug(f"Created directory: {directory_path}") + verbose_logger.debug("Created directory: %s", directory_path) # Get file client from directory (same pattern as logger) file_client = directory_client.get_file_client(file_name) @@ -247,7 +247,7 @@ class AzureBlobStorageBackend(BaseFileStorageBackend, AzureBlobStorageLogger): return await self._download_file_with_azure_ad(file_path) except Exception as e: - verbose_logger.exception(f"Error downloading file from Azure Blob Storage: {e}") + verbose_logger.exception("Error downloading file from Azure Blob Storage: %s", e) raise async def _download_file_with_account_key(self, file_path: str) -> bytes: diff --git a/litellm/llms/base_llm/files/storage_backend_factory.py b/litellm/llms/base_llm/files/storage_backend_factory.py index 8fd918af0dc..0cf8164bc4a 100644 --- a/litellm/llms/base_llm/files/storage_backend_factory.py +++ b/litellm/llms/base_llm/files/storage_backend_factory.py @@ -29,7 +29,7 @@ def get_storage_backend(backend_type: str) -> BaseFileStorageBackend: Raises: ValueError: If backend_type is not supported """ - verbose_logger.debug(f"Creating storage backend: type={backend_type}") + verbose_logger.debug("Creating storage backend: type=%s", backend_type) if backend_type == "azure_storage": return AzureBlobStorageBackend() diff --git a/litellm/llms/base_llm/managed_resources/base_managed_resource.py b/litellm/llms/base_llm/managed_resources/base_managed_resource.py index fd9eaf9801b..44ad555377b 100644 --- a/litellm/llms/base_llm/managed_resources/base_managed_resource.py +++ b/litellm/llms/base_llm/managed_resources/base_managed_resource.py @@ -156,7 +156,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]): user_api_key_dict: User API key authentication details additional_db_fields: Additional fields to store in database """ - verbose_logger.info(f"Storing LiteLLM Managed {self.resource_type} with id={unified_resource_id} in cache") + verbose_logger.info("Storing LiteLLM Managed %s with id=%s in cache", self.resource_type, unified_resource_id) # Prepare cache data cache_data = { @@ -215,7 +215,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]): result = await table.create(data=db_data) verbose_logger.debug( - f"LiteLLM Managed {self.resource_type} with id={unified_resource_id} stored in db: {result}" + "LiteLLM Managed %s with id=%s stored in db: %s", self.resource_type, unified_resource_id, result ) async def get_unified_resource_id( @@ -579,7 +579,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]): except Exception as e: verbose_logger.warning( - f"Failed to parse {self.resource_type} object {resource.unified_resource_id}: {e}" + "Failed to parse %s object %s: %s", self.resource_type, resource.unified_resource_id, e ) continue diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index d7b7806a4a8..bb5ee111f2e 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -816,7 +816,10 @@ class BaseAWSLLM: import boto3 verbose_logger.debug( - f"IN Web Identity Token: {aws_web_identity_token} | Role Name: {aws_role_name} | Session Name: {aws_session_name}" + "IN Web Identity Token: %s | Role Name: %s | Session Name: %s", + aws_web_identity_token, + aws_role_name, + aws_session_name, ) # get_secret() expands environment-variable references (an os.environ/ @@ -932,7 +935,8 @@ class BaseAWSLLM: if sts_response["PackedPolicySize"] > BEDROCK_MAX_POLICY_SIZE: verbose_logger.warning( - f"The policy size is greater than 75% of the allowed size, PackedPolicySize: {sts_response['PackedPolicySize']}" + "The policy size is greater than 75%% of the allowed size, PackedPolicySize: %s", + sts_response["PackedPolicySize"], ) with tracer.trace("boto3.Session(**iam_creds_dict)"): @@ -970,7 +974,7 @@ class BaseAWSLLM: sts_client = boto3.client("sts", **irsa_sts_kwargs) # Manually assume the IRSA role with the session name - verbose_logger.debug(f"Manually assuming IRSA role {irsa_role_arn} with session {aws_session_name}") + verbose_logger.debug("Manually assuming IRSA role %s with session %s", irsa_role_arn, aws_session_name) irsa_response = sts_client.assume_role_with_web_identity( RoleArn=irsa_role_arn, RoleSessionName=aws_session_name, @@ -994,13 +998,13 @@ class BaseAWSLLM: try: caller_identity = sts_client_with_creds.get_caller_identity() verbose_logger.debug( - f"Current identity after manual IRSA assumption: {caller_identity.get('Arn', 'unknown')}" + "Current identity after manual IRSA assumption: %s", caller_identity.get("Arn", "unknown") ) except Exception as e: - verbose_logger.debug(f"Failed to get caller identity: {e}") + verbose_logger.debug("Failed to get caller identity: %s", e) # Now assume the target role - verbose_logger.debug(f"Attempting to assume target role: {aws_role_name} with session: {aws_session_name}") + verbose_logger.debug("Attempting to assume target role: %s with session: %s", aws_role_name, aws_session_name) assume_role_params = { "RoleArn": aws_role_name, "RoleSessionName": aws_session_name, @@ -1035,12 +1039,12 @@ class BaseAWSLLM: # Get current caller identity for debugging try: caller_identity = sts_client.get_caller_identity() - verbose_logger.debug(f"Current IRSA identity: {caller_identity.get('Arn', 'unknown')}") + verbose_logger.debug("Current IRSA identity: %s", caller_identity.get("Arn", "unknown")) except Exception as e: - verbose_logger.debug(f"Failed to get caller identity: {e}") + verbose_logger.debug("Failed to get caller identity: %s", e) # Assume the role - verbose_logger.debug(f"Attempting to assume role: {aws_role_name} with session: {aws_session_name}") + verbose_logger.debug("Attempting to assume role: %s with session: %s", aws_role_name, aws_session_name) assume_role_params = { "RoleArn": aws_role_name, "RoleSessionName": aws_session_name, @@ -1142,7 +1146,7 @@ class BaseAWSLLM: if web_identity_token_file and irsa_role_arn and aws_access_key_id is None and aws_secret_access_key is None: # For cross-account role assumption with specific session names, # we need to manually assume the IRSA role first with the correct session name - verbose_logger.debug(f"IRSA detected: using web identity token from {web_identity_token_file}") + verbose_logger.debug("IRSA detected: using web identity token from %s", web_identity_token_file) try: # Check if we need to do cross-account role assumption @@ -1168,13 +1172,13 @@ class BaseAWSLLM: return self._extract_credentials_and_ttl(sts_response) except Exception as e: - verbose_logger.debug(f"Failed to assume role via IRSA: {e}") + verbose_logger.debug("Failed to assume role via IRSA: %s", e) if "AccessDenied" in str(e) and "is not authorized to perform: sts:AssumeRole" in str(e): # Provide a more helpful error message for trust policy issues verbose_logger.error( - f"Access denied when trying to assume role {aws_role_name}. " - f"Please ensure the trust policy of {aws_role_name} allows " - f"the current role to assume it. Current identity: check logs with verbose mode." + "Access denied when trying to assume role %s. Please ensure the trust policy of %s allows the current role to assume it. Current identity: check logs with verbose mode.", + aws_role_name, + aws_role_name, ) # Re-raise the exception instead of falling through raise diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index d6626562393..7efad7a906d 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -131,7 +131,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): jwt_token = api_key or optional_params.get("api_key") if jwt_token: verbose_logger.debug( - f"AgentCore: Using Bearer token authentication (Cognito/JWT) - token: {jwt_token[:50]}..." + "AgentCore: Using Bearer token authentication (Cognito/JWT) - token: %s...", jwt_token[:50] ) headers["Content-Type"] = "application/json" headers["Authorization"] = f"Bearer {jwt_token}" @@ -182,12 +182,12 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): """ session_id = optional_params.get("runtimeSessionId", None) if session_id: - verbose_logger.debug(f"Using provided runtimeSessionId: {session_id}") + verbose_logger.debug("Using provided runtimeSessionId: %s", session_id) return session_id # Generate a session ID with 33+ characters generated_id = f"litellm-session-{uuid.uuid4()}" - verbose_logger.debug(f"Generated new session ID: {generated_id}") + verbose_logger.debug("Generated new session ID: %s", generated_id) return generated_id def _get_runtime_user_id(self, optional_params: dict) -> str | None: @@ -196,7 +196,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): """ user_id = optional_params.get("runtimeUserId", None) if user_id: - verbose_logger.debug(f"Using provided runtimeUserId: {user_id}") + verbose_logger.debug("Using provided runtimeUserId: %s", user_id) return user_id def transform_request( @@ -231,7 +231,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): dict: Payload dict containing the prompt and (optionally) the OpenAI content list. """ - verbose_logger.debug(f"AgentCore transform_request - optional_params keys: {list(optional_params.keys())}") + verbose_logger.debug("AgentCore transform_request - optional_params keys: %s", list(optional_params.keys())) # Use the last message content as the prompt prompt = convert_content_list_to_str(messages[-1]) @@ -264,7 +264,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): # The request data is the payload dict (will be JSON encoded by the HTTP handler) # Qualifier will be handled as a query parameter in get_complete_url - verbose_logger.debug(f"PAYLOAD: {payload}") + verbose_logger.debug("PAYLOAD: %s", payload) return payload @staticmethod @@ -302,7 +302,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): # Skip non-dict data (some lines contain JSON strings) return data if isinstance(data, dict) else None except json.JSONDecodeError: - verbose_logger.debug(f"Skipping non-JSON line: {line[:100]}") + verbose_logger.debug("Skipping non-JSON line: %s", line[:100]) return None def _extract_usage_from_event(self, event_data: dict) -> AgentCoreUsage | None: @@ -361,7 +361,10 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): total_tokens = prompt_tokens + completion_tokens verbose_logger.debug( - f"Calculated usage - prompt: {prompt_tokens}, completion: {completion_tokens}, total: {total_tokens}" + "Calculated usage - prompt: %s, completion: %s, total: %s", + prompt_tokens, + completion_tokens, + total_tokens, ) return Usage( @@ -370,7 +373,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): total_tokens=total_tokens, ) except Exception as e: - verbose_logger.warning(f"Failed to calculate token usage: {e}") + verbose_logger.warning("Failed to calculate token usage: %s", e) return None def _parse_json_response(self, response_json: dict) -> AgentCoreParsedResponse: @@ -439,8 +442,8 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): # Strategy 4: fallback - return raw JSON as content verbose_logger.warning( - f"AgentCore: Could not extract content from JSON response keys " - f"{list(response_json.keys())}. Returning raw JSON as content." + "AgentCore: Could not extract content from JSON response keys %s. Returning raw JSON as content.", + list(response_json.keys()), ) return AgentCoreParsedResponse( content=json.dumps(response_json), @@ -459,20 +462,20 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): AgentCoreParsedResponse: Parsed response data """ content_type = raw_response.headers.get("content-type", "").lower() - verbose_logger.debug(f"AgentCore response Content-Type: {content_type}") + verbose_logger.debug("AgentCore response Content-Type: %s", content_type) # Parse response based on content type if "application/json" in content_type: # Direct JSON response verbose_logger.debug("Parsing JSON response") response_json = raw_response.json() - verbose_logger.debug(f"Response JSON: {response_json}") + verbose_logger.debug("Response JSON: %s", response_json) return self._parse_json_response(response_json) else: # SSE stream response (text/event-stream or default) verbose_logger.debug("Parsing SSE stream response") response_text = raw_response.text - verbose_logger.debug(f"AgentCore response (first 500 chars): {response_text[:500]}") + verbose_logger.debug("AgentCore response (first 500 chars): %s", response_text[:500]) return self._parse_sse_stream(response_text) def _parse_sse_stream(self, response_text: str) -> AgentCoreParsedResponse: @@ -496,7 +499,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): if not data: continue - verbose_logger.debug(f"SSE event keys: {list(data.keys())}") + verbose_logger.debug("SSE event keys: %s", list(data.keys())) # Check for final complete message if "message" in data and isinstance(data["message"], dict): @@ -506,12 +509,12 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): # Process event data if "event" in data and isinstance(data["event"], dict): event_payload = data["event"] - verbose_logger.debug(f"Event payload keys: {list(event_payload.keys())}") + verbose_logger.debug("Event payload keys: %s", list(event_payload.keys())) # Extract usage metadata if usage := self._extract_usage_from_event(data): usage_data = usage - verbose_logger.debug(f"Found usage data: {usage_data}") + verbose_logger.debug("Found usage data: %s", usage_data) # Collect content deltas if text := self._extract_content_delta(data): @@ -520,7 +523,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): # Build final content content = self._extract_content_from_message(final_message) if final_message else "".join(content_blocks) - verbose_logger.debug(f"Final usage_data: {usage_data}") + verbose_logger.debug("Final usage_data: %s", usage_data) return AgentCoreParsedResponse(content=content, usage=usage_data, final_message=final_message) @@ -624,7 +627,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): yield chunk except json.JSONDecodeError: - verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}") + verbose_logger.debug("Skipping non-JSON SSE line: %s", line[:100]) continue def get_sync_custom_stream_wrapper( @@ -651,7 +654,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): if client is None or not isinstance(client, HTTPHandler): client = _get_httpx_client(params={}) - verbose_logger.debug(f"Making sync streaming request to: {api_base}") + verbose_logger.debug("Making sync streaming request to: %s", api_base) # Make streaming request response = client.post( @@ -837,7 +840,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): yield chunk except json.JSONDecodeError: - verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}") + verbose_logger.debug("Skipping non-JSON SSE line: %s", line[:100]) continue async def get_async_custom_stream_wrapper( @@ -864,7 +867,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): if client is None or not isinstance(client, AsyncHTTPHandler): client = get_async_httpx_client(llm_provider=cast(Any, "bedrock"), params={}) - verbose_logger.debug(f"Making async streaming request to: {api_base}") + verbose_logger.debug("Making async streaming request to: %s", api_base) # Make async streaming request response = await client.post( @@ -990,8 +993,8 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): content = parsed_data["content"] usage_data = parsed_data["usage"] - verbose_logger.debug(f"Parsed content length: {len(content)}") - verbose_logger.debug(f"Usage data: {usage_data}") + verbose_logger.debug("Parsed content length: %s", len(content)) + verbose_logger.debug("Usage data: %s", usage_data) # Create the message message = Message(content=content, role="assistant") @@ -1023,7 +1026,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): return model_response except Exception as e: - verbose_logger.error(f"Error processing Bedrock AgentCore response: {e}") + verbose_logger.error("Error processing Bedrock AgentCore response: %s", e) raise BedrockError( message=f"Error processing response: {e}", status_code=raw_response.status_code, diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 2b34c9f2654..5096cc44b76 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -352,8 +352,8 @@ class AmazonConverseConfig(BaseConfig): # Model strings can be like: "amazon.nova-pro-v1:0", "us.amazon.nova-pro-v1:0", etc. if "nova" not in model.lower(): verbose_logger.debug( - f"web_search_options passed but model {model} is not a Nova model. " - "Nova grounding is only supported on Amazon Nova models." + "web_search_options passed but model %s is not a Nova model. Nova grounding is only supported on Amazon Nova models.", + model, ) return None @@ -950,8 +950,8 @@ class AmazonConverseConfig(BaseConfig): if isinstance(tool_choice_block, dict): if "any" in tool_choice_block or "tool" in tool_choice_block: verbose_logger.info( - f"{model} does not support forced tool use (tool_choice='required' or specific tool) " - f"when reasoning is enabled. Changing tool_choice to 'auto'." + "%s does not support forced tool use (tool_choice='required' or specific tool) when reasoning is enabled. Changing tool_choice to 'auto'.", + model, ) optional_params["tool_choice"] = ToolChoiceValuesBlock(auto={}) diff --git a/litellm/llms/bedrock/chat/invoke_agent/transformation.py b/litellm/llms/bedrock/chat/invoke_agent/transformation.py index da6224ec487..625ace20614 100644 --- a/litellm/llms/bedrock/chat/invoke_agent/transformation.py +++ b/litellm/llms/bedrock/chat/invoke_agent/transformation.py @@ -212,12 +212,12 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): } events.append(parsed_event) except json.JSONDecodeError as e: - verbose_logger.warning(f"Failed to parse trace event JSON: {e}") + verbose_logger.warning("Failed to parse trace event JSON: %s", e) else: - verbose_logger.debug(f"Unknown event type: {event_type}") + verbose_logger.debug("Unknown event type: %s", event_type) except Exception as e: - verbose_logger.error(f"Error processing event: {e}") + verbose_logger.error("Error processing event: %s", e) continue return events @@ -226,11 +226,11 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): """Extract message content from an AWS event, adapted from AWSEventStreamDecoder.""" try: response_dict = event.to_response_dict() - verbose_logger.debug(f"Response dict: {response_dict}") + verbose_logger.debug("Response dict: %s", response_dict) # Use the same response shape parsing as the existing decoder parsed_response = parser.parse(response_dict, self._get_response_stream_shape()) - verbose_logger.debug(f"Parsed response: {parsed_response}") + verbose_logger.debug("Parsed response: %s", parsed_response) if response_dict["status_code"] != 200: decoded_body = response_dict["body"].decode() @@ -259,7 +259,7 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): return chunk.decode() except Exception as e: - verbose_logger.debug(f"Error parsing message from event: {e}") + verbose_logger.debug("Error parsing message from event: %s", e) return None def _extract_headers_from_event(self, event) -> InvokeAgentEventHeaders: @@ -275,7 +275,7 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): message_type=headers.get(":message-type", ""), ) except Exception as e: - verbose_logger.debug(f"Error extracting headers: {e}") + verbose_logger.debug("Error extracting headers: %s", e) return InvokeAgentEventHeaders(event_type="", content_type="", message_type="") def _get_response_stream_shape(self): @@ -302,7 +302,7 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): decoded_content = base64.b64decode(encoded_bytes).decode("utf-8") response_parts.append(decoded_content) except Exception as e: - verbose_logger.warning(f"Failed to decode chunk content: {e}") + verbose_logger.warning("Failed to decode chunk content: %s", e) return "".join(response_parts) @@ -324,7 +324,7 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): if not trace_data: continue - verbose_logger.debug(f"Trace event: {trace_data}") + verbose_logger.debug("Trace event: %s", trace_data) # Extract usage from pre-processing trace self._extract_and_update_preprocessing_usage( @@ -443,11 +443,11 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): try: # Get the raw binary content raw_content = raw_response.content - verbose_logger.debug(f"Processing {len(raw_content)} bytes of AWS event stream data") + verbose_logger.debug("Processing %s bytes of AWS event stream data", len(raw_content)) # Parse the AWS event stream format events = self._parse_aws_event_stream(raw_content) - verbose_logger.debug(f"Parsed {len(events)} events from stream") + verbose_logger.debug("Parsed %s events from stream", len(events)) # Extract response content from chunk events content = self._extract_response_content(events) @@ -464,7 +464,7 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): ) except Exception as e: - verbose_logger.error(f"Error processing Bedrock Invoke Agent response: {e}") + verbose_logger.error("Error processing Bedrock Invoke Agent response: %s", e) raise BedrockError( message=f"Error processing response: {e}", status_code=raw_response.status_code, diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 4a429b639d2..a5ffe6abff4 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -530,7 +530,7 @@ class AWSEventStreamDecoder: # and use it as the consistent ID for all subsequent chunks. self._initialize_converse_response_id(chunk_data) - verbose_logger.debug(f"\n\nRaw Chunk: {chunk_data}\n\n") + verbose_logger.debug("\n\nRaw Chunk: %s\n\n", chunk_data) text = "" tool_use: ChatCompletionToolCallChunk | None = None finish_reason = "" diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py index 934d416d256..23998c62644 100644 --- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -88,7 +88,7 @@ class BedrockTokenCounter(BaseTokenCounter): original_response=result, ) except BedrockError as e: - verbose_logger.warning(f"Bedrock CountTokens API error: status={e.status_code}, message={e.message}") + verbose_logger.warning("Bedrock CountTokens API error: status=%s, message=%s", e.status_code, e.message) return TokenCountResponse( total_tokens=0, request_model=request_model, @@ -99,7 +99,7 @@ class BedrockTokenCounter(BaseTokenCounter): status_code=e.status_code, ) except Exception as e: - verbose_logger.warning(f"Error calling Bedrock CountTokens API: {e}") + verbose_logger.warning("Error calling Bedrock CountTokens API: %s", e) return TokenCountResponse( total_tokens=0, request_model=request_model, diff --git a/litellm/llms/bedrock/count_tokens/handler.py b/litellm/llms/bedrock/count_tokens/handler.py index 44cc535385d..100f0753b51 100644 --- a/litellm/llms/bedrock/count_tokens/handler.py +++ b/litellm/llms/bedrock/count_tokens/handler.py @@ -43,7 +43,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): # Validate the request self.validate_count_tokens_request(request_data) - verbose_logger.debug(f"Processing CountTokens request for resolved model: {resolved_model}") + verbose_logger.debug("Processing CountTokens request for resolved model: %s", resolved_model) # Get AWS region using existing LiteLLM function aws_region_name = self._get_aws_region_name( @@ -52,12 +52,12 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): model_id=None, ) - verbose_logger.debug(f"Retrieved AWS region: {aws_region_name}") + verbose_logger.debug("Retrieved AWS region: %s", aws_region_name) # Transform request to Bedrock format (supports both Converse and InvokeModel) bedrock_request = self.transform_anthropic_to_bedrock_count_tokens(request_data=request_data) - verbose_logger.debug(f"Transformed request: {bedrock_request}") + verbose_logger.debug("Transformed request: %s", bedrock_request) # Get endpoint URL using simplified function api_base = litellm_params.get("api_base", None) @@ -69,7 +69,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, ) - verbose_logger.debug(f"Making request to: {endpoint_url}") + verbose_logger.debug("Making request to: %s", endpoint_url) # Use existing _sign_request method from BaseAWSLLM # Extract api_key for bearer token auth if provided @@ -94,11 +94,11 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): timeout=30.0, ) - verbose_logger.debug(f"Response status: {response.status_code}") + verbose_logger.debug("Response status: %s", response.status_code) if response.status_code != 200: error_text = response.text - verbose_logger.error(f"AWS Bedrock error: {error_text}") + verbose_logger.error("AWS Bedrock error: %s", error_text) raise BedrockError( status_code=response.status_code, message=error_text, @@ -106,12 +106,12 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): bedrock_response = response.json() - verbose_logger.debug(f"Bedrock response: {bedrock_response}") + verbose_logger.debug("Bedrock response: %s", bedrock_response) # Transform response back to expected format final_response = self.transform_bedrock_response_to_anthropic(bedrock_response) - verbose_logger.debug(f"Final response: {final_response}") + verbose_logger.debug("Final response: %s", final_response) return final_response @@ -120,13 +120,13 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): raise except httpx.HTTPStatusError as e: # HTTP errors - preserve the actual status code - verbose_logger.error(f"HTTP error in CountTokens handler: {e}") + verbose_logger.error("HTTP error in CountTokens handler: %s", e) raise BedrockError( status_code=e.response.status_code, message=e.response.text, ) except Exception as e: - verbose_logger.error(f"Error in CountTokens handler: {e}") + verbose_logger.error("Error in CountTokens handler: %s", e) raise BedrockError( status_code=500, message=f"CountTokens processing error: {e}", diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index d3e61829681..9f61c50b25f 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -652,7 +652,8 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): ) except Exception as e: verbose_logger.exception( - f"litellm.llms.bedrock.files.transformation.py::_transform_openai_jsonl_content_to_bedrock_jsonl_content() - Error inferring custom_llm_provider - {e}" + "litellm.llms.bedrock.files.transformation.py::_transform_openai_jsonl_content_to_bedrock_jsonl_content() - Error inferring custom_llm_provider - %s", + e, ) # Determine provider from model name diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index a8969894dda..874052d2b29 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -83,7 +83,7 @@ class BedrockRealtime(BaseAWSLLM): else: endpoint_uri = f"https://bedrock-runtime.{aws_region_name}.amazonaws.com" - verbose_proxy_logger.debug(f"Bedrock Realtime: Connecting to {endpoint_uri} with model {model}") + verbose_proxy_logger.debug("Bedrock Realtime: Connecting to %s with model %s", endpoint_uri, model) credentials = self.get_credentials( aws_access_key_id=aws_access_key_id, @@ -173,7 +173,7 @@ class BedrockRealtime(BaseAWSLLM): ) except Exception as e: - verbose_proxy_logger.exception(f"Error in BedrockRealtime.async_realtime: {e}") + verbose_proxy_logger.exception("Error in BedrockRealtime.async_realtime: %s", e) try: await websocket.close(code=1011, reason=_redact_string(f"Internal error: {e}")) except Exception: @@ -200,13 +200,13 @@ class BedrockRealtime(BaseAWSLLM): value=BidirectionalInputPayloadPart(bytes_=bedrock_message.encode("utf-8")) ) await bedrock_stream.input_stream.send(event) - verbose_proxy_logger.debug(f"Bedrock Realtime: Sent to Bedrock: {bedrock_message[:200]}") + verbose_proxy_logger.debug("Bedrock Realtime: Sent to Bedrock: %s", bedrock_message[:200]) try: while True: # Receive message from client message = await client_ws.receive_text() - verbose_proxy_logger.debug(f"Bedrock Realtime: Received from client: {message[:200]}") + verbose_proxy_logger.debug("Bedrock Realtime: Received from client: %s", message[:200]) # Transform OpenAI format to Bedrock format transformed_messages = transformation_config.transform_realtime_request( @@ -237,7 +237,7 @@ class BedrockRealtime(BaseAWSLLM): ) except Exception as e: - verbose_proxy_logger.debug(f"Client to Bedrock forwarding ended: {e}", exc_info=True) + verbose_proxy_logger.debug("Client to Bedrock forwarding ended: %s", e, exc_info=True) for close_message in transformation_config.session_close_messages(): with contextlib.suppress(Exception): await send_to_bedrock(close_message) @@ -266,7 +266,7 @@ class BedrockRealtime(BaseAWSLLM): if result.value and result.value.bytes_: bedrock_response = result.value.bytes_.decode("utf-8") - verbose_proxy_logger.debug(f"Bedrock Realtime: Received from Bedrock: {bedrock_response[:200]}") + verbose_proxy_logger.debug("Bedrock Realtime: Received from Bedrock: %s", bedrock_response[:200]) # Transform Bedrock format to OpenAI format from litellm.types.realtime import RealtimeResponseTransformInput @@ -306,10 +306,10 @@ class BedrockRealtime(BaseAWSLLM): for openai_message in openai_messages: message_json = json.dumps(openai_message) await client_ws.send_text(message_json) - verbose_proxy_logger.debug(f"Bedrock Realtime: Sent to client: {message_json[:200]}") + verbose_proxy_logger.debug("Bedrock Realtime: Sent to client: %s", message_json[:200]) except Exception as e: - verbose_proxy_logger.debug(f"Bedrock to client forwarding ended: {e}", exc_info=True) + verbose_proxy_logger.debug("Bedrock to client forwarding ended: %s", e, exc_info=True) finally: # Close the client WebSocket try: diff --git a/litellm/llms/bedrock/realtime/transformation.py b/litellm/llms/bedrock/realtime/transformation.py index 39f5d25cf89..68782c8b412 100644 --- a/litellm/llms/bedrock/realtime/transformation.py +++ b/litellm/llms/bedrock/realtime/transformation.py @@ -601,7 +601,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): try: json_message = json.loads(message) except json.JSONDecodeError: - verbose_logger.warning(f"Invalid JSON message: {message[:200]}") + verbose_logger.warning("Invalid JSON message: %s", message[:200]) return [] message_type = json_message.get("type") @@ -620,7 +620,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): elif message_type == "response.cancel": return self.transform_response_cancel_event(json_message) else: - verbose_logger.warning(f"Unknown message type: {message_type}") + verbose_logger.warning("Unknown message type: %s", message_type) return [] def _session_object( @@ -866,7 +866,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): Tuple of (events, reset_delta_chunks) """ content_end = event["contentEnd"] - verbose_logger.debug(f"Handling contentEnd: {content_end}") + verbose_logger.debug("Handling contentEnd: %s", content_end) if not current_output_item_id or not current_response_id: return [], current_delta_chunks @@ -1149,7 +1149,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): message_preview = ( message[:200].decode("utf-8", errors="replace") if isinstance(message, bytes) else message[:200] ) - verbose_logger.warning(f"Invalid JSON message: {message_preview}") + verbose_logger.warning("Invalid JSON message: %s", message_preview) return { "response": [], "current_output_item_id": realtime_response_transform_input.get("current_output_item_id"), @@ -1230,7 +1230,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): ) returned_messages.extend(events) # Store tool call info for potential use - verbose_logger.debug(f"Tool use event: {tool_name} (ID: {tool_call_id})") + verbose_logger.debug("Tool use event: %s (ID: %s)", tool_name, tool_call_id) elif "promptEnd" in event or "completionEnd" in event: ( diff --git a/litellm/llms/bedrock_mantle/chat/transformation.py b/litellm/llms/bedrock_mantle/chat/transformation.py index d85edbd0c86..bd1664ff95a 100644 --- a/litellm/llms/bedrock_mantle/chat/transformation.py +++ b/litellm/llms/bedrock_mantle/chat/transformation.py @@ -102,7 +102,7 @@ class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig): if "reasoning_effort" not in base_params: base_params.append("reasoning_effort") except Exception as e: - verbose_logger.debug(f"BedrockMantleChatConfig: error checking reasoning support: {e}") + verbose_logger.debug("BedrockMantleChatConfig: error checking reasoning support: %s", e) return base_params def get_model_response_iterator( diff --git a/litellm/llms/black_forest_labs/image_edit/handler.py b/litellm/llms/black_forest_labs/image_edit/handler.py index cf0cc31283b..6a6c01e8b24 100644 --- a/litellm/llms/black_forest_labs/image_edit/handler.py +++ b/litellm/llms/black_forest_labs/image_edit/handler.py @@ -342,7 +342,7 @@ class BlackForestLabsImageEdit: polling_headers = {"x-key": headers.get("x-key", "")} start_time = time.time() - verbose_logger.debug(f"BFL starting sync polling at {polling_url}") + verbose_logger.debug("BFL starting sync polling at %s", polling_url) while time.time() - start_time < max_wait: response = sync_client.get( @@ -359,7 +359,7 @@ class BlackForestLabsImageEdit: data = response.json() status = data.get("status") - verbose_logger.debug(f"BFL poll status: {status}") + verbose_logger.debug("BFL poll status: %s", status) if status == "Ready": return response @@ -433,7 +433,7 @@ class BlackForestLabsImageEdit: polling_headers = {"x-key": headers.get("x-key", "")} start_time = time.time() - verbose_logger.debug(f"BFL starting async polling at {polling_url}") + verbose_logger.debug("BFL starting async polling at %s", polling_url) while time.time() - start_time < max_wait: response = await async_client.get( @@ -450,7 +450,7 @@ class BlackForestLabsImageEdit: data = response.json() status = data.get("status") - verbose_logger.debug(f"BFL poll status: {status}") + verbose_logger.debug("BFL poll status: %s", status) if status == "Ready": return response diff --git a/litellm/llms/black_forest_labs/image_generation/handler.py b/litellm/llms/black_forest_labs/image_generation/handler.py index 054d28003f1..4df3b49d7f1 100644 --- a/litellm/llms/black_forest_labs/image_generation/handler.py +++ b/litellm/llms/black_forest_labs/image_generation/handler.py @@ -336,7 +336,7 @@ class BlackForestLabsImageGeneration: polling_headers = {"x-key": headers.get("x-key", "")} start_time = time.time() - verbose_logger.debug(f"BFL starting sync polling at {polling_url}") + verbose_logger.debug("BFL starting sync polling at %s", polling_url) while time.time() - start_time < max_wait: response = sync_client.get( @@ -353,7 +353,7 @@ class BlackForestLabsImageGeneration: data = response.json() status = data.get("status") - verbose_logger.debug(f"BFL poll status: {status}") + verbose_logger.debug("BFL poll status: %s", status) if status == "Ready": return response @@ -427,7 +427,7 @@ class BlackForestLabsImageGeneration: polling_headers = {"x-key": headers.get("x-key", "")} start_time = time.time() - verbose_logger.debug(f"BFL starting async polling at {polling_url}") + verbose_logger.debug("BFL starting async polling at %s", polling_url) while time.time() - start_time < max_wait: response = await async_client.get( @@ -444,7 +444,7 @@ class BlackForestLabsImageGeneration: data = response.json() status = data.get("status") - verbose_logger.debug(f"BFL poll status: {status}") + verbose_logger.debug("BFL poll status: %s", status) if status == "Ready": return response diff --git a/litellm/llms/custom_httpx/aiohttp_transport.py b/litellm/llms/custom_httpx/aiohttp_transport.py index ac7a8908616..9ac67272b3e 100644 --- a/litellm/llms/custom_httpx/aiohttp_transport.py +++ b/litellm/llms/custom_httpx/aiohttp_transport.py @@ -289,7 +289,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport): if self._owns_session: self._close_recycled_session(old_session) except Exception as e: - verbose_logger.debug(f"Error closing old session: {e}") + verbose_logger.debug("Error closing old session: %s", e) # Create a new session in the current event loop self.client = self._rebuild_session() @@ -302,9 +302,9 @@ class LiteLLMAiohttpTransport(AiohttpTransport): try: self._close_recycled_session(old_session) except (RuntimeError, AttributeError, OSError) as close_error: - verbose_logger.debug(f"Error closing old session: {close_error}") + verbose_logger.debug("Error closing old session: %s", close_error) self.client = self._rebuild_session() - verbose_logger.debug(f"Error checking session loop, created new session: {e}") + verbose_logger.debug("Error checking session loop, created new session: %s", e) return self.client @@ -397,7 +397,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport): except RuntimeError as e: # Handle the case where session was closed between our check and actual use if "Session is closed" in str(e): - verbose_logger.debug(f"Session closed during request, retrying with new session: {e}") + verbose_logger.debug("Session closed during request, retrying with new session: %s", e) # Dispose of the session that actually faulted. Do NOT read # self.client here: a concurrent task may already have # replaced it with a healthy session that must stay open. @@ -436,7 +436,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport): try: proxy = self._proxy_from_env(request.url) except Exception as e: # pragma: no cover - best effort - verbose_logger.debug(f"Error reading proxy env: {e}") + verbose_logger.debug("Error reading proxy env: %s", e) return proxy diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 3e34b483002..861adf919c0 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -215,19 +215,20 @@ def _create_ssl_context( if ssl_ecdh_curve and isinstance(ssl_ecdh_curve, str): try: custom_ssl_context.set_ecdh_curve(ssl_ecdh_curve) - verbose_logger.debug(f"SSL ECDH curve set to: {ssl_ecdh_curve}") + verbose_logger.debug("SSL ECDH curve set to: %s", ssl_ecdh_curve) except AttributeError: verbose_logger.warning( - f"SSL ECDH curve configuration not supported. " - f"Python version: {sys.version.split()[0]}, OpenSSL version: {ssl.OPENSSL_VERSION}. " - f"Requested curve: {ssl_ecdh_curve}. Continuing with default curves." + "SSL ECDH curve configuration not supported. Python version: %s, OpenSSL version: %s. Requested curve: %s. Continuing with default curves.", + sys.version.split()[0], + ssl.OPENSSL_VERSION, + ssl_ecdh_curve, ) except ValueError as e: # Invalid curve name verbose_logger.warning( - f"Invalid SSL ECDH curve name: '{ssl_ecdh_curve}'. {e}. " - f"Common valid curves: X25519, prime256v1, secp384r1, secp521r1. " - f"Continuing with default curves (including PQC)." + "Invalid SSL ECDH curve name: '%s'. %s. Common valid curves: X25519, prime256v1, secp384r1, secp521r1. Continuing with default curves (including PQC).", + ssl_ecdh_curve, + e, ) return custom_ssl_context @@ -1033,7 +1034,7 @@ class AsyncHTTPHandler: # Use shared session if provided and valid if shared_session is not None and not shared_session.closed: - verbose_logger.debug(f"SHARED SESSION: Reusing existing ClientSession (ID: {id(shared_session)})") + verbose_logger.debug("SHARED SESSION: Reusing existing ClientSession (ID: %s)", id(shared_session)) return LiteLLMAiohttpTransport( client=shared_session, ssl_verify=ssl_for_transport, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index f7bf174f9ac..fcb039c9b4f 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -370,7 +370,7 @@ class BaseLLMHTTPHandler: ): if client is None: verbose_logger.debug( - f"Creating HTTP client with shared_session: {id(shared_session) if shared_session else None}" + "Creating HTTP client with shared_session: %s", id(shared_session) if shared_session else None ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), @@ -2676,7 +2676,8 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): verbose_logger.debug( - f"Creating HTTP client for responses API with shared_session: {id(shared_session) if shared_session else None}" + "Creating HTTP client for responses API with shared_session: %s", + id(shared_session) if shared_session else None, ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), @@ -2859,7 +2860,8 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): verbose_logger.debug( - f"Creating HTTP client for delete_response with shared_session: {id(shared_session) if shared_session else None}" + "Creating HTTP client for delete_response with shared_session: %s", + id(shared_session) if shared_session else None, ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), @@ -3112,7 +3114,8 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): verbose_logger.debug( - f"Creating HTTP client for get_responses with shared_session: {id(shared_session) if shared_session else None}" + "Creating HTTP client for get_responses with shared_session: %s", + id(shared_session) if shared_session else None, ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), @@ -3156,7 +3159,7 @@ class BaseLLMHTTPHandler: response = await async_httpx_client.get(url=url, headers=headers, params=data) response.raise_for_status() except Exception as e: - verbose_logger.debug(f"Error retrieving response: {e}") + verbose_logger.debug("Error retrieving response: %s", e) raise self._handle_error( e=e, provider_config=responses_api_provider_config, @@ -3275,7 +3278,8 @@ class BaseLLMHTTPHandler: ) -> dict: if client is None or not isinstance(client, AsyncHTTPHandler): verbose_logger.debug( - f"Creating HTTP client for list_input_items with shared_session: {id(shared_session) if shared_session else None}" + "Creating HTTP client for list_input_items with shared_session: %s", + id(shared_session) if shared_session else None, ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), @@ -3495,7 +3499,7 @@ class BaseLLMHTTPHandler: timeout=timeout, ) except Exception as e: - verbose_logger.exception(f"Error creating file: {e}") + verbose_logger.exception("Error creating file: %s", e) raise self._handle_error(e=e, provider_config=provider_config) elif isinstance(transformed_request, str) or isinstance(transformed_request, bytes): # Handle traditional file uploads @@ -3626,7 +3630,7 @@ class BaseLLMHTTPHandler: if initial_response_data: litellm_params["initial_file_response"] = initial_response_data except Exception as e: - verbose_logger.exception(f"Error creating file: {e}") + verbose_logger.exception("Error creating file: %s", e) raise self._handle_error( e=e, provider_config=provider_config, @@ -3657,7 +3661,7 @@ class BaseLLMHTTPHandler: timeout=timeout, ) except Exception as e: - verbose_logger.exception(f"Error creating file: {e}") + verbose_logger.exception("Error creating file: %s", e) raise self._handle_error(e=e, provider_config=provider_config) elif isinstance(transformed_request, str) or isinstance(transformed_request, bytes): # Handle traditional file uploads @@ -3868,7 +3872,7 @@ class BaseLLMHTTPHandler: timeout=timeout, ) except Exception as e: - verbose_logger.exception(f"Error creating batch: {e}") + verbose_logger.exception("Error creating batch: %s", e) raise self._handle_error( e=e, provider_config=provider_config, @@ -3960,7 +3964,7 @@ class BaseLLMHTTPHandler: headers=headers, ) except Exception as e: - verbose_logger.exception(f"Error retrieving batch: {e}") + verbose_logger.exception("Error retrieving batch: %s", e) raise self._handle_error( e=e, provider_config=provider_config, @@ -4033,7 +4037,7 @@ class BaseLLMHTTPHandler: timeout=timeout, ) except Exception as e: - verbose_logger.exception(f"Error creating batch: {e}") + verbose_logger.exception("Error creating batch: %s", e) raise self._handle_error( e=e, provider_config=provider_config, @@ -4117,7 +4121,7 @@ class BaseLLMHTTPHandler: headers=headers, ) except Exception as e: - verbose_logger.exception(f"Error retrieving batch: {e}") + verbose_logger.exception("Error retrieving batch: %s", e) raise self._handle_error( e=e, provider_config=provider_config, @@ -4230,7 +4234,8 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): verbose_logger.debug( - f"Creating HTTP client for cancel_response with shared_session: {id(shared_session) if shared_session else None}" + "Creating HTTP client for cancel_response with shared_session: %s", + id(shared_session) if shared_session else None, ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), @@ -4404,7 +4409,8 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): verbose_logger.debug( - f"Creating HTTP client for compact_response with shared_session: {id(shared_session) if shared_session else None}" + "Creating HTTP client for compact_response with shared_session: %s", + id(shared_session) if shared_session else None, ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), @@ -5659,7 +5665,7 @@ class BaseLLMHTTPHandler: fingerprint=fingerprint, ) except Exception as e: - verbose_logger.exception(f"LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: {e}") + verbose_logger.exception("LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: %s", e) # Check if we need to convert response to fake stream for chat completions # This happens when: @@ -5901,10 +5907,10 @@ class BaseLLMHTTPHandler: await realtime_streaming.bidirectional_forward() except websockets.exceptions.InvalidStatusCode as e: # type: ignore - verbose_logger.exception(f"Error connecting to backend: {e}") + verbose_logger.exception("Error connecting to backend: %s", e) await websocket.close(code=e.status_code, reason=_redact_string(str(e))) except Exception as e: - verbose_logger.exception(f"Error connecting to backend: {e}") + verbose_logger.exception("Error connecting to backend: %s", e) try: await websocket.close(code=1011, reason=_redact_string(f"Internal server error: {e}")) except RuntimeError as close_error: @@ -6298,10 +6304,10 @@ class BaseLLMHTTPHandler: await streaming.bidirectional_forward() except websockets.exceptions.InvalidStatusCode as e: # type: ignore - verbose_logger.exception(f"Error connecting to responses WS backend: {e}") + verbose_logger.exception("Error connecting to responses WS backend: %s", e) await websocket.close(code=e.status_code, reason=_redact_string(str(e))) except Exception as e: - verbose_logger.exception(f"Error in responses WS: {e}") + verbose_logger.exception("Error in responses WS: %s", e) try: await websocket.close(code=1011, reason=_redact_string(f"Internal server error: {e}")) except RuntimeError as close_error: diff --git a/litellm/llms/databricks/common_utils.py b/litellm/llms/databricks/common_utils.py index 62e2245db99..d69b97b32a1 100644 --- a/litellm/llms/databricks/common_utils.py +++ b/litellm/llms/databricks/common_utils.py @@ -386,7 +386,7 @@ class DatabricksBase: headers["User-Agent"] = self._build_user_agent(custom_user_agent) # Debug logging with redaction (never log actual tokens) - verbose_logger.debug(f"Databricks request headers: {self.redact_headers_for_logging(headers)}") + verbose_logger.debug("Databricks request headers: %s", self.redact_headers_for_logging(headers)) if endpoint_type == "chat_completions" and custom_endpoint is not True: api_base = f"{api_base}/chat/completions" diff --git a/litellm/llms/databricks/streaming_utils.py b/litellm/llms/databricks/streaming_utils.py index 74216888111..635aa04b0be 100644 --- a/litellm/llms/databricks/streaming_utils.py +++ b/litellm/llms/databricks/streaming_utils.py @@ -126,7 +126,9 @@ class ModelResponseIterator: except StopIteration: raise StopIteration except ValueError as e: - verbose_logger.debug(f"Error parsing chunk: {e},\nReceived chunk: {chunk}. Defaulting to empty chunk here.") + verbose_logger.debug( + "Error parsing chunk: %s,\nReceived chunk: %s. Defaulting to empty chunk here.", e, chunk + ) return GenericStreamingChunk( text="", is_finished=False, @@ -171,7 +173,9 @@ class ModelResponseIterator: except StopAsyncIteration: raise StopAsyncIteration except ValueError as e: - verbose_logger.debug(f"Error parsing chunk: {e},\nReceived chunk: {chunk}. Defaulting to empty chunk here.") + verbose_logger.debug( + "Error parsing chunk: %s,\nReceived chunk: %s. Defaulting to empty chunk here.", e, chunk + ) return GenericStreamingChunk( text="", is_finished=False, diff --git a/litellm/llms/gemini/files/transformation.py b/litellm/llms/gemini/files/transformation.py index 89ac56979bb..274e5474101 100644 --- a/litellm/llms/gemini/files/transformation.py +++ b/litellm/llms/gemini/files/transformation.py @@ -190,7 +190,7 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): status_details=None, ) except Exception as e: - verbose_logger.exception(f"Error parsing file upload response: {e}") + verbose_logger.exception("Error parsing file upload response: %s", e) raise ValueError(f"Error parsing file upload response: {e}") def transform_retrieve_file_request( @@ -263,9 +263,9 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): Transform Gemini's file retrieval response into OpenAI-style FileObject """ try: - verbose_logger.debug(f"Retrieve file response: {raw_response.text}") + verbose_logger.debug("Retrieve file response: %s", raw_response.text) response_json = raw_response.json() - verbose_logger.debug(f"Response JSON: {response_json}") + verbose_logger.debug("Response JSON: %s", response_json) # Map Gemini state to OpenAI status gemini_state = response_json.get("state", "STATE_UNSPECIFIED") # Explicitly type status as the Literal union @@ -294,7 +294,7 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): status_details=(str(response_json.get("error", "")) if gemini_state == "FAILED" else None), ) except Exception as e: - verbose_logger.exception(f"Error parsing file retrieve response: {e}") + verbose_logger.exception("Error parsing file retrieve response: %s", e) raise ValueError(f"Error parsing file retrieve response: {e}") def transform_delete_file_request( @@ -362,7 +362,7 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): else: raise ValueError(f"Failed to delete file: {raw_response.text}") except Exception as e: - verbose_logger.exception(f"Error parsing file delete response: {e}") + verbose_logger.exception("Error parsing file delete response: %s", e) raise ValueError(f"Error parsing file delete response: {e}") def transform_list_files_request( diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 6631c9d9ec7..0d56e3f92bc 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -154,7 +154,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): if "parts" in model_turn: parts = model_turn["parts"] if len(parts) != 1: - verbose_logger.warning(f"Realtime: Expected 1 part, got {len(parts)} for Gemini model turn event.") + verbose_logger.warning("Realtime: Expected 1 part, got %s for Gemini model turn event.", len(parts)) part = parts[0] if "text" in part: return OpenAIRealtimeEventTypes.RESPONSE_TEXT_DELTA @@ -472,7 +472,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): call_id = item.get("call_id", "") output = item.get("output", "{}") - verbose_logger.debug(f"Gemini Realtime: Transforming function_call_output for call_id={call_id}") + verbose_logger.debug("Gemini Realtime: Transforming function_call_output for call_id=%s", call_id) # Gemini functionResponses[].response must be a dict; wrap non-dicts. try: @@ -487,8 +487,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): self._tool_call_id_to_name.move_to_end(call_id) else: verbose_logger.warning( - f"Gemini Realtime: Function name not found for call_id={call_id}. " - "This may cause Gemini to reject the response." + "Gemini Realtime: Function name not found for call_id=%s. This may cause Gemini to reject the response.", + call_id, ) function_response: dict[str, Any] = {"response": output_dict} @@ -868,7 +868,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): resolved_response_id = response_id or f"resp_{uuid.uuid4()}" resolved_output_item_id = output_item_id or f"item_{uuid.uuid4()}" - verbose_logger.debug(f"Gemini Realtime: Transforming {len(function_calls)} tool call(s) to OpenAI format") + verbose_logger.debug("Gemini Realtime: Transforming %s tool call(s) to OpenAI format", len(function_calls)) events: list[OpenAIRealtimeFunctionCallArgumentsDone] = [] for idx, fc in enumerate(function_calls): diff --git a/litellm/llms/gigachat/authenticator.py b/litellm/llms/gigachat/authenticator.py index 356d438c6b2..9fc60ee058d 100644 --- a/litellm/llms/gigachat/authenticator.py +++ b/litellm/llms/gigachat/authenticator.py @@ -162,7 +162,7 @@ def _request_token_sync( } data = {"scope": scope} - verbose_logger.debug(f"Requesting GigaChat access token from {auth_url}") + verbose_logger.debug("Requesting GigaChat access token from %s", auth_url) try: client = _get_http_client() @@ -194,7 +194,7 @@ async def _request_token_async( } data = {"scope": scope} - verbose_logger.debug(f"Requesting GigaChat access token from {auth_url}") + verbose_logger.debug("Requesting GigaChat access token from %s", auth_url) try: client = get_async_httpx_client( diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index 4007588cfc5..d907ae2dcdf 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -268,7 +268,7 @@ class GigaChatConfig(BaseConfig): api_base=self._current_api_base, ) except Exception as e: - verbose_logger.error(f"Failed to upload image: {e}") + verbose_logger.error("Failed to upload image: %s", e) return None def transform_request( diff --git a/litellm/llms/gigachat/file_handler.py b/litellm/llms/gigachat/file_handler.py index ee16a6c5870..6c2e6d17915 100644 --- a/litellm/llms/gigachat/file_handler.py +++ b/litellm/llms/gigachat/file_handler.py @@ -97,7 +97,7 @@ def upload_file_sync( # Check cache if url_hash in _file_cache: - verbose_logger.debug(f"Image found in cache: {url_hash[:16]}...") + verbose_logger.debug("Image found in cache: %s...", url_hash[:16]) return _file_cache[url_hash] try: @@ -107,7 +107,7 @@ def upload_file_sync( content_bytes, content_type, ext = parsed verbose_logger.debug("Decoded base64 image") else: - verbose_logger.debug(f"Downloading image from URL: {image_url[:80]}...") + verbose_logger.debug("Downloading image from URL: %s...", image_url[:80]) content_bytes, content_type, ext = _download_image_sync(image_url) filename = f"{uuid.uuid4()}.{ext}" @@ -133,12 +133,12 @@ def upload_file_sync( file_id = result.get("id") if file_id: _file_cache[url_hash] = file_id - verbose_logger.debug(f"File uploaded successfully, file_id: {file_id}") + verbose_logger.debug("File uploaded successfully, file_id: %s", file_id) return file_id except Exception as e: - verbose_logger.error(f"Error uploading file to GigaChat: {e}") + verbose_logger.error("Error uploading file to GigaChat: %s", e) return None @@ -162,7 +162,7 @@ async def upload_file_async( # Check cache if url_hash in _file_cache: - verbose_logger.debug(f"Image found in cache: {url_hash[:16]}...") + verbose_logger.debug("Image found in cache: %s...", url_hash[:16]) return _file_cache[url_hash] try: @@ -172,7 +172,7 @@ async def upload_file_async( content_bytes, content_type, ext = parsed verbose_logger.debug("Decoded base64 image") else: - verbose_logger.debug(f"Downloading image from URL: {image_url[:80]}...") + verbose_logger.debug("Downloading image from URL: %s...", image_url[:80]) content_bytes, content_type, ext = await _download_image_async(image_url) filename = f"{uuid.uuid4()}.{ext}" @@ -201,10 +201,10 @@ async def upload_file_async( file_id = result.get("id") if file_id: _file_cache[url_hash] = file_id - verbose_logger.debug(f"File uploaded successfully, file_id: {file_id}") + verbose_logger.debug("File uploaded successfully, file_id: %s", file_id) return file_id except Exception as e: - verbose_logger.error(f"Error uploading file to GigaChat: {e}") + verbose_logger.error("Error uploading file to GigaChat: %s", e) return None diff --git a/litellm/llms/github_copilot/authenticator.py b/litellm/llms/github_copilot/authenticator.py index 180c2215212..b4bd9c4de3d 100644 --- a/litellm/llms/github_copilot/authenticator.py +++ b/litellm/llms/github_copilot/authenticator.py @@ -58,7 +58,7 @@ class Authenticator: verbose_logger.warning("No existing access token found or error reading file") for attempt in range(3): - verbose_logger.debug(f"Access token acquisition attempt {attempt + 1}/3") + verbose_logger.debug("Access token acquisition attempt %s/3", attempt + 1) try: access_token = self._login() try: @@ -68,7 +68,7 @@ class Authenticator: verbose_logger.error("Error saving access token to file") return access_token except (GetDeviceCodeError, GetAccessTokenError, RefreshAPIKeyError) as e: - verbose_logger.warning(f"Failed attempt {attempt + 1}: {e}") + verbose_logger.warning("Failed attempt %s: %s", attempt + 1, e) continue raise GetAccessTokenError( @@ -100,7 +100,7 @@ class Authenticator: except OSError: verbose_logger.warning("No API key file found or error opening file") except (json.JSONDecodeError, KeyError) as e: - verbose_logger.warning(f"Error reading API key from file: {e}") + verbose_logger.warning("Error reading API key from file: %s", e) except APIKeyExpiredError: pass # Already logged in the try block @@ -117,7 +117,7 @@ class Authenticator: status_code=401, ) except OSError as e: - verbose_logger.error(f"Error saving API key to file: {e}") + verbose_logger.error("Error saving API key to file: %s", e) raise GetAPIKeyError( message=f"Failed to save API key: {e}", status_code=500, @@ -142,7 +142,7 @@ class Authenticator: api_endpoint = endpoints.get("api") return api_endpoint except (OSError, json.JSONDecodeError, KeyError) as e: - verbose_logger.warning(f"Error reading API endpoint from file: {e}") + verbose_logger.warning("Error reading API endpoint from file: %s", e) return None def _refresh_api_key(self) -> dict[str, Any]: @@ -171,11 +171,11 @@ class Authenticator: if "token" in response_json: return response_json else: - verbose_logger.warning(f"API key response missing token: {response_json}") + verbose_logger.warning("API key response missing token: %s", response_json) except httpx.HTTPStatusError as e: - verbose_logger.error(f"HTTP error refreshing API key (attempt {attempt + 1}/{max_retries}): {e}") + verbose_logger.error("HTTP error refreshing API key (attempt %s/%s): %s", attempt + 1, max_retries, e) except Exception as e: - verbose_logger.error(f"Unexpected error refreshing API key: {e}") + verbose_logger.error("Unexpected error refreshing API key: %s", e) raise RefreshAPIKeyError( message="Failed to refresh API key after maximum retries", @@ -237,7 +237,7 @@ class Authenticator: required_fields = ["device_code", "user_code", "verification_uri"] if not all(field in resp_json for field in required_fields): - verbose_logger.error(f"Response missing required fields: {resp_json}") + verbose_logger.error("Response missing required fields: %s", resp_json) raise GetDeviceCodeError( message="Response missing required fields", status_code=400, @@ -245,19 +245,19 @@ class Authenticator: return resp_json except httpx.HTTPStatusError as e: - verbose_logger.error(f"HTTP error getting device code: {e}") + verbose_logger.error("HTTP error getting device code: %s", e) raise GetDeviceCodeError( message=f"Failed to get device code: {e}", status_code=400, ) except json.JSONDecodeError as e: - verbose_logger.error(f"Error decoding JSON response: {e}") + verbose_logger.error("Error decoding JSON response: %s", e) raise GetDeviceCodeError( message=f"Failed to decode device code response: {e}", status_code=400, ) except Exception as e: - verbose_logger.error(f"Unexpected error getting device code: {e}") + verbose_logger.error("Unexpected error getting device code: %s", e) raise GetDeviceCodeError( message=f"Failed to get device code: {e}", status_code=400, @@ -300,23 +300,23 @@ class Authenticator: verbose_logger.info("Authentication successful!") return resp_json["access_token"] elif "error" in resp_json and resp_json.get("error") == "authorization_pending": - verbose_logger.debug(f"Authorization pending (attempt {attempt + 1}/{max_attempts})") + verbose_logger.debug("Authorization pending (attempt %s/%s)", attempt + 1, max_attempts) else: - verbose_logger.warning(f"Unexpected response: {resp_json}") + verbose_logger.warning("Unexpected response: %s", resp_json) except httpx.HTTPStatusError as e: - verbose_logger.error(f"HTTP error polling for access token: {e}") + verbose_logger.error("HTTP error polling for access token: %s", e) raise GetAccessTokenError( message=f"Failed to get access token: {e}", status_code=400, ) except json.JSONDecodeError as e: - verbose_logger.error(f"Error decoding JSON response: {e}") + verbose_logger.error("Error decoding JSON response: %s", e) raise GetAccessTokenError( message=f"Failed to decode access token response: {e}", status_code=400, ) except Exception as e: - verbose_logger.error(f"Unexpected error polling for access token: {e}") + verbose_logger.error("Unexpected error polling for access token: %s", e) raise GetAccessTokenError( message=f"Failed to get access token: {e}", status_code=400, diff --git a/litellm/llms/github_copilot/embedding/transformation.py b/litellm/llms/github_copilot/embedding/transformation.py index 75c2d0e8c40..89e195e2d76 100644 --- a/litellm/llms/github_copilot/embedding/transformation.py +++ b/litellm/llms/github_copilot/embedding/transformation.py @@ -75,7 +75,7 @@ class GithubCopilotEmbeddingConfig(BaseEmbeddingConfig): # Merge with existing headers (user's extra_headers take priority) merged_headers = {**default_headers, **headers} - verbose_logger.debug(f"GitHub Copilot Embedding API: Successfully configured headers for model {model}") + verbose_logger.debug("GitHub Copilot Embedding API: Successfully configured headers for model %s", model) return merged_headers diff --git a/litellm/llms/github_copilot/responses/transformation.py b/litellm/llms/github_copilot/responses/transformation.py index 170ad938efb..079ad760aea 100644 --- a/litellm/llms/github_copilot/responses/transformation.py +++ b/litellm/llms/github_copilot/responses/transformation.py @@ -222,14 +222,14 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): if input_param is not None: initiator = self._get_initiator(input_param) merged_headers["X-Initiator"] = initiator - verbose_logger.debug(f"GitHub Copilot Responses API: Set X-Initiator={initiator}") + verbose_logger.debug("GitHub Copilot Responses API: Set X-Initiator=%s", initiator) # Add vision header if input contains images if self._has_vision_input(input_param): merged_headers["copilot-vision-request"] = "true" verbose_logger.debug("GitHub Copilot Responses API: Enabled vision request") - verbose_logger.debug(f"GitHub Copilot Responses API: Successfully configured headers for model {model}") + verbose_logger.debug("GitHub Copilot Responses API: Successfully configured headers for model %s", model) return merged_headers @@ -295,7 +295,8 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): filtered_item[k] = v verbose_logger.debug( - f"GitHub Copilot reasoning item processed, encrypted_content preserved: {encrypted_content is not None}" + "GitHub Copilot reasoning item processed, encrypted_content preserved: %s", + encrypted_content is not None, ) return filtered_item return item @@ -379,7 +380,7 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): """ if depth > max_depth: verbose_logger.warning( - f"[GitHub Copilot] Max recursion depth {max_depth} reached while checking for vision content" + "[GitHub Copilot] Max recursion depth %s reached while checking for vision content", max_depth ) return False diff --git a/litellm/llms/groq/chat/transformation.py b/litellm/llms/groq/chat/transformation.py index 64537e33d0e..35a4a14057f 100644 --- a/litellm/llms/groq/chat/transformation.py +++ b/litellm/llms/groq/chat/transformation.py @@ -99,7 +99,7 @@ class GroqChatConfig(OpenAILikeChatConfig): if litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider): base_params.append("reasoning_effort") except Exception as e: - verbose_logger.debug(f"Error checking if model supports reasoning: {e}") + verbose_logger.debug("Error checking if model supports reasoning: %s", e) return base_params diff --git a/litellm/llms/huggingface/chat/transformation.py b/litellm/llms/huggingface/chat/transformation.py index da1ebd7c23a..b02e5c174a8 100644 --- a/litellm/llms/huggingface/chat/transformation.py +++ b/litellm/llms/huggingface/chat/transformation.py @@ -148,7 +148,7 @@ class HuggingFaceChatConfig(OpenAIGPTConfig): provider_mapping = provider_mapping[provider] if provider_mapping["status"] == "staging": logger.warning( - f"Model {model_id} is in staging mode for provider {provider}. Meant for test purposes only." + "Model %s is in staging mode for provider %s. Meant for test purposes only.", model_id, provider ) mapped_model = provider_mapping["providerId"] diff --git a/litellm/llms/langflow/chat/transformation.py b/litellm/llms/langflow/chat/transformation.py index 69af32fc840..e84b7677ad2 100644 --- a/litellm/llms/langflow/chat/transformation.py +++ b/litellm/llms/langflow/chat/transformation.py @@ -168,7 +168,7 @@ class LangFlowConfig(BaseConfig): if session_id: payload["session_id"] = session_id - verbose_logger.debug(f"LangFlow request payload: {payload}") + verbose_logger.debug("LangFlow request payload: %s", payload) return payload def _extract_content_from_response(self, response_json: dict) -> str | None: @@ -235,7 +235,7 @@ class LangFlowConfig(BaseConfig): status_code=raw_response.status_code, ) - verbose_logger.debug(f"LangFlow response: {response_json}") + verbose_logger.debug("LangFlow response: %s", response_json) content = self._extract_content_from_response(response_json) if content is None: @@ -265,7 +265,7 @@ class LangFlowConfig(BaseConfig): ) setattr(model_response, "usage", usage) except Exception as e: - verbose_logger.warning(f"Failed to calculate token usage: {e}") + verbose_logger.warning("Failed to calculate token usage: %s", e) return model_response diff --git a/litellm/llms/langgraph/chat/sse_iterator.py b/litellm/llms/langgraph/chat/sse_iterator.py index bdaa34871cf..8815d5c93da 100644 --- a/litellm/llms/langgraph/chat/sse_iterator.py +++ b/litellm/llms/langgraph/chat/sse_iterator.py @@ -62,7 +62,7 @@ class LangGraphSSEStreamIterator: data = json.loads(json_str) return self._process_data(data) except json.JSONDecodeError: - verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}") + verbose_logger.debug("Skipping non-JSON SSE line: %s", line[:100]) return None return None @@ -196,7 +196,7 @@ class LangGraphSSEStreamIterator: except httpx.StreamClosed: raise StopIteration except Exception as e: - verbose_logger.error(f"Error in LangGraph SSE stream: {e}") + verbose_logger.error("Error in LangGraph SSE stream: %s", e) raise StopIteration async def __anext__(self) -> ModelResponseStream: @@ -224,5 +224,5 @@ class LangGraphSSEStreamIterator: except httpx.StreamClosed: raise StopAsyncIteration except Exception as e: - verbose_logger.error(f"Error in LangGraph SSE stream: {e}") + verbose_logger.error("Error in LangGraph SSE stream: %s", e) raise StopAsyncIteration diff --git a/litellm/llms/langgraph/chat/transformation.py b/litellm/llms/langgraph/chat/transformation.py index 2aa96ddb978..3f6b0c327d2 100644 --- a/litellm/llms/langgraph/chat/transformation.py +++ b/litellm/llms/langgraph/chat/transformation.py @@ -223,7 +223,7 @@ class LangGraphConfig(BaseConfig): if "thread_id" in optional_params: payload["thread_id"] = optional_params["thread_id"] - verbose_logger.debug(f"LangGraph request payload: {payload}") + verbose_logger.debug("LangGraph request payload: %s", payload) return payload def _extract_content_from_response(self, response_json: dict) -> str: @@ -297,7 +297,7 @@ class LangGraphConfig(BaseConfig): if client is None or not isinstance(client, HTTPHandler): client = _get_httpx_client(params={}) - verbose_logger.debug(f"Making sync streaming request to: {api_base}") + verbose_logger.debug("Making sync streaming request to: %s", api_base) # Make streaming request response = client.post( @@ -356,7 +356,7 @@ class LangGraphConfig(BaseConfig): if client is None or not isinstance(client, AsyncHTTPHandler): client = get_async_httpx_client(llm_provider=cast(Any, "langgraph"), params={}) - verbose_logger.debug(f"Making async streaming request to: {api_base}") + verbose_logger.debug("Making async streaming request to: %s", api_base) # Make async streaming request response = await client.post( @@ -422,7 +422,7 @@ class LangGraphConfig(BaseConfig): """ try: response_json = raw_response.json() - verbose_logger.debug(f"LangGraph response: {response_json}") + verbose_logger.debug("LangGraph response: %s", response_json) content = self._extract_content_from_response(response_json) @@ -451,12 +451,12 @@ class LangGraphConfig(BaseConfig): ) setattr(model_response, "usage", usage) except Exception as e: - verbose_logger.warning(f"Failed to calculate token usage: {e}") + verbose_logger.warning("Failed to calculate token usage: %s", e) return model_response except Exception as e: - verbose_logger.error(f"Error processing LangGraph response: {e}") + verbose_logger.error("Error processing LangGraph response: %s", e) raise LangGraphError( message=f"Error processing response: {e}", status_code=raw_response.status_code, diff --git a/litellm/llms/litellm_proxy/skills/code_execution.py b/litellm/llms/litellm_proxy/skills/code_execution.py index f1142a8e355..11fd80db377 100644 --- a/litellm/llms/litellm_proxy/skills/code_execution.py +++ b/litellm/llms/litellm_proxy/skills/code_execution.py @@ -141,7 +141,7 @@ class CodeExecutionHandler: response: Any = None # Initialize to avoid possibly unbound error for iteration in range(self.max_iterations): - verbose_logger.debug(f"CodeExecutionHandler: Iteration {iteration + 1}/{self.max_iterations}") + verbose_logger.debug("CodeExecutionHandler: Iteration %s/%s", iteration + 1, self.max_iterations) # Make LLM call response = await litellm.acompletion( @@ -175,7 +175,7 @@ class CodeExecutionHandler: # Check if we're done (no tool calls or not tool_calls finish reason) if stop_reason != "tool_calls" or not assistant_message.tool_calls: - verbose_logger.debug(f"CodeExecutionHandler: Completed after {iteration + 1} iterations") + verbose_logger.debug("CodeExecutionHandler: Completed after %s iterations", iteration + 1) return { "response": response, "files": generated_files, # Files returned directly with base64 content @@ -193,14 +193,14 @@ class CodeExecutionHandler: args = json.loads(tool_call.function.arguments) code = args.get("code", "") - verbose_logger.debug(f"CodeExecutionHandler: Executing code ({len(code)} chars)") + verbose_logger.debug("CodeExecutionHandler: Executing code (%s chars)", len(code)) exec_result = executor.execute( code=code, skill_files=skill_files, ) - verbose_logger.debug(f"CodeExecutionHandler: Execution result: {exec_result}") + verbose_logger.debug("CodeExecutionHandler: Execution result: %s", exec_result) execution_results.append( { @@ -232,7 +232,7 @@ class CodeExecutionHandler: tool_result += f"\n- {f['name']} ({len(file_content)} bytes)" verbose_logger.debug( - f"CodeExecutionHandler: Generated file {f['name']} ({len(file_content)} bytes)" + "CodeExecutionHandler: Generated file %s (%s bytes)", f["name"], len(file_content) ) if exec_result["error"]: @@ -268,7 +268,7 @@ class CodeExecutionHandler: ) # Max iterations reached - verbose_logger.warning(f"CodeExecutionHandler: Max iterations ({self.max_iterations}) reached") + verbose_logger.warning("CodeExecutionHandler: Max iterations (%s) reached", self.max_iterations) return { "response": response, "files": generated_files, diff --git a/litellm/llms/litellm_proxy/skills/handler.py b/litellm/llms/litellm_proxy/skills/handler.py index cc307917af4..efe5c27b7ba 100644 --- a/litellm/llms/litellm_proxy/skills/handler.py +++ b/litellm/llms/litellm_proxy/skills/handler.py @@ -100,7 +100,7 @@ class LiteLLMSkillsHandler: if data.file_type is not None: skill_data["file_type"] = data.file_type - verbose_logger.debug(f"LiteLLMSkillsHandler: Creating skill {skill_id} with title={data.display_title}") + verbose_logger.debug("LiteLLMSkillsHandler: Creating skill %s with title=%s", skill_id, data.display_title) new_skill = await SkillsRepository(prisma_client).table.create(data=skill_data) return _prisma_skill_to_litellm(new_skill) @@ -113,7 +113,7 @@ class LiteLLMSkillsHandler: ) -> list[LiteLLM_SkillsTable]: prisma_client = await LiteLLMSkillsHandler._get_prisma_client() - verbose_logger.debug(f"LiteLLMSkillsHandler: Listing skills with limit={limit}, offset={offset}") + verbose_logger.debug("LiteLLMSkillsHandler: Listing skills with limit=%s, offset=%s", limit, offset) find_many_kwargs: dict[str, Any] = { "take": limit, @@ -150,7 +150,7 @@ class LiteLLMSkillsHandler: skill_id: str, user_api_key_dict: UserAPIKeyAuth | None = None, ) -> LiteLLM_SkillsTable: - verbose_logger.debug(f"LiteLLMSkillsHandler: Getting skill {skill_id}") + verbose_logger.debug("LiteLLMSkillsHandler: Getting skill %s", skill_id) skill = await LiteLLMSkillsHandler._load_skill(skill_id) # Same "not found" message for both "missing" and "cross-tenant" @@ -166,7 +166,7 @@ class LiteLLMSkillsHandler: user_api_key_dict: UserAPIKeyAuth | None = None, ) -> dict[str, str]: prisma_client = await LiteLLMSkillsHandler._get_prisma_client() - verbose_logger.debug(f"LiteLLMSkillsHandler: Deleting skill {skill_id}") + verbose_logger.debug("LiteLLMSkillsHandler: Deleting skill %s", skill_id) skill = await LiteLLMSkillsHandler._load_skill(skill_id) if skill is None or not user_can_access_resource_owner(getattr(skill, "created_by", None), user_api_key_dict): @@ -189,5 +189,5 @@ class LiteLLMSkillsHandler: except ValueError: return None except Exception as e: - verbose_logger.warning(f"LiteLLMSkillsHandler: Error fetching skill {skill_id}: {e}") + verbose_logger.warning("LiteLLMSkillsHandler: Error fetching skill %s: %s", skill_id, e) return None diff --git a/litellm/llms/litellm_proxy/skills/prompt_injection.py b/litellm/llms/litellm_proxy/skills/prompt_injection.py index 244d4196404..f411da8f04c 100644 --- a/litellm/llms/litellm_proxy/skills/prompt_injection.py +++ b/litellm/llms/litellm_proxy/skills/prompt_injection.py @@ -66,7 +66,7 @@ class SkillPromptInjectionHandler: return f"## Skill: {skill.display_title or skill.skill_id}\n\n{content}" except Exception as e: verbose_logger.warning( - f"SkillPromptInjectionHandler: Error extracting content from skill {skill.skill_id}: {e}" + "SkillPromptInjectionHandler: Error extracting content from skill %s: %s", skill.skill_id, e ) return skill.instructions @@ -111,14 +111,16 @@ class SkillPromptInjectionHandler: normalized = posixpath.normpath(clean_path) if normalized.startswith("..") or posixpath.isabs(normalized): verbose_logger.warning( - f"SkillPromptInjectionHandler: Skipping entry with invalid path in skill {skill.skill_id}: {name}" + "SkillPromptInjectionHandler: Skipping entry with invalid path in skill %s: %s", + skill.skill_id, + name, ) continue files[normalized] = zf.read(name) except Exception as e: verbose_logger.warning( - f"SkillPromptInjectionHandler: Error extracting files from skill {skill.skill_id}: {e}" + "SkillPromptInjectionHandler: Error extracting files from skill %s: %s", skill.skill_id, e ) return files diff --git a/litellm/llms/litellm_proxy/skills/sandbox_executor.py b/litellm/llms/litellm_proxy/skills/sandbox_executor.py index e79b0c948c4..78939f3e03b 100644 --- a/litellm/llms/litellm_proxy/skills/sandbox_executor.py +++ b/litellm/llms/litellm_proxy/skills/sandbox_executor.py @@ -96,7 +96,7 @@ class SkillsSandboxExecutor: # Create the file in temp directory local_path = os.path.abspath(os.path.join(tmpdir, path)) if not local_path.startswith(tmpdir_abs + os.sep): - verbose_logger.warning(f"SkillsSandboxExecutor: Skipping file with invalid path: {path}") + verbose_logger.warning("SkillsSandboxExecutor: Skipping file with invalid path: %s", path) continue os.makedirs(os.path.dirname(local_path), exist_ok=True) with open(local_path, "wb") as f: @@ -106,7 +106,7 @@ class SkillsSandboxExecutor: sandbox_path = f"/sandbox/{path}" session.copy_to_runtime(local_path, sandbox_path) - verbose_logger.debug(f"SkillsSandboxExecutor: Copied {len(skill_files)} files to sandbox") + verbose_logger.debug("SkillsSandboxExecutor: Copied %s files to sandbox", len(skill_files)) # 2. Install requirements if present. Let pip parse the # requirements file inside the sandbox so standard syntax like @@ -171,10 +171,10 @@ sys.path.insert(0, '/sandbox') verbose_logger.debug("SkillsSandboxExecutor: Code execution succeeded") else: verbose_logger.debug( - f"SkillsSandboxExecutor: Code execution failed with exit code {result.exit_code}" + "SkillsSandboxExecutor: Code execution failed with exit code %s", result.exit_code ) - verbose_logger.debug(f"SkillsSandboxExecutor: stderr: {error[:500] if error else 'No stderr'}") - verbose_logger.debug(f"SkillsSandboxExecutor: stdout: {output[:500] if output else 'No stdout'}") + verbose_logger.debug("SkillsSandboxExecutor: stderr: %s", error[:500] if error else "No stderr") + verbose_logger.debug("SkillsSandboxExecutor: stdout: %s", output[:500] if output else "No stdout") # 4. Collect generated files generated_files = self._collect_generated_files(session, skill_files) @@ -187,7 +187,7 @@ sys.path.insert(0, '/sandbox') } except Exception as e: - verbose_logger.error(f"SkillsSandboxExecutor: Execution failed: {e}") + verbose_logger.error("SkillsSandboxExecutor: Execution failed: %s", e) return { "success": False, "output": "", @@ -270,15 +270,15 @@ print(json.dumps(files)) } ) - verbose_logger.debug(f"SkillsSandboxExecutor: Collected generated file: {rel_path}") + verbose_logger.debug("SkillsSandboxExecutor: Collected generated file: %s", rel_path) except Exception as e: - verbose_logger.warning(f"SkillsSandboxExecutor: Error copying file {filepath}: {e}") + verbose_logger.warning("SkillsSandboxExecutor: Error copying file %s: %s", filepath, e) finally: if os.path.exists(tmp_path): os.unlink(tmp_path) except Exception as e: - verbose_logger.warning(f"SkillsSandboxExecutor: Error collecting generated files: {e}") + verbose_logger.warning("SkillsSandboxExecutor: Error collecting generated files: %s", e) return generated_files diff --git a/litellm/llms/manus/files/transformation.py b/litellm/llms/manus/files/transformation.py index 325f6f36814..667910fab28 100644 --- a/litellm/llms/manus/files/transformation.py +++ b/litellm/llms/manus/files/transformation.py @@ -245,10 +245,10 @@ class ManusFilesConfig(BaseFilesConfig): response_json = initial_response_data else: # Log raw response for debugging - verbose_logger.debug(f"Manus raw response text: {raw_response.text}") + verbose_logger.debug("Manus raw response text: %s", raw_response.text) response_json = raw_response.json() - verbose_logger.debug(f"Manus file response: {response_json}") + verbose_logger.debug("Manus file response: %s", response_json) # Parse created_at timestamp created_at_str = response_json.get("created_at", "") @@ -279,7 +279,7 @@ class ManusFilesConfig(BaseFilesConfig): status_details=response_json.get("status_details"), ) except Exception as e: - verbose_logger.exception(f"Error parsing Manus file response: {e}") + verbose_logger.exception("Error parsing Manus file response: %s", e) raise ValueError(f"Error parsing Manus file response: {e}") def transform_retrieve_file_request( diff --git a/litellm/llms/manus/responses/transformation.py b/litellm/llms/manus/responses/transformation.py index 25d4d0b8db6..80ffb87c76f 100644 --- a/litellm/llms/manus/responses/transformation.py +++ b/litellm/llms/manus/responses/transformation.py @@ -157,7 +157,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): if extra_body: base_request.update(extra_body) - verbose_logger.debug(f"Manus: Using agent_profile={agent_profile}, task_mode=agent") + verbose_logger.debug("Manus: Using agent_profile=%s, task_mode=agent", agent_profile) return base_request @@ -219,7 +219,9 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): try: response = ResponsesAPIResponse.model_validate(raw_response_json) except Exception: - verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct") + verbose_logger.debug( + "Error constructing ResponsesAPIResponse: %s, using model_construct", raw_response_json + ) response = ResponsesAPIResponse.model_construct(**raw_response_json) # Store processed headers in additional_headers so they get returned to the client @@ -307,7 +309,9 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): try: response = ResponsesAPIResponse.model_validate(raw_response_json) except Exception: - verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct") + verbose_logger.debug( + "Error constructing ResponsesAPIResponse: %s, using model_construct", raw_response_json + ) response = ResponsesAPIResponse.model_construct(**raw_response_json) # Store processed headers in additional_headers so they get returned to the client diff --git a/litellm/llms/mistral/ocr/transformation.py b/litellm/llms/mistral/ocr/transformation.py index e9d8280cc85..dbcb3307980 100644 --- a/litellm/llms/mistral/ocr/transformation.py +++ b/litellm/llms/mistral/ocr/transformation.py @@ -175,7 +175,7 @@ class MistralOCRConfig(BaseOCRConfig): Returns: OCRRequestData with JSON data """ - verbose_logger.debug(f"Mistral OCR transform_ocr_request - model: {model}") + verbose_logger.debug("Mistral OCR transform_ocr_request - model: %s", model) # Document parameter is the Mistral-format dict from the user # Just pass it through as-is to the Mistral API @@ -226,7 +226,7 @@ class MistralOCRConfig(BaseOCRConfig): try: response_json = raw_response.json() - verbose_logger.debug(f"Mistral OCR response keys: {response_json.keys()}") + verbose_logger.debug("Mistral OCR response keys: %s", response_json.keys()) # Return native Mistral format - no transformation return OCRResponse( @@ -237,5 +237,5 @@ class MistralOCRConfig(BaseOCRConfig): object="ocr", ) except Exception as e: - verbose_logger.error(f"Error parsing Mistral OCR response: {e}") + verbose_logger.error("Error parsing Mistral OCR response: %s", e) raise e diff --git a/litellm/llms/ollama/common_utils.py b/litellm/llms/ollama/common_utils.py index 83c697d7cb5..ecc56e6f110 100644 --- a/litellm/llms/ollama/common_utils.py +++ b/litellm/llms/ollama/common_utils.py @@ -119,7 +119,7 @@ class OllamaModelInfo(BaseLLMModelInfo): if isinstance(nm, str): names.add(nm if nm.startswith("ollama/") else f"ollama/{nm}") except Exception as e: - verbose_logger.warning(f"Error retrieving ollama tag endpoint: {e}") + verbose_logger.warning("Error retrieving ollama tag endpoint: %s", e) # If tags endpoint fails, fall back to static list try: from litellm import models_by_provider @@ -127,7 +127,7 @@ class OllamaModelInfo(BaseLLMModelInfo): static = models_by_provider.get("ollama", []) or [] return [f"ollama/{m}" for m in static] except Exception as e1: - verbose_logger.warning(f"Error retrieving static ollama models as fallback: {e1}") + verbose_logger.warning("Error retrieving static ollama models as fallback: %s", e1) return [] # assemble full model names result = sorted(names) diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 5823c2dad75..ccc9fe666f2 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -538,5 +538,5 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator): ) # raise Exception(f"Unable to parse ollama chunk - {chunk}") except Exception as e: - verbose_proxy_logger.error(f"Unable to parse ollama chunk - {chunk}") + verbose_proxy_logger.error("Unable to parse ollama chunk - %s", chunk) raise e diff --git a/litellm/llms/openai/chat/o_series_transformation.py b/litellm/llms/openai/chat/o_series_transformation.py index 0aaf0315f2a..e5c4de0b29f 100644 --- a/litellm/llms/openai/chat/o_series_transformation.py +++ b/litellm/llms/openai/chat/o_series_transformation.py @@ -66,7 +66,7 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig): model, custom_llm_provider, api_base, api_key = get_llm_provider(model=model) except Exception: verbose_logger.debug( - f"Unable to infer model provider for model={model}, defaulting to openai for o1 supported param check" + "Unable to infer model provider for model=%s, defaulting to openai for o1 supported param check", model ) custom_llm_provider = "openai" diff --git a/litellm/llms/openai/cost_calculation.py b/litellm/llms/openai/cost_calculation.py index 87568a4d399..40cfd1dd621 100644 --- a/litellm/llms/openai/cost_calculation.py +++ b/litellm/llms/openai/cost_calculation.py @@ -111,13 +111,19 @@ def cost_per_second(model: str, custom_llm_provider: str | None, duration: float ## Speech / Audio cost calculation if "output_cost_per_second" in model_info and model_info["output_cost_per_second"] is not None: verbose_logger.debug( - f"For model={model} - output_cost_per_second: {model_info.get('output_cost_per_second')}; duration: {duration}" + "For model=%s - output_cost_per_second: %s; duration: %s", + model, + model_info.get("output_cost_per_second"), + duration, ) ## COST PER SECOND ## completion_cost = model_info["output_cost_per_second"] * duration elif "input_cost_per_second" in model_info and model_info["input_cost_per_second"] is not None: verbose_logger.debug( - f"For model={model} - input_cost_per_second: {model_info.get('input_cost_per_second')}; duration: {duration}" + "For model=%s - input_cost_per_second: %s; duration: %s", + model, + model_info.get("input_cost_per_second"), + duration, ) ## COST PER SECOND ## prompt_cost = model_info["input_cost_per_second"] * duration @@ -199,19 +205,23 @@ def video_generation_cost( video_cost_per_second = model_info.get("output_cost_per_video_per_second") if video_cost_per_second is not None: verbose_logger.debug( - f"For model={model} - output_cost_per_video_per_second: {video_cost_per_second}; duration: {duration_seconds}" + "For model=%s - output_cost_per_video_per_second: %s; duration: %s", + model, + video_cost_per_second, + duration_seconds, ) return video_cost_per_second * duration_seconds output_cost_per_second = _video_output_cost_per_second(model_info, video_resolution) if output_cost_per_second is not None: verbose_logger.debug( - f"For model={model} - output_cost_per_second: {output_cost_per_second}; duration: {duration_seconds}" + "For model=%s - output_cost_per_second: %s; duration: %s", model, output_cost_per_second, duration_seconds ) return output_cost_per_second * duration_seconds # If no cost information found, return 0 verbose_logger.warning( - f"No cost information found for video model {model}. Please add pricing to model_prices_and_context_window.json" + "No cost information found for video model %s. Please add pricing to model_prices_and_context_window.json", + model, ) return 0.0 diff --git a/litellm/llms/openai/image_generation/cost_calculator.py b/litellm/llms/openai/image_generation/cost_calculator.py index b134ecc9a24..33893fd6fa7 100644 --- a/litellm/llms/openai/image_generation/cost_calculator.py +++ b/litellm/llms/openai/image_generation/cost_calculator.py @@ -20,7 +20,7 @@ def cost_calculator( """Calculate cost for OpenAI gpt-image models (token-based pricing).""" usage = getattr(image_response, "usage", None) if usage is None: - verbose_logger.debug(f"No usage data available for {model}, cannot calculate token-based cost") + verbose_logger.debug("No usage data available for %s, cannot calculate token-based cost", model) return 0.0 provider = custom_llm_provider or "openai" diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index f01730a06a5..151241dfb50 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -557,7 +557,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): except Exception as e: verbose_logger.exception( - f"LiteLLM.AgenticHookError: Exception in agentic completion hooks for OpenAI: {e}" + "LiteLLM.AgenticHookError: Exception in agentic completion hooks for OpenAI: %s", e ) return None diff --git a/litellm/llms/openai/responses/count_tokens/handler.py b/litellm/llms/openai/responses/count_tokens/handler.py index e59a28c2d09..e1deb47f457 100644 --- a/litellm/llms/openai/responses/count_tokens/handler.py +++ b/litellm/llms/openai/responses/count_tokens/handler.py @@ -45,7 +45,7 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig): try: self.validate_request(model, input) - verbose_logger.debug(f"Processing OpenAI CountTokens request for model: {model}") + verbose_logger.debug("Processing OpenAI CountTokens request for model: %s", model) request_body = self.transform_request_to_count_tokens( model=model, @@ -56,7 +56,7 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig): endpoint_url = self.get_openai_count_tokens_endpoint(api_base) - verbose_logger.debug(f"Making request to: {endpoint_url}") + verbose_logger.debug("Making request to: %s", endpoint_url) headers = self.get_required_headers(api_key) @@ -71,30 +71,30 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig): timeout=request_timeout, ) - verbose_logger.debug(f"Response status: {response.status_code}") + verbose_logger.debug("Response status: %s", response.status_code) if response.status_code != 200: error_text = response.text - verbose_logger.error(f"OpenAI API error: {error_text}") + verbose_logger.error("OpenAI API error: %s", error_text) raise OpenAIError( status_code=response.status_code, message=error_text, ) openai_response = response.json() - verbose_logger.debug(f"OpenAI response: {openai_response}") + verbose_logger.debug("OpenAI response: %s", openai_response) return openai_response except OpenAIError: raise except httpx.HTTPStatusError as e: - verbose_logger.error(f"HTTP error in CountTokens handler: {e}") + verbose_logger.error("HTTP error in CountTokens handler: %s", e) raise OpenAIError( status_code=e.response.status_code, message=e.response.text, ) except (httpx.RequestError, json.JSONDecodeError, ValueError) as e: - verbose_logger.error(f"Error in CountTokens handler: {e}") + verbose_logger.error("Error in CountTokens handler: %s", e) raise OpenAIError( status_code=500, message=f"CountTokens processing error: {e}", diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index d4494759f6c..af65cdbe91d 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -89,7 +89,7 @@ class OpenAITokenCounter(BaseTokenCounter): original_response=result, ) except OpenAIError as e: - verbose_logger.warning(f"OpenAI CountTokens API error: status={e.status_code}, message={e.message}") + verbose_logger.warning("OpenAI CountTokens API error: status=%s, message=%s", e.status_code, e.message) return TokenCountResponse( total_tokens=0, request_model=request_model, @@ -100,7 +100,7 @@ class OpenAITokenCounter(BaseTokenCounter): status_code=e.status_code, ) except Exception as e: - verbose_logger.warning(f"Error calling OpenAI CountTokens API: {e}") + verbose_logger.warning("Error calling OpenAI CountTokens API: %s", e) return TokenCountResponse( total_tokens=0, request_model=request_model, diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 2d0ce47e595..d84a9d2cda7 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -209,7 +209,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): elif isinstance(item, dict): # Handle reasoning items specifically to filter out status=None if item.get("type") == "reasoning": - verbose_logger.debug(f"Handling reasoning item: {item}") + verbose_logger.debug("Handling reasoning item: %s", item) # Type assertion since we know it's a dict at this point dict_item = cast(dict[str, Any], item) filtered_item = self._handle_reasoning_item(dict_item) @@ -251,7 +251,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return dict_reasoning_item except Exception as e: - verbose_logger.debug(f"Failed to create ResponseReasoningItem, falling back to manual filtering: {e}") + verbose_logger.debug("Failed to create ResponseReasoningItem, falling back to manual filtering: %s", e) # Fallback: manually filter out known None fields filtered_item = { k: v @@ -282,7 +282,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): try: response = ResponsesAPIResponse.model_validate(raw_response_json) except Exception: - verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct") + verbose_logger.debug( + "Error constructing ResponsesAPIResponse: %s, using model_construct", raw_response_json + ) response = ResponsesAPIResponse.model_construct(**raw_response_json) # Store processed headers in additional_headers so they get returned to the client @@ -429,7 +431,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ): return True except Exception as e: - verbose_logger.debug(f"Error getting model info in OpenAIResponsesAPIConfig: {e}") + verbose_logger.debug("Error getting model info in OpenAIResponsesAPIConfig: %s", e) return False def supports_native_websocket(self) -> bool: @@ -649,7 +651,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): try: response = ResponsesAPIResponse.model_validate(raw_response_json) except Exception: - verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct") + verbose_logger.debug( + "Error constructing ResponsesAPIResponse: %s, using model_construct", raw_response_json + ) response = ResponsesAPIResponse.model_construct(**raw_response_json) response._hidden_params["additional_headers"] = processed_headers diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index 40c3e2a07a7..57af268fe6a 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -110,8 +110,9 @@ def create_config_class(provider: SimpleProviderConfig): if param in supported_params: supported_params.remove(param) verbose_logger.debug( - f"Model {model} on provider {provider.slug} does not support " - f"function calling — removed tool-related params from supported params." + "Model %s on provider %s does not support function calling — removed tool-related params from supported params.", + model, + provider.slug, ) _supports_reasoning = supports_reasoning(model=model, custom_llm_provider=provider.slug) diff --git a/litellm/llms/openai_like/json_loader.py b/litellm/llms/openai_like/json_loader.py index bc10b7bd62f..5c6e8f643ce 100644 --- a/litellm/llms/openai_like/json_loader.py +++ b/litellm/llms/openai_like/json_loader.py @@ -51,7 +51,7 @@ class JSONProviderRegistry: cls._loaded = True except Exception as e: - verbose_logger.warning(f"Warning: Failed to load JSON provider configs: {e}") + verbose_logger.warning("Warning: Failed to load JSON provider configs: %s", e) cls._loaded = True @classmethod diff --git a/litellm/llms/perplexity/chat/transformation.py b/litellm/llms/perplexity/chat/transformation.py index c6fb750ec1b..5a4e3440201 100644 --- a/litellm/llms/perplexity/chat/transformation.py +++ b/litellm/llms/perplexity/chat/transformation.py @@ -52,13 +52,13 @@ class PerplexityChatConfig(OpenAIGPTConfig): if litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider): base_openai_params.append("reasoning_effort") except Exception as e: - verbose_logger.debug(f"Error checking if model supports reasoning: {e}") + verbose_logger.debug("Error checking if model supports reasoning: %s", e) try: if litellm.supports_web_search(model=model, custom_llm_provider=self.custom_llm_provider): base_openai_params.append("web_search_options") except Exception as e: - verbose_logger.debug(f"Error checking if model supports web search: {e}") + verbose_logger.debug("Error checking if model supports web search: %s", e) return base_openai_params @@ -97,7 +97,7 @@ class PerplexityChatConfig(OpenAIGPTConfig): self._enhance_usage_with_perplexity_fields(model_response, raw_response_json) self._add_citations_as_annotations(model_response, raw_response_json) except Exception as e: - verbose_logger.debug(f"Error extracting Perplexity-specific usage fields: {e}") + verbose_logger.debug("Error extracting Perplexity-specific usage fields: %s", e) return model_response diff --git a/litellm/llms/runwayml/image_generation/transformation.py b/litellm/llms/runwayml/image_generation/transformation.py index ba51a7ba093..5833892c866 100644 --- a/litellm/llms/runwayml/image_generation/transformation.py +++ b/litellm/llms/runwayml/image_generation/transformation.py @@ -170,7 +170,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): """ status = response_data.get("status", "").upper() - verbose_logger.debug(f"RunwayML task status: {status}") + verbose_logger.debug("RunwayML task status: %s", status) if status == "SUCCEEDED": return "succeeded" @@ -216,7 +216,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): api_base = api_base.rstrip("/") task_url = f"{api_base}/v1/tasks/{task_id}" - verbose_logger.debug(f"Polling RunwayML task: {task_url}") + verbose_logger.debug("Polling RunwayML task: %s", task_url) while True: self._check_timeout(start_time=start_time, timeout_secs=timeout_secs) @@ -265,7 +265,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): api_base = api_base.rstrip("/") task_url = f"{api_base}/v1/tasks/{task_id}" - verbose_logger.debug(f"Polling RunwayML task (async): {task_url}") + verbose_logger.debug("Polling RunwayML task (async): %s", task_url) while True: self._check_timeout(start_time=start_time, timeout_secs=timeout_secs) diff --git a/litellm/llms/runwayml/text_to_speech/transformation.py b/litellm/llms/runwayml/text_to_speech/transformation.py index 46a5f606853..21ac716df01 100644 --- a/litellm/llms/runwayml/text_to_speech/transformation.py +++ b/litellm/llms/runwayml/text_to_speech/transformation.py @@ -259,7 +259,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): """ status = response_data.get("status", "").upper() - verbose_logger.debug(f"RunwayML TTS task status: {status}") + verbose_logger.debug("RunwayML TTS task status: %s", status) if status == "SUCCEEDED": return "succeeded" @@ -305,7 +305,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): api_base = api_base.rstrip("/") task_url = f"{api_base}/v1/tasks/{task_id}" - verbose_logger.debug(f"Polling RunwayML TTS task: {task_url}") + verbose_logger.debug("Polling RunwayML TTS task: %s", task_url) while True: self._check_timeout(start_time=start_time, timeout_secs=timeout_secs) @@ -353,7 +353,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): api_base = api_base.rstrip("/") task_url = f"{api_base}/v1/tasks/{task_id}" - verbose_logger.debug(f"Polling RunwayML TTS task (async): {task_url}") + verbose_logger.debug("Polling RunwayML TTS task (async): %s", task_url) while True: self._check_timeout(start_time=start_time, timeout_secs=timeout_secs) diff --git a/litellm/llms/sagemaker/common_utils.py b/litellm/llms/sagemaker/common_utils.py index 5f5c0250273..5916747bf28 100644 --- a/litellm/llms/sagemaker/common_utils.py +++ b/litellm/llms/sagemaker/common_utils.py @@ -130,7 +130,7 @@ class AWSEventStreamDecoder: yield self._chunk_parser(chunk_data=_data) except json.JSONDecodeError: # Handle or log any unparseable data at the end - verbose_logger.error(f"Warning: Unparseable JSON data remained: {accumulated_json}") + verbose_logger.error("Warning: Unparseable JSON data remained: %s", accumulated_json) yield None async def aiter_bytes( @@ -168,10 +168,10 @@ class AWSEventStreamDecoder: # If it's not valid JSON yet, continue to the next event continue except UnicodeDecodeError as e: - verbose_logger.warning(f"UnicodeDecodeError: {e}. Attempting to combine with next event.") + verbose_logger.warning("UnicodeDecodeError: %s. Attempting to combine with next event.", e) continue except Exception as e: - verbose_logger.error(f"Error parsing message: {e}. Attempting to combine with next event.") + verbose_logger.error("Error parsing message: %s. Attempting to combine with next event.", e) continue # Handle any remaining data after the iterator is exhausted @@ -184,10 +184,10 @@ class AWSEventStreamDecoder: yield self._chunk_parser(chunk_data=_data) except json.JSONDecodeError: # Handle or log any unparseable data at the end - verbose_logger.error(f"Warning: Unparseable JSON data remained: {accumulated_json}") + verbose_logger.error("Warning: Unparseable JSON data remained: %s", accumulated_json) yield None except Exception as e: - verbose_logger.error(f"Final error parsing accumulated JSON: {e}") + verbose_logger.error("Final error parsing accumulated JSON: %s", e) def _parse_message_from_event(self, event) -> str | None: response_stream_shape = get_sagemaker_response_stream_shape() diff --git a/litellm/llms/sap/credentials.py b/litellm/llms/sap/credentials.py index 5b2e02875b8..fe5df75fc72 100644 --- a/litellm/llms/sap/credentials.py +++ b/litellm/llms/sap/credentials.py @@ -45,7 +45,7 @@ def _get_nested(d: dict[str, Any] | str, path: Sequence[str]) -> Any: for k in path: if not isinstance(cur, dict): verbose_logger.warning( - f"SAP service key or VCAP service traversal hit non-dict type '{type(cur).__name__}' at key '{k}'." + "SAP service key or VCAP service traversal hit non-dict type '%s' at key '%s'.", type(cur).__name__, k ) return None if k not in cur: @@ -173,7 +173,7 @@ def resolve_credentials(sources: list[Source]) -> dict[str, str]: for source in sources: credentials = extract_credentials(source) if credentials: - verbose_logger.debug(f"Resolved SAP credentials from source {source.name}") + verbose_logger.debug("Resolved SAP credentials from source %s", source.name) return credentials raise ValueError("No credentials found in any source") @@ -184,7 +184,7 @@ def resolve_resource_group(sources: list[Source]) -> str | None: for source in sources: value = source.get(rg_cred) if value is not None: - verbose_logger.debug(f"Resolved GEN AI Hub resource_group from source {source.name}") + verbose_logger.debug("Resolved GEN AI Hub resource_group from source %s", source.name) return value return rg_cred.default @@ -208,7 +208,7 @@ def _parse_service_key_once( verbose_logger.warning("SAP service key is a string but not valid JSON. Skipping this source.") return None verbose_logger.warning( - f"SAP service key has unexpected type '{type(service_key).__name__}'. Expected str or dict. Ignoring." + "SAP service key has unexpected type '%s'. Expected str or dict. Ignoring.", type(service_key).__name__ ) return None diff --git a/litellm/llms/together_ai/chat.py b/litellm/llms/together_ai/chat.py index 5920f02a44e..0eefd1ff1d6 100644 --- a/litellm/llms/together_ai/chat.py +++ b/litellm/llms/together_ai/chat.py @@ -29,7 +29,7 @@ class TogetherAIConfig(OpenAIGPTConfig): try: supports_fc = supports_function_calling(model, custom_llm_provider="together_ai") except Exception as e: - verbose_logger.debug(f"Error getting supported openai params: {e}") + verbose_logger.debug("Error getting supported openai params: %s", e) optional_params = super().get_supported_openai_params(model) if supports_fc is not True: diff --git a/litellm/llms/vertex_ai/agent_engine/transformation.py b/litellm/llms/vertex_ai/agent_engine/transformation.py index 9f01a9ee506..b40e38916ed 100644 --- a/litellm/llms/vertex_ai/agent_engine/transformation.py +++ b/litellm/llms/vertex_ai/agent_engine/transformation.py @@ -137,7 +137,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): # (create_session, get_session, list_sessions, delete_session, etc.) endpoint = f"{base_url}/v1beta1/{resource_path}:streamQuery" - verbose_logger.debug(f"Vertex Agent Engine URL: {endpoint}") + verbose_logger.debug("Vertex Agent Engine URL: %s", endpoint) return endpoint def _get_auth_headers( @@ -155,7 +155,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): project_id=vertex_project, ) - verbose_logger.debug(f"Vertex Agent Engine: Authenticated for project {project_id}") + verbose_logger.debug("Vertex Agent Engine: Authenticated for project %s", project_id) return { "Authorization": f"Bearer {access_token}", @@ -219,7 +219,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): "input": input_data, } - verbose_logger.debug(f"Vertex Agent Engine payload: {payload}") + verbose_logger.debug("Vertex Agent Engine payload: %s", payload) return payload def validate_environment( @@ -270,7 +270,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): total_tokens=total_tokens, ) except Exception as e: - verbose_logger.warning(f"Failed to calculate token usage: {e}") + verbose_logger.warning("Failed to calculate token usage: %s", e) return None def transform_response( @@ -295,11 +295,11 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): """ try: content_type = raw_response.headers.get("content-type", "").lower() - verbose_logger.debug(f"Vertex Agent Engine response Content-Type: {content_type}") + verbose_logger.debug("Vertex Agent Engine response Content-Type: %s", content_type) # Parse the SSE response response_text = raw_response.text - verbose_logger.debug(f"Response (first 500 chars): {response_text[:500]}") + verbose_logger.debug("Response (first 500 chars): %s", response_text[:500]) # Extract content from SSE stream content = "" @@ -335,7 +335,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): return model_response except Exception as e: - verbose_logger.error(f"Error processing Vertex Agent Engine response: {e}") + verbose_logger.error("Error processing Vertex Agent Engine response: %s", e) raise VertexAgentEngineError( message=f"Error processing response: {e}", status_code=raw_response.status_code, diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 81d084e7e03..8d63df1ea3d 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -221,7 +221,8 @@ def get_supports_system_message( supports_system_message = True except Exception as e: verbose_logger.warning( - f"Unable to identify if system message supported. Defaulting to 'False'. Received error message - {e}\nAdd it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json" + "Unable to identify if system message supported. Defaulting to 'False'. Received error message - %s\nAdd it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json", + e, ) supports_system_message = False diff --git a/litellm/llms/vertex_ai/cost_calculator.py b/litellm/llms/vertex_ai/cost_calculator.py index a53e54e5fc2..572a4ed14db 100644 --- a/litellm/llms/vertex_ai/cost_calculator.py +++ b/litellm/llms/vertex_ai/cost_calculator.py @@ -114,7 +114,8 @@ def cost_per_character( prompt_cost = prompt_characters * model_info["input_cost_per_character"] except Exception as e: verbose_logger.debug( - f"litellm.litellm_core_utils.llm_cost_calc.google.py::cost_per_character(): Exception occured - {e}\nDefaulting to None" + "litellm.litellm_core_utils.llm_cost_calc.google.py::cost_per_character(): Exception occured - %s\nDefaulting to None", + e, ) prompt_cost, _ = cost_per_token( model=model, @@ -152,7 +153,8 @@ def cost_per_character( completion_cost = completion_characters * model_info["output_cost_per_character"] except Exception as e: verbose_logger.debug( - f"litellm.litellm_core_utils.llm_cost_calc.google.py::cost_per_character(): Exception occured - {e}\nDefaulting to None" + "litellm.litellm_core_utils.llm_cost_calc.google.py::cost_per_character(): Exception occured - %s\nDefaulting to None", + e, ) _, completion_cost = cost_per_token( model=model, diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index cadc8760601..c3977b19c87 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -447,8 +447,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): transformed_config["environment"] = env_value else: verbose_logger.info( - f"Invalid environment value for computer_use: {env_value}. " - f"Supported: 'browser', 'unspecified', 'ENVIRONMENT_BROWSER', 'ENVIRONMENT_UNSPECIFIED'" + "Invalid environment value for computer_use: %s. Supported: 'browser', 'unspecified', 'ENVIRONMENT_BROWSER', 'ENVIRONMENT_UNSPECIFIED'", + env_value, ) # Transform excluded_predefined_functions to camelCase @@ -626,7 +626,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "web_search", "web_search_preview", ): - verbose_logger.info(f"Gemini: Transforming OpenAI-style '{tool['type']}' tool to googleSearch") + verbose_logger.info("Gemini: Transforming OpenAI-style '%s' tool to googleSearch", tool["type"]) tool = {VertexToolName.GOOGLE_SEARCH.value: {}} # Handle tools with 'type' field (OpenAI spec compliance) Ignore this field -> https://github.com/BerriAI/litellm/issues/14644#issuecomment-3342061838 elif "type" in tool: @@ -1087,36 +1087,29 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if VertexGeminiConfig._is_gemini_3_or_newer(model): if value is not None and value < 1.0: verbose_logger.info( - f"Warning: Setting temperature < 1.0 for Gemini 3 models ({model}) " - "can cause infinite loops, degraded reasoning performance, and failure on complex tasks. " - "Strongly recommended to use temperature = 1.0 (default)." + "Warning: Setting temperature < 1.0 for Gemini 3 models (%s) can cause infinite loops, degraded reasoning performance, and failure on complex tasks. Strongly recommended to use temperature = 1.0 (default).", + model, ) if not gemini_sampling_params_warned: verbose_logger.warning( - "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to " - f"function for Gemini 3+ ({model}) but are planned for removal in a " - "future release. Move sampling guidance into the `system` " - "instructions instead." + "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.", + model, ) gemini_sampling_params_warned = True optional_params["temperature"] = value elif param == "top_p": if VertexGeminiConfig._is_gemini_3_or_newer(model) and not gemini_sampling_params_warned: verbose_logger.warning( - "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to " - f"function for Gemini 3+ ({model}) but are planned for removal in a " - "future release. Move sampling guidance into the `system` " - "instructions instead." + "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.", + model, ) gemini_sampling_params_warned = True optional_params["top_p"] = value elif param == "top_k": if VertexGeminiConfig._is_gemini_3_or_newer(model) and not gemini_sampling_params_warned: verbose_logger.warning( - "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to " - f"function for Gemini 3+ ({model}) but are planned for removal in a " - "future release. Move sampling guidance into the `system` " - "instructions instead." + "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.", + model, ) gemini_sampling_params_warned = True optional_params["top_k"] = value @@ -1977,7 +1970,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): prompt_feedback = processed_chunk.get("promptFeedback") if prompt_feedback and "blockReason" in prompt_feedback: verbose_logger.debug( - f"Prompt blocked due to: {prompt_feedback.get('blockReason')} - {prompt_feedback.get('blockReasonMessage')}" + "Prompt blocked due to: %s - %s", + prompt_feedback.get("blockReason"), + prompt_feedback.get("blockReasonMessage"), ) # Create a content_filter response (consistent with non-streaming _handle_blocked_response) @@ -3248,7 +3243,7 @@ class ModelResponseIterator: def chunk_parser(self, chunk: dict) -> Optional["ModelResponseStream"]: try: - verbose_logger.debug(f"RAW GEMINI CHUNK: {chunk}") + verbose_logger.debug("RAW GEMINI CHUNK: %s", chunk) # Detect mid-stream error chunks (e.g. 429 RESOURCE_EXHAUSTED). # Vertex AI can return errors as HTTP 200 but with an "error" field in the SSE body. diff --git a/litellm/llms/vertex_ai/ocr/deepseek_transformation.py b/litellm/llms/vertex_ai/ocr/deepseek_transformation.py index b66dead91b1..f148f8e8be8 100644 --- a/litellm/llms/vertex_ai/ocr/deepseek_transformation.py +++ b/litellm/llms/vertex_ai/ocr/deepseek_transformation.py @@ -259,7 +259,7 @@ class VertexAIDeepSeekOCRConfig(BaseOCRConfig): OCRResponse in standard format """ verbose_logger.debug("Vertex AI DeepSeek OCR transform_ocr_response called") - verbose_logger.debug(f"Raw response: {raw_response.text}") + verbose_logger.debug("Raw response: %s", raw_response.text) try: response_json = raw_response.json() @@ -345,7 +345,7 @@ class VertexAIDeepSeekOCRConfig(BaseOCRConfig): ) except Exception as e: - verbose_logger.error(f"Error parsing Vertex AI DeepSeek OCR response: {e}") + verbose_logger.error("Error parsing Vertex AI DeepSeek OCR response: %s", e) raise e async def async_transform_ocr_response( diff --git a/litellm/llms/vertex_ai/ocr/transformation.py b/litellm/llms/vertex_ai/ocr/transformation.py index 0fb9523f3eb..e1254c5f833 100644 --- a/litellm/llms/vertex_ai/ocr/transformation.py +++ b/litellm/llms/vertex_ai/ocr/transformation.py @@ -138,13 +138,13 @@ class VertexAIOCRConfig(MistralOCRConfig): Returns: Base64 data URI string """ - verbose_logger.debug(f"Vertex AI OCR: Converting URL to base64 data URI (sync): {url}") + verbose_logger.debug("Vertex AI OCR: Converting URL to base64 data URI (sync): %s", url) # Fetch and convert to base64 data URI # convert_url_to_base64 already returns a full data URI like "data:image/jpeg;base64,..." data_uri = convert_url_to_base64(url=url) - verbose_logger.debug(f"Vertex AI OCR: Converted URL to data URI (length: {len(data_uri)})") + verbose_logger.debug("Vertex AI OCR: Converted URL to data URI (length: %s)", len(data_uri)) return data_uri @@ -161,13 +161,13 @@ class VertexAIOCRConfig(MistralOCRConfig): Returns: Base64 data URI string """ - verbose_logger.debug(f"Vertex AI OCR: Converting URL to base64 data URI (async): {url}") + verbose_logger.debug("Vertex AI OCR: Converting URL to base64 data URI (async): %s", url) # Fetch and convert to base64 data URI asynchronously # async_convert_url_to_base64 already returns a full data URI like "data:image/jpeg;base64,..." data_uri = await async_convert_url_to_base64(url=url) - verbose_logger.debug(f"Vertex AI OCR: Converted URL to data URI (length: {len(data_uri)})") + verbose_logger.debug("Vertex AI OCR: Converted URL to data URI (length: %s)", len(data_uri)) return data_uri @@ -252,7 +252,7 @@ class VertexAIOCRConfig(MistralOCRConfig): Returns: OCRRequestData with JSON data """ - verbose_logger.debug(f"Vertex AI OCR async_transform_ocr_request - model: {model}") + verbose_logger.debug("Vertex AI OCR async_transform_ocr_request - model: %s", model) if not isinstance(document, dict): raise ValueError(f"Expected document dict, got {type(document)}") diff --git a/litellm/llms/vertex_ai/rag_engine/ingestion.py b/litellm/llms/vertex_ai/rag_engine/ingestion.py index edafa2f8f7a..cb30dc0ce0a 100644 --- a/litellm/llms/vertex_ai/rag_engine/ingestion.py +++ b/litellm/llms/vertex_ai/rag_engine/ingestion.py @@ -133,7 +133,7 @@ class VertexAIRAGIngestion(BaseRAGIngestion): file_tuple = (filename, file_content, content_type) verbose_logger.debug( - f"Uploading file to GCS via litellm.files.acreate_file: {filename} (bucket: {self.gcs_bucket})" + "Uploading file to GCS via litellm.files.acreate_file: %s (bucket: %s)", filename, self.gcs_bucket ) # Upload to GCS using LiteLLM's file upload @@ -148,7 +148,7 @@ class VertexAIRAGIngestion(BaseRAGIngestion): # The response.id should be the GCS URI gcs_uri = response.id - verbose_logger.info(f"Uploaded file to GCS: {gcs_uri}") + verbose_logger.info("Uploaded file to GCS: %s", gcs_uri) return gcs_uri finally: @@ -185,7 +185,7 @@ class VertexAIRAGIngestion(BaseRAGIngestion): transformation_config = self._build_transformation_config() corpus_name = self._get_corpus_name() - verbose_logger.debug(f"Importing {gcs_uri} into corpus {self.corpus_id}") + verbose_logger.debug("Importing %s into corpus %s", gcs_uri, self.corpus_id) if self.wait_for_import: # Synchronous import - wait for completion @@ -195,7 +195,7 @@ class VertexAIRAGIngestion(BaseRAGIngestion): transformation_config=transformation_config, timeout=self.import_timeout, ) - verbose_logger.info(f"Import complete: {response.imported_rag_files_count} files imported") + verbose_logger.info("Import complete: %s files imported", response.imported_rag_files_count) else: # Async import - don't wait _ = rag.import_files_async( @@ -293,7 +293,7 @@ class VertexAIRAGIngestion(BaseRAGIngestion): try: await self._import_file_to_corpus_via_sdk(gcs_uri=gcs_uri) except Exception as e: - verbose_logger.error(f"Failed to import file into RAG corpus: {e}") + verbose_logger.error("Failed to import file into RAG corpus: %s", e) raise RuntimeError(f"Failed to import file into RAG corpus: {e}") from e return str(self.corpus_id), gcs_uri diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 2def1acb708..c0a6bd01167 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -766,7 +766,7 @@ class VertexBase: The original error if reauthentication fails """ verbose_logger.debug( - f"Handling reauthentication for project_id: {project_id}. Clearing cache and retrying once." + "Handling reauthentication for project_id: %s. Clearing cache and retrying once.", project_id ) # Clear the cached credentials @@ -782,8 +782,10 @@ class VertexBase: ) except Exception as retry_error: verbose_logger.error( - f"Reauthentication retry failed for project_id: {project_id}. " - f"Original error: {error}. Retry error: {retry_error}" + "Reauthentication retry failed for project_id: %s. Original error: %s. Retry error: %s", + project_id, + error, + retry_error, ) # Re-raise the original error for better context raise error @@ -799,7 +801,7 @@ class VertexBase: Async reauthentication retry that stays within the per-key async lock. """ verbose_logger.debug( - f"Handling async reauthentication for project_id: {project_id}. Clearing cache and retrying once." + "Handling async reauthentication for project_id: %s. Clearing cache and retrying once.", project_id ) self._credentials_project_mapping.pop(credential_cache_key, None) @@ -836,8 +838,10 @@ class VertexBase: return _credentials.token, project_id except Exception as retry_error: verbose_logger.error( - f"Async reauthentication retry failed for project_id: {project_id}. " - f"Original error: {error}. Retry error: {retry_error}" + "Async reauthentication retry failed for project_id: %s. Original error: %s. Retry error: %s", + project_id, + error, + retry_error, ) raise error @@ -870,10 +874,10 @@ class VertexBase: credential_cache_key = (cache_credentials, project_id) _credentials: GoogleCredentialsObject | None = None - verbose_logger.debug(f"Checking cached credentials for project_id: {project_id}") + verbose_logger.debug("Checking cached credentials for project_id: %s", project_id) if credential_cache_key in self._credentials_project_mapping: - verbose_logger.debug(f"Cached credentials found for project_id: {project_id}.") + verbose_logger.debug("Cached credentials found for project_id: %s.", project_id) # Retrieve both credentials and cached project_id cached_entry = self._credentials_project_mapping[credential_cache_key] verbose_logger.debug("cached_entry: %s", cached_entry) @@ -890,14 +894,15 @@ class VertexBase: else: verbose_logger.debug( - f"Credential cache key not found for project_id: {project_id}, loading new credentials" + "Credential cache key not found for project_id: %s, loading new credentials", project_id ) try: _credentials, credential_project_id = self.load_auth(credentials=credentials, project_id=project_id) except Exception as e: verbose_logger.exception( - f"Failed to load vertex credentials. Check to see if credentials containing partial/invalid information. Error: {e}" + "Failed to load vertex credentials. Check to see if credentials containing partial/invalid information. Error: %s", + e, ) raise e diff --git a/litellm/llms/watsonx/chat/transformation.py b/litellm/llms/watsonx/chat/transformation.py index adc7035d2f7..0990edc5c42 100644 --- a/litellm/llms/watsonx/chat/transformation.py +++ b/litellm/llms/watsonx/chat/transformation.py @@ -190,7 +190,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig): # Log the exception for debugging but don't raise it # The caller will fall back to default prompt factory try: - verbose_logger.debug(f"Failed to apply HuggingFace template for model {hf_model}: {e}") + verbose_logger.debug("Failed to apply HuggingFace template for model %s: %s", hf_model, e) except Exception: # If logging fails, silently continue - don't break the flow pass diff --git a/litellm/llms/xai/chat/transformation.py b/litellm/llms/xai/chat/transformation.py index 98d8fe5fadd..fa2f957260e 100644 --- a/litellm/llms/xai/chat/transformation.py +++ b/litellm/llms/xai/chat/transformation.py @@ -144,7 +144,7 @@ class XAIChatConfig(OpenAIGPTConfig): if litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider): base_openai_params.append("reasoning_effort") except Exception as e: - verbose_logger.debug(f"Error checking if model supports reasoning: {e}") + verbose_logger.debug("Error checking if model supports reasoning: %s", e) return base_openai_params @@ -277,7 +277,7 @@ class XAIChatConfig(OpenAIGPTConfig): raw_response_json = raw_response.json() self._enhance_usage_with_xai_web_search_fields(response, raw_response_json) except Exception as e: - verbose_logger.debug(f"Error extracting X.AI web search usage: {e}") + verbose_logger.debug("Error extracting X.AI web search usage: %s", e) self._fold_reasoning_tokens_into_completion(response) self._normalize_openai_compatible_usage_totals(getattr(response, "usage", None)) @@ -369,7 +369,7 @@ class XAIChatConfig(OpenAIGPTConfig): usage.prompt_tokens_details.web_search_requests = int(num_sources_used) setattr(usage, "num_sources_used", int(num_sources_used)) - verbose_logger.debug(f"X.AI web search sources used: {num_sources_used}") + verbose_logger.debug("X.AI web search sources used: %s", num_sources_used) @staticmethod def _normalize_openai_compatible_usage_totals( diff --git a/litellm/main.py b/litellm/main.py index 731a545a267..8b2c3c72f76 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -550,7 +550,7 @@ async def acompletion( # Log shared session usage if shared_session is not None: - verbose_logger.debug(f"🔄 SHARED SESSION: acompletion called with shared_session (ID: {id(shared_session)})") + verbose_logger.debug("🔄 SHARED SESSION: acompletion called with shared_session (ID: %s)", id(shared_session)) else: verbose_logger.debug("🔄 NO SHARED SESSION: acompletion called without shared_session") @@ -1002,7 +1002,7 @@ def responses_api_bridge_check( model = model.replace("responses/", "") except Exception as e: - verbose_logger.debug(f"Error getting model info: {e}") + verbose_logger.debug("Error getting model info: %s", e) if model.startswith("responses/"): # handle azure models - `azure/responses/` model = model.replace("responses/", "") @@ -2817,7 +2817,7 @@ def _complete_cohere_chat(ctx: _CompletionDispatchContext) -> _CompletionDispatc ) cohere_route = CohereModelInfo.get_cohere_route(model) - verbose_logger.debug(f"Cohere route: {cohere_route}") + verbose_logger.debug("Cohere route: %s", cohere_route) # Set API base based on route if cohere_route == "v2": api_base = api_base or litellm.api_base or get_secret_str("COHERE_API_BASE") or "https://api.cohere.com/v2/chat" @@ -2834,8 +2834,8 @@ def _complete_cohere_chat(ctx: _CompletionDispatchContext) -> _CompletionDispatc if extra_headers is not None: headers.update(extra_headers) - verbose_logger.debug(f"Model: {model}, API Base: {api_base}") - verbose_logger.debug(f"Provider Config: {provider_config}") + verbose_logger.debug("Model: %s, API Base: %s", model, api_base) + verbose_logger.debug("Provider Config: %s", provider_config) return base_llm_http_handler.completion( model=model, stream=stream, @@ -4998,7 +4998,7 @@ def completion( # type: ignore proxy_headers = litellm.proxy_auth.get_auth_headers() headers.update(proxy_headers) except Exception as e: - verbose_logger.warning(f"Failed to get proxy auth headers: {e}") + verbose_logger.warning("Failed to get proxy auth headers: %s", e) num_retries = kwargs.get( "num_retries", None ) ## alt. param for 'max_retries'. Use this to pass retries w/ instructor. @@ -5965,7 +5965,7 @@ def embedding( proxy_headers = litellm.proxy_auth.get_auth_headers() headers.update(proxy_headers) except Exception as e: - verbose_logger.warning(f"Failed to get proxy auth headers: {e}") + verbose_logger.warning("Failed to get proxy auth headers: %s", e) ### CUSTOM MODEL COST ### input_cost_per_token = kwargs.get("input_cost_per_token", None) output_cost_per_token = kwargs.get("output_cost_per_token", None) @@ -8669,7 +8669,7 @@ def stream_chunk_builder( processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj) return response except Exception as e: - verbose_logger.exception(f"litellm.main.py::stream_chunk_builder() - Exception occurred - {e}") + verbose_logger.exception("litellm.main.py::stream_chunk_builder() - Exception occurred - %s", e) raise litellm.APIError( status_code=500, message="Error building chunks for logging/streaming usage calculation", @@ -8759,7 +8759,7 @@ async def acount_tokens( if result is not None and not result.error: return result except Exception as e: - verbose_logger.debug(f"Provider token counting failed for model={model}, falling back to local: {e}") + verbose_logger.debug("Provider token counting failed for model=%s, falling back to local: %s", model, e) # Fallback to local tiktoken-based token counting fallback_messages = messages or [] diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index f53f32eecfa..171f3286a7d 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -119,7 +119,7 @@ def _prepare_ocr_request( if ocr_provider_config is None: raise ValueError(f"OCR is not supported for provider: {custom_llm_provider}") - verbose_logger.debug(f"OCR call - model: {model}, provider: {custom_llm_provider}") + verbose_logger.debug("OCR call - model: %s, provider: %s", model, custom_llm_provider) litellm_params = GenericLiteLLMParams.model_validate(kwargs) @@ -135,7 +135,7 @@ def _prepare_ocr_request( model=model, ) - verbose_logger.debug(f"OCR optional_params after mapping: {optional_params}") + verbose_logger.debug("OCR optional_params after mapping: %s", optional_params) effective_timeout = timeout or request_timeout @@ -553,14 +553,18 @@ def convert_file_document_to_url_document(document: dict[str, Any]) -> dict[str, if mime_type.startswith("image/"): verbose_logger.debug( - f"OCR file input: Converted file to image_url data URI " - f"(mime={mime_type}, size={len(file_bytes)} bytes, name={file_name})" + "OCR file input: Converted file to image_url data URI (mime=%s, size=%s bytes, name=%s)", + mime_type, + len(file_bytes), + file_name, ) return {"type": "image_url", "image_url": data_uri} verbose_logger.debug( - f"OCR file input: Converted file to document_url data URI " - f"(mime={mime_type}, size={len(file_bytes)} bytes, name={file_name})" + "OCR file input: Converted file to document_url data URI (mime=%s, size=%s bytes, name=%s)", + mime_type, + len(file_bytes), + file_name, ) return {"type": "document_url", "document_url": data_uri} diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index a256653f0f9..07a4eb164c3 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -384,14 +384,14 @@ class MCPRequestHandler: # Parse MCP servers from header mcp_servers_header = headers.get(MCPRequestHandler.LITELLM_MCP_SERVERS_HEADER_NAME) - verbose_logger.debug(f"Raw MCP servers header: {mcp_servers_header}") + verbose_logger.debug("Raw MCP servers header: %s", mcp_servers_header) mcp_servers = None if mcp_servers_header is not None: try: mcp_servers = [s.strip() for s in mcp_servers_header.split(",") if s.strip()] - verbose_logger.debug(f"Parsed MCP servers: {mcp_servers}") + verbose_logger.debug("Parsed MCP servers: %s", mcp_servers) except Exception as e: - verbose_logger.debug(f"Error parsing mcp_servers header: {e}") + verbose_logger.debug("Error parsing mcp_servers header: %s", e) mcp_servers = None if mcp_servers_header == "" or (mcp_servers is not None and len(mcp_servers) == 0): mcp_servers = [] @@ -1008,7 +1008,7 @@ class MCPRequestHandler: limits[source.team_id] = applicable return limits or None except Exception as e: # noqa: BLE001 # throttling metadata must never fail an allowed request - verbose_logger.warning(f"Failed to resolve per-team MCP rpm limits for admitted subject: {e}") + verbose_logger.warning("Failed to resolve per-team MCP rpm limits for admitted subject: %s", e) return None @staticmethod @@ -1091,7 +1091,7 @@ class MCPRequestHandler: user_id_upsert=False, ) except Exception as e: # noqa: BLE001 # mirror the builder's fail-open user lookup; DB errors are of any type - verbose_logger.debug(f"bridge admission: user lookup failed, skipping SCIM gate: {e}") + verbose_logger.debug("bridge admission: user lookup failed, skipping SCIM gate: %s", e) user_object = None if user_object is None or not isinstance(user_object.metadata, dict): return @@ -1194,8 +1194,8 @@ class MCPRequestHandler: auth_header = headers.get(mcp_client_side_auth_header_name) if auth_header: verbose_logger.warning( - f"The '{mcp_client_side_auth_header_name}' header is deprecated. " - f"Please use server-specific auth headers in the format 'x-mcp-{{server_alias}}-{{header_name}}' instead." + "The '%s' header is deprecated. Please use server-specific auth headers in the format 'x-mcp-{server_alias}-{header_name}' instead.", + mcp_client_side_auth_header_name, ) return auth_header @@ -1245,7 +1245,10 @@ class MCPRequestHandler: server_auth_headers[server_alias][auth_header_name] = header_value verbose_logger.debug( - f"Found server auth header: {server_alias} -> {auth_header_name}: {header_value[:10]}..." + "Found server auth header: %s -> %s: %s...", + server_alias, + auth_header_name, + header_value[:10], ) return server_auth_headers @@ -1331,7 +1334,7 @@ class MCPRequestHandler: headers_dict = {name.decode("latin-1"): value.decode("latin-1") for name, value in raw_headers} return Headers(headers_dict) except (UnicodeDecodeError, AttributeError, TypeError) as e: - verbose_logger.exception(f"Error getting headers from scope: {e}") + verbose_logger.exception("Error getting headers from scope: %s", e) # Return empty Headers object with empty dict return Headers({}) @@ -1455,7 +1458,9 @@ class MCPRequestHandler: if len(allowed_mcp_servers_for_end_user) > 0: has_lower_level_mcp_restrictions = True verbose_logger.debug( - f"End user {user_api_key_auth.end_user_id} has explicit MCP permissions: {allowed_mcp_servers_for_end_user}" + "End user %s has explicit MCP permissions: %s", + user_api_key_auth.end_user_id, + allowed_mcp_servers_for_end_user, ) # Always apply intersection: key/team AND end_user @@ -1466,12 +1471,13 @@ class MCPRequestHandler: filtered_servers.append(_mcp_server) allowed_mcp_servers = filtered_servers verbose_logger.debug( - f"Applied end_user intersection filter. Final allowed servers: {allowed_mcp_servers}" + "Applied end_user intersection filter. Final allowed servers: %s", allowed_mcp_servers ) # If flag is enabled but end_user has no permissions, block all access elif general_settings.get("require_end_user_mcp_access_defined", False): verbose_logger.debug( - f"require_end_user_mcp_access_defined=True and end_user {user_api_key_auth.end_user_id} has no MCP permissions - blocking MCP access" + "require_end_user_mcp_access_defined=True and end_user %s has no MCP permissions - blocking MCP access", + user_api_key_auth.end_user_id, ) return [] @@ -1487,7 +1493,7 @@ class MCPRequestHandler: # Intersect: agent can only use servers allowed by BOTH key/team AND agent config allowed_mcp_servers = [s for s in allowed_mcp_servers if s in allowed_mcp_servers_for_agent] verbose_logger.debug( - f"Applied agent intersection filter. Final allowed servers: {allowed_mcp_servers}" + "Applied agent intersection filter. Final allowed servers: %s", allowed_mcp_servers ) ######################################################### @@ -1514,9 +1520,9 @@ class MCPRequestHandler: if isinstance(e, UnloadableEntitlementError): # A ceiling we KNOW exists and cannot read. Denying is the only answer that does not # widen this caller past what an operator configured, for both caller shapes. - verbose_logger.warning(f"Denying MCP access, entitlement unreadable: {e}") + verbose_logger.warning("Denying MCP access, entitlement unreadable: %s", e) else: - verbose_logger.warning(f"Failed to get allowed MCP servers: {e}") + verbose_logger.warning("Failed to get allowed MCP servers: %s", e) return [] @staticmethod @@ -1543,8 +1549,9 @@ class MCPRequestHandler: allowed_mcp_servers_for_org = await MCPRequestHandler._get_allowed_mcp_servers_for_org(user_api_key_auth) if allowed_mcp_servers_for_org is None: verbose_logger.warning( - f"MCP org ceiling unresolved for org_id={user_api_key_auth.org_id!r}; " - f"{'denying (keyless admitted subject)' if keyless_source else 'leaving uncapped (key auth)'}" + "MCP org ceiling unresolved for org_id=%r; %s", + user_api_key_auth.org_id, + "denying (keyless admitted subject)" if keyless_source else "leaving uncapped (key auth)", ) return [] if keyless_source else allowed_mcp_servers if len(allowed_mcp_servers_for_org) == 0: @@ -1556,7 +1563,7 @@ class MCPRequestHandler: else: # No lower-level restrictions → org list becomes the ceiling. capped = allowed_mcp_servers_for_org - verbose_logger.debug(f"Applied org ceiling filter. Final allowed servers: {capped}") + verbose_logger.debug("Applied org ceiling filter. Final allowed servers: %s", capped) return capped @staticmethod @@ -1649,7 +1656,7 @@ class MCPRequestHandler: # Fault isolation is per SOURCE: an unresolvable team contributes nothing (fail closed for # it alone, access only narrows) while every other source stands. Raising would collapse the # whole union to deny-all over one momentarily-unreadable row. - verbose_logger.warning(f"MCP admitted-subject source team {team_id!r} unresolvable, skipping: {e}") + verbose_logger.warning("MCP admitted-subject source team %r unresolvable, skipping: %s", team_id, e) return None if team_obj is None: return None @@ -1682,10 +1689,10 @@ class MCPRequestHandler: proxy_logging_obj=proxy_logging_obj, ) except BudgetExceededError as e: - verbose_logger.info(f"MCP admitted-subject source team {team_id!r} over budget, not a grantor: {e}") + verbose_logger.info("MCP admitted-subject source team %r over budget, not a grantor: %s", team_id, e) return None except Exception as e: # noqa: BLE001 # per-source isolation: a budget-check fault narrows, never raises - verbose_logger.warning(f"MCP budget check failed for source team {team_id!r}, skipping source: {e}") + verbose_logger.warning("MCP budget check failed for source team %r, skipping source: %s", team_id, e) return None return team_obj @@ -1738,7 +1745,7 @@ class MCPRequestHandler: billed.org_id = source.org_id return billed except Exception as e: # noqa: BLE001 # attribution must never fail an authorized call - verbose_logger.warning(f"MCP billing attribution failed for {tool_name!r}, billing the user: {e}") + verbose_logger.warning("MCP billing attribution failed for %r, billing the user: %s", tool_name, e) return auth @staticmethod @@ -1828,7 +1835,7 @@ class MCPRequestHandler: ) verbose_logger.debug( - f"MCP team permission lookup: team_id={user_api_key_auth.team_id if user_api_key_auth else None}" + "MCP team permission lookup: team_id=%s", user_api_key_auth.team_id if user_api_key_auth else None ) if not user_api_key_auth or not user_api_key_auth.team_id or not prisma_client: return None @@ -1946,9 +1953,9 @@ class MCPRequestHandler: # than the None (allow-all) key auth gets for an indeterminate fault. unreadable_entitlement = isinstance(e, UnloadableEntitlementError) if unreadable_entitlement: - verbose_logger.warning(f"Denying MCP tools, entitlement unreadable: {e}") + verbose_logger.warning("Denying MCP tools, entitlement unreadable: %s", e) else: - verbose_logger.warning(f"Failed to get allowed tools for server: {e}") + verbose_logger.warning("Failed to get allowed tools for server: %s", e) # Fail CLOSED for a keyless admitted subject: ANY error must deny the server's tools ([]), # not collapse to allow-all (None); key/JWT auth keeps its prior allow-all-on-error. Both # keyless_source AND the marker are needed: each source resolves through an UNMARKED auth, so @@ -1998,8 +2005,9 @@ class MCPRequestHandler: if keyless_source or isinstance(e, UnloadableEntitlementError): raise verbose_logger.warning( - f"MCP org tool ceiling unresolvable for org_id={user_api_key_auth.org_id!r}; " - f"skipping org intersect, key/team/agent restrictions stand: {e}" + "MCP org tool ceiling unresolvable for org_id=%r; skipping org intersect, key/team/agent restrictions stand: %s", + user_api_key_auth.org_id, + e, ) return allowed_tools org_tools = ( @@ -2102,7 +2110,7 @@ class MCPRequestHandler: # Permission entries may be server_ids OR names/aliases — expand to ids. return global_mcp_server_manager.expand_permission_list(raw_server_ids) except Exception as e: - verbose_logger.warning(f"Failed to get key access group MCP server grants: {e}") + verbose_logger.warning("Failed to get key access group MCP server grants: %s", e) return [] @staticmethod @@ -2180,7 +2188,7 @@ class MCPRequestHandler: all_servers = direct_mcp_servers + access_group_servers + tool_perm_servers + toolset_servers return list(set(all_servers)) except Exception as e: - verbose_logger.warning(f"Failed to get allowed MCP servers for key: {e}") + verbose_logger.warning("Failed to get allowed MCP servers for key: %s", e) return [] @staticmethod @@ -2238,7 +2246,7 @@ class MCPRequestHandler: proxy_logging_obj=proxy_logging_obj, ) except Exception as e: # noqa: BLE001 # a team-resolution blip narrows access, never raises - verbose_logger.warning(f"Failed to resolve user teams for MCP grant: {e}") + verbose_logger.warning("Failed to resolve user teams for MCP grant: %s", e) return [] if user_object is None or not user_object.teams: return [] @@ -2323,7 +2331,7 @@ class MCPRequestHandler: servers = await MCPRequestHandler._team_granted_servers(team_obj, team_access_group_servers) return list(servers) except Exception as e: - verbose_logger.warning(f"Failed to get allowed MCP servers for team: {e}") + verbose_logger.warning("Failed to get allowed MCP servers for team: %s", e) return [] @staticmethod @@ -2403,7 +2411,7 @@ class MCPRequestHandler: # CONFIRMED absent: places no ceiling. Every OTHER exception propagates as an unresolvable # ceiling (denies for a keyless source, fail-open for a key); catching bare Exception here # would treat a DB outage as "no org" and silently drop a real ceiling for its duration. - verbose_logger.debug(f"MCP org ceiling: org {user_api_key_auth.org_id!r} does not exist: {e}") + verbose_logger.debug("MCP org ceiling: org %r does not exist: %s", user_api_key_auth.org_id, e) return None if org_obj is None or not org_obj.object_permission_id: @@ -2462,7 +2470,7 @@ class MCPRequestHandler: # A NAMED-but-unreadable ceiling is a stronger fact than "unresolved" and denies everywhere. if isinstance(e, UnloadableEntitlementError): raise - verbose_logger.warning(f"Failed to get allowed MCP servers for org: {e}") + verbose_logger.warning("Failed to get allowed MCP servers for org: %s", e) return None @staticmethod @@ -2490,7 +2498,7 @@ class MCPRequestHandler: route="/mcp", ) except Exception as e: # noqa: BLE001 # entitlement unknown, not known-absent: no ceiling, as before this level - verbose_logger.warning(f"Failed to resolve end_user for MCP permissions: {e}") + verbose_logger.warning("Failed to resolve end_user for MCP permissions: %s", e) return None if end_user_obj is None: @@ -2554,7 +2562,7 @@ class MCPRequestHandler: all_servers = direct_mcp_servers + access_group_servers + tool_perm_servers return list(set(all_servers)) except Exception as e: - verbose_logger.warning(f"Failed to get allowed MCP servers for end_user: {e}") + verbose_logger.warning("Failed to get allowed MCP servers for end_user: %s", e) return [] @staticmethod @@ -2637,7 +2645,7 @@ class MCPRequestHandler: ) return object_permission_id except Exception as e: # noqa: BLE001 # unknown whether entitled at all: no ceiling, as before - verbose_logger.warning(f"MCP user entitlement: link for {user_id!r} unresolved, no ceiling: {e}") + verbose_logger.warning("MCP user entitlement: link for %r unresolved, no ceiling: %s", user_id, e) return None @staticmethod @@ -2669,7 +2677,7 @@ class MCPRequestHandler: ) return list(set(direct_mcp_servers + access_group_servers + tool_perm_servers)) except Exception as e: # noqa: BLE001 # any resolution fault is an unresolved ceiling, never "no ceiling" - verbose_logger.warning(f"Failed to get allowed MCP servers for user: {e}") + verbose_logger.warning("Failed to get allowed MCP servers for user: %s", e) return None @staticmethod @@ -2700,7 +2708,7 @@ class MCPRequestHandler: if not entitled: return tuple(allowed_mcp_servers), False capped = tuple(server for server in allowed_mcp_servers if server in set(entitled)) - verbose_logger.debug(f"Applied user ceiling filter. Final allowed servers: {capped}") + verbose_logger.debug("Applied user ceiling filter. Final allowed servers: %s", capped) return capped, True @staticmethod @@ -2739,7 +2747,7 @@ class MCPRequestHandler: try: object_permissions = await MCPRequestHandler._get_user_object_permission(user_api_key_auth) except Exception as e: # noqa: BLE001 # an unresolved human entitlement must deny, not widen - verbose_logger.warning(f"MCP user tool ceiling unresolvable, denying tools on {server_id!r}: {e}") + verbose_logger.warning("MCP user tool ceiling unresolvable, denying tools on %r: %s", server_id, e) return [] if object_permissions is None or not object_permissions.mcp_tool_permissions: @@ -2785,7 +2793,7 @@ class MCPRequestHandler: ) return object_permission_id except Exception as e: # noqa: BLE001 # entitlement unknown, not known-absent: no ceiling, as before this level - verbose_logger.warning(f"Failed to resolve object_permission_id for agent {agent_id!r}: {e}") + verbose_logger.warning("Failed to resolve object_permission_id for agent %r: %s", agent_id, e) return None @staticmethod @@ -2869,7 +2877,7 @@ class MCPRequestHandler: all_servers = expanded_direct_servers + access_group_servers return list(set(all_servers)) except Exception as e: - verbose_logger.warning(f"Failed to get allowed MCP servers for agent: {e}") + verbose_logger.warning("Failed to get allowed MCP servers for agent: %s", e) return [] @staticmethod @@ -2911,7 +2919,7 @@ class MCPRequestHandler: tools = global_mcp_server_manager.expand_tool_permissions(mcp_tool_permissions).get(server_id) return list(tools) if tools else None except Exception as e: - verbose_logger.warning(f"Failed to get agent tool permissions for server: {e}") + verbose_logger.warning("Failed to get agent tool permissions for server: %s", e) return None @staticmethod @@ -2940,7 +2948,7 @@ class MCPRequestHandler: for server in mcp_servers: server_ids.add(server.server_id) except Exception as e: - verbose_logger.debug(f"Error getting MCP servers from access groups: {e}") + verbose_logger.debug("Error getting MCP servers from access groups: %s", e) return server_ids @staticmethod @@ -2969,7 +2977,7 @@ class MCPRequestHandler: return list(server_ids) except Exception as e: - verbose_logger.warning(f"Failed to get MCP servers from access groups: {e}") + verbose_logger.warning("Failed to get MCP servers from access groups: %s", e) return [] @staticmethod @@ -3029,7 +3037,7 @@ class MCPRequestHandler: return key_object_permission.mcp_access_groups or [] except Exception as e: - verbose_logger.warning(f"Failed to get MCP access groups for key: {e}") + verbose_logger.warning("Failed to get MCP access groups for key: %s", e) return [] @staticmethod @@ -3077,7 +3085,7 @@ class MCPRequestHandler: return object_permissions.mcp_access_groups or [] except Exception as e: - verbose_logger.warning(f"Failed to get MCP access groups for team: {e}") + verbose_logger.warning("Failed to get MCP access groups for team: %s", e) return [] @staticmethod diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 672396afd05..c5469785778 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -570,7 +570,7 @@ async def get_all_mcp_servers( decrypt_global_env_var_values(table.env_vars) return tables except Exception as e: - verbose_proxy_logger.debug(f"litellm.proxy._experimental.mcp_server.db.py::get_all_mcp_servers - {e}") + verbose_proxy_logger.debug("litellm.proxy._experimental.mcp_server.db.py::get_all_mcp_servers - %s", e) return [] diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index e16fb0d0e00..40844ec1937 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -2159,8 +2159,9 @@ async def _build_oauth_protected_resource_response( upstream_metadata = await fetch_upstream_oauth_protected_resource(mcp_server) except Exception as exc: verbose_logger.warning( - "Failed to fetch upstream oauth-protected-resource metadata " - f"for pass-through MCP server {mcp_server.name!r}: {exc}" + "Failed to fetch upstream oauth-protected-resource metadata for pass-through MCP server %r: %s", + mcp_server.name, + exc, ) raise HTTPException( status_code=502, @@ -2179,7 +2180,7 @@ async def _build_oauth_protected_resource_response( # so we must not fall through to the default gateway metadata — # that would point clients at the wrong IdP. verbose_logger.warning( - f"Upstream oauth-protected-resource metadata unavailable for pass-through MCP server {mcp_server.name!r}" + "Upstream oauth-protected-resource metadata unavailable for pass-through MCP server %r", mcp_server.name ) raise HTTPException( status_code=502, diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 0a6a0374d13..66a55f30b74 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -1566,7 +1566,7 @@ class MCPServerManager: if target_server_name == server_name and alias_name not in used_aliases: alias = alias_name used_aliases.add(alias_name) - verbose_logger.debug(f"Mapped alias '{alias_name}' to server '{server_name}'") + verbose_logger.debug("Mapped alias '%s' to server '%s'", alias_name, server_name) break # Create a temporary server object to use with get_server_prefix utility @@ -1785,7 +1785,7 @@ class MCPServerManager: # Check if this is an OpenAPI-based server spec_path = server_config.get("spec_path", None) if spec_path: - verbose_logger.info(f"Loading OpenAPI spec from {spec_path} for server {server_name}") + verbose_logger.info("Loading OpenAPI spec from %s for server %s", spec_path, server_name) await self._register_openapi_tools( spec_path=spec_path, server=new_server, @@ -1793,7 +1793,7 @@ class MCPServerManager: ) verbose_logger.debug( - f"Loaded MCP Servers: {json.dumps(_redacted_registry_dump(self.config_mcp_servers), indent=4)}" + "Loaded MCP Servers: %s", json.dumps(_redacted_registry_dump(self.config_mcp_servers), indent=4) ) await self._hydrate_config_servers_dcr_clients() @@ -1856,7 +1856,7 @@ class MCPServerManager: # Use base_url from config if provided, otherwise extract from spec if not base_url: base_url = get_openapi_base_url(spec, spec_path) - verbose_logger.info(f"Registering OpenAPI tools for server {server.name} with base URL: {base_url}") + verbose_logger.info("Registering OpenAPI tools for server %s with base URL: %s", server.name, base_url) # Get server prefix for tool naming server_prefix = get_server_prefix(server) @@ -1892,7 +1892,7 @@ class MCPServerManager: ) verbose_logger.debug( - f"Using headers for OpenAPI tools (excluding sensitive values): {list(headers.keys())}" + "Using headers for OpenAPI tools (excluding sensitive values): %s", list(headers.keys()) ) # Extract and register tools from OpenAPI paths @@ -1900,7 +1900,7 @@ class MCPServerManager: components = spec.get("components", {}) registered_count = 0 - verbose_logger.debug(f"Processing {len(paths)} paths from OpenAPI spec") + verbose_logger.debug("Processing %s paths from OpenAPI spec", len(paths)) for path, path_item in paths.items(): for method in ["get", "post", "put", "delete", "patch"]: @@ -1946,12 +1946,12 @@ class MCPServerManager: self.tool_name_to_mcp_server_name_mapping[prefixed_tool_name] = server_prefix registered_count += 1 - verbose_logger.debug(f"Registered OpenAPI tool: {prefixed_tool_name} for server {server.name}") + verbose_logger.debug("Registered OpenAPI tool: %s for server %s", prefixed_tool_name, server.name) - verbose_logger.info(f"Successfully registered {registered_count} OpenAPI tools for server {server.name}") + verbose_logger.info("Successfully registered %s OpenAPI tools for server %s", registered_count, server.name) except Exception as e: - verbose_logger.error(f"Failed to register OpenAPI tools for server {server.name}: {e}") + verbose_logger.error("Failed to register OpenAPI tools for server %s: %s", server.name, e) raise e def _cleanup_server_tool_routing_artifacts(self, server: MCPServer) -> None: @@ -2000,7 +2000,7 @@ class MCPServerManager: verbose_logger.debug("Removed MCP Server: %s", mcp_server.server_id or mcp_server.server_name) self._cleanup_server_tool_routing_artifacts(evicted) else: - verbose_logger.warning(f"Server ID {mcp_server.server_id} not found in registry") + verbose_logger.warning("Server ID %s not found in registry", mcp_server.server_id) def _resolve_env_vars_list( self, @@ -2295,7 +2295,7 @@ class MCPServerManager: async def _maybe_register_openapi_tools(self, server: MCPServer, *, initialize_mapping: bool = True): """Register OpenAPI tools if the server has a spec_path configured.""" if server.spec_path: - verbose_logger.info(f"Loading OpenAPI spec from {server.spec_path} for server {server.name}") + verbose_logger.info("Loading OpenAPI spec from %s for server %s", server.spec_path, server.name) await self._register_openapi_tools( spec_path=server.spec_path, server=server, @@ -2323,10 +2323,10 @@ class MCPServerManager: self._assign_unique_short_prefix(new_server) self.registry[mcp_server.server_id] = new_server await self._maybe_register_openapi_tools(new_server) - verbose_logger.debug(f"Added MCP Server: {new_server.name}") + verbose_logger.debug("Added MCP Server: %s", new_server.name) except Exception as e: - verbose_logger.debug(f"Failed to add MCP server: {e}") + verbose_logger.debug("Failed to add MCP server: %s", e) raise e async def update_server(self, mcp_server: LiteLLM_MCPServerTable): @@ -2357,10 +2357,10 @@ class MCPServerManager: self._assign_unique_short_prefix(new_server) self.registry[mcp_server.server_id] = new_server await self._maybe_register_openapi_tools(new_server) - verbose_logger.debug(f"Updated MCP Server: {new_server.name}") + verbose_logger.debug("Updated MCP Server: %s", new_server.name) except Exception as e: - verbose_logger.debug(f"Failed to udpate MCP server: {e}") + verbose_logger.debug("Failed to udpate MCP server: %s", e) raise e def get_all_mcp_server_ids(self) -> set[str]: @@ -2386,7 +2386,7 @@ class MCPServerManager: await user_api_key_cache.async_delete_cache(key=self.get_byom_submitted_servers_cache_key(user_id)) except Exception as e: # noqa: BLE001 - verbose_logger.warning(f"Failed to invalidate BYOM submitted MCP server cache: {e}") + verbose_logger.warning("Failed to invalidate BYOM submitted MCP server cache: %s", e) async def _get_active_submitted_mcp_server_ids_for_user( self, user_api_key_auth: UserAPIKeyAuth | None @@ -2401,7 +2401,7 @@ class MCPServerManager: ) from litellm.proxy.proxy_server import prisma_client, user_api_key_cache except Exception as e: # noqa: BLE001 - verbose_logger.warning(f"Failed to load BYOM submitted MCP server cache dependencies: {e}") + verbose_logger.warning("Failed to load BYOM submitted MCP server cache dependencies: %s", e) return [] byom_cache_key = self.get_byom_submitted_servers_cache_key(submitter_user_id) @@ -2411,7 +2411,7 @@ class MCPServerManager: if cached_submitted_server_ids is not None: submitted_server_ids = cast(list[str], cached_submitted_server_ids) except Exception as e: # noqa: BLE001 - verbose_logger.warning(f"Failed to read BYOM submitted MCP server cache: {e}") + verbose_logger.warning("Failed to read BYOM submitted MCP server cache: %s", e) if submitted_server_ids is None: if prisma_client is None: @@ -2422,7 +2422,7 @@ class MCPServerManager: prisma_client, submitter_user_id ) except Exception as e: # noqa: BLE001 - verbose_logger.warning(f"Failed to read BYOM submitted MCP servers from database: {e}") + verbose_logger.warning("Failed to read BYOM submitted MCP servers from database: %s", e) submitted_server_ids = [] try: await user_api_key_cache.async_set_cache( @@ -2431,7 +2431,7 @@ class MCPServerManager: ttl=60, ) except Exception as e: # noqa: BLE001 - verbose_logger.warning(f"Failed to write BYOM submitted MCP server cache: {e}") + verbose_logger.warning("Failed to write BYOM submitted MCP server cache: %s", e) return [server_id for server_id in submitted_server_ids if self.get_mcp_server_by_id(server_id) is not None] @@ -2522,7 +2522,7 @@ class MCPServerManager: key_object_permission.mcp_servers is not None ) if has_explicit_object_permission: - verbose_logger.debug(f"Object permission mcp_servers explicitly set: {key_object_permission.mcp_servers}") + verbose_logger.debug("Object permission mcp_servers explicitly set: %s", key_object_permission.mcp_servers) # BYOM creator visibility never widens a key that was explicitly scoped: # only keys without their own mcp_servers list get submitted servers unioned in. @@ -2551,7 +2551,7 @@ class MCPServerManager: # Get allowed servers from object permissions (respects object_permission even for admins) allowed_mcp_servers = await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth) - verbose_logger.debug(f"Allowed MCP Servers for user api key auth: {allowed_mcp_servers}") + verbose_logger.debug("Allowed MCP Servers for user api key auth: %s", allowed_mcp_servers) combined_servers = set(allowed_mcp_servers) combined_servers.update( await self.operator_open_server_ids( @@ -2647,7 +2647,7 @@ class MCPServerManager: ) return tool_permissions except Exception as e: - verbose_logger.warning(f"Failed to resolve toolset permissions: {e}") + verbose_logger.warning("Failed to resolve toolset permissions: %s", e) return {} def invalidate_toolset_cache(self, toolset_id: str | None = None) -> None: @@ -2682,7 +2682,7 @@ class MCPServerManager: for k in keys_to_remove: cache_dict.pop(k, None) except Exception as e: - verbose_logger.warning(f"invalidate_toolset_cache: failed to evict in-memory entries: {e}") + verbose_logger.warning("invalidate_toolset_cache: failed to evict in-memory entries: %s", e) async def get_toolset_by_name_cached( self, @@ -2760,11 +2760,11 @@ class MCPServerManager: try: server = self.get_mcp_server_by_id(server_id) if server is None: - verbose_logger.warning(f"MCP Server {server_id} not found") + verbose_logger.warning("MCP Server %s not found", server_id) return [] return await self._get_tools_from_server(server) except Exception as e: - verbose_logger.warning(f"Failed to get tools from server {server_id}: {e}") + verbose_logger.warning("Failed to get tools from server %s: %s", server_id, e) return [] async def list_tools( @@ -2793,7 +2793,7 @@ class MCPServerManager: """Fetch tools from a single server with error handling.""" server = self.get_mcp_server_by_id(server_id) if server is None: - verbose_logger.warning(f"MCP Server {server_id} not found") + verbose_logger.warning("MCP Server %s not found", server_id) return [] # Get server-specific auth header if available @@ -2822,7 +2822,7 @@ class MCPServerManager: return tools except Exception as e: verbose_logger.warning( - f"Failed to list tools from server {server.name}: {e}. Continuing with other servers." + "Failed to list tools from server %s: %s. Continuing with other servers.", server.name, e ) return [] @@ -2833,7 +2833,7 @@ class MCPServerManager: # Flatten results into single list list_tools_result: list[MCPTool] = [tool for tools in results for tool in tools] - verbose_logger.info(f"Successfully fetched {len(list_tools_result)} tools total from all servers") + verbose_logger.info("Successfully fetched %s tools total from all servers", len(list_tools_result)) return list_tools_result ######################################################### @@ -3345,8 +3345,8 @@ class MCPServerManager: global_mcp_tool_registry, ) - verbose_logger.debug(f"Connecting to url: {server.url}") - verbose_logger.info(f"_get_tools_from_server for {server.name}...") + verbose_logger.debug("Connecting to url: %s", server.url) + verbose_logger.info("_get_tools_from_server for %s...", server.name) client = None @@ -3476,12 +3476,12 @@ class MCPServerManager: www_authenticate=None if server.is_dcr_bridge else challenge_header, server_name=server.name, ) from e - verbose_logger.warning(f"Failed to get tools from server {server.name}: {e}") + verbose_logger.warning("Failed to get tools from server %s: %s", server.name, e) raise MCPServerListError(ServerListFault(tag="internal", status_code=e.status_code), server.name) from e except MCPServerListError: raise except Exception as e: - verbose_logger.warning(f"Failed to get tools from server {server.name}: {e}") + verbose_logger.warning("Failed to get tools from server %s: %s", server.name, e) raise_classified_list_failure(e, server.name, suppress_challenge=server.is_dcr_bridge) async def get_prompts_from_server( @@ -3503,8 +3503,8 @@ class MCPServerManager: List[Prompt]: List of prompts available on the server with prefixed names """ - verbose_logger.debug(f"Connecting to url: {server.url}") - verbose_logger.info(f"get_prompts_from_server for {server.name}...") + verbose_logger.debug("Connecting to url: %s", server.url) + verbose_logger.info("get_prompts_from_server for %s...", server.name) client = None @@ -3532,7 +3532,7 @@ class MCPServerManager: return prefixed_or_original_prompts except Exception as e: - verbose_logger.warning(f"Failed to get prompts from server {server.name}: {e}") + verbose_logger.warning("Failed to get prompts from server %s: %s", server.name, e) return [] async def get_resources_from_server( @@ -3545,8 +3545,8 @@ class MCPServerManager: ) -> list[Resource]: """Fetch available resources from a single MCP server.""" - verbose_logger.debug(f"Connecting to url: {server.url}") - verbose_logger.info(f"get_resources_from_server for {server.name}...") + verbose_logger.debug("Connecting to url: %s", server.url) + verbose_logger.info("get_resources_from_server for %s...", server.name) client = None @@ -3574,7 +3574,7 @@ class MCPServerManager: return prefixed_resources except Exception as e: - verbose_logger.warning(f"Failed to get resources from server {server.name}: {e}") + verbose_logger.warning("Failed to get resources from server %s: %s", server.name, e) return [] async def get_resource_templates_from_server( @@ -3587,8 +3587,8 @@ class MCPServerManager: ) -> list[ResourceTemplate]: """Fetch available resource templates from a single MCP server.""" - verbose_logger.debug(f"Connecting to url: {server.url}") - verbose_logger.info(f"get_resource_templates_from_server for {server.name}...") + verbose_logger.debug("Connecting to url: %s", server.url) + verbose_logger.info("get_resource_templates_from_server for %s...", server.name) client = None @@ -3618,7 +3618,7 @@ class MCPServerManager: return prefixed_templates except Exception as e: - verbose_logger.warning(f"Failed to get resource templates from server {server.name}: {e}") + verbose_logger.warning("Failed to get resource templates from server %s: %s", server.name, e) return [] async def read_resource_from_server( @@ -3631,8 +3631,8 @@ class MCPServerManager: ) -> ReadResourceResult: """Read resource contents from a specific MCP server.""" - verbose_logger.debug(f"Connecting to url: {server.url}") - verbose_logger.info(f"read_resource_from_server for {server.name}...") + verbose_logger.debug("Connecting to url: %s", server.url) + verbose_logger.info("read_resource_from_server for %s...", server.name) if server.static_headers: if extra_headers is None: @@ -3663,8 +3663,8 @@ class MCPServerManager: ) -> GetPromptResult: """Fetch a specific prompt definition from a single MCP server.""" - verbose_logger.debug(f"Connecting to url: {server.url}") - verbose_logger.info(f"get_prompt_from_server for {server.name}...") + verbose_logger.debug("Connecting to url: %s", server.url) + verbose_logger.info("get_prompt_from_server for %s...", server.name) if server.static_headers: if extra_headers is None: @@ -4206,19 +4206,19 @@ class MCPServerManager: try: with anyio.fail_after(MCP_TOOL_LISTING_TIMEOUT): tools = await client.list_tools(raise_on_error=True) - verbose_logger.debug(f"Tools from {server_name}: {tools}") + verbose_logger.debug("Tools from %s: %s", server_name, tools) return tools except TimeoutError as e: - verbose_logger.warning(f"Timeout while listing tools from {server_name}") + verbose_logger.warning("Timeout while listing tools from %s", server_name) raise MCPServerListError(ServerListFault(tag="timeout"), server_name) from e except asyncio.CancelledError as e: - verbose_logger.warning(f"Task cancelled while listing tools from {server_name}") + verbose_logger.warning("Task cancelled while listing tools from %s", server_name) raise MCPServerListError(ServerListFault(tag="internal"), server_name) from e except ConnectionError as e: - verbose_logger.warning(f"Connection error while listing tools from {server_name}: {e}") + verbose_logger.warning("Connection error while listing tools from %s: %s", server_name, e) raise MCPServerListError(ServerListFault(tag="unreachable"), server_name) from e except Exception as e: - verbose_logger.warning(f"Error listing tools from {server_name}: {e}") + verbose_logger.warning("Error listing tools from %s: %s", server_name, e) raise_classified_list_failure(e, server_name) _SHORT_PREFIX_MAX_REHASH_ATTEMPTS = 1024 @@ -4315,7 +4315,7 @@ class MCPServerManager: for spelling in iter_known_tool_name_spellings(original_name, server): self.tool_name_to_mcp_server_name_mapping[spelling] = prefix - verbose_logger.info(f"Successfully fetched {len(prefixed_tools)} tools from server {server.name}") + verbose_logger.info("Successfully fetched %s tools from server %s", len(prefixed_tools), server.name) return prefixed_tools def _create_prefixed_prompts( @@ -4342,7 +4342,7 @@ class MCPServerManager: prompt.name = name_to_use prefixed_prompts.append(prompt) - verbose_logger.info(f"Successfully fetched {len(prefixed_prompts)} prompts from server {server.name}") + verbose_logger.info("Successfully fetched %s prompts from server %s", len(prefixed_prompts), server.name) return prefixed_prompts def _create_prefixed_resources( @@ -4358,7 +4358,7 @@ class MCPServerManager: resource.name = name_to_use prefixed_resources.append(resource) - verbose_logger.info(f"Successfully fetched {len(prefixed_resources)} resources from server {server.name}") + verbose_logger.info("Successfully fetched %s resources from server %s", len(prefixed_resources), server.name) return prefixed_resources def _create_prefixed_resource_templates( @@ -4380,7 +4380,7 @@ class MCPServerManager: prefixed_templates.append(resource_template) verbose_logger.info( - f"Successfully fetched {len(prefixed_templates)} resource templates from server {server.name}" + "Successfully fetched %s resource templates from server %s", len(prefixed_templates), server.name ) return prefixed_templates @@ -4639,7 +4639,7 @@ class MCPServerManager: HTTPException, ) as e: # Re-raise guardrail exceptions to properly fail the MCP call - verbose_logger.error(f"Guardrail blocked MCP tool call pre call: {e}") + verbose_logger.error("Guardrail blocked MCP tool call pre call: %s", e) raise e return hook_result @@ -4995,7 +4995,7 @@ class MCPServerManager: GuardrailRaisedException, HTTPException, ) as e: - verbose_logger.error(f"Guardrail blocked MCP tool call during result check: {e}") + verbose_logger.error("Guardrail blocked MCP tool call during result check: %s", e) raise e # If proxy_logging_obj is None, the tool call result is at index 0 @@ -5194,7 +5194,7 @@ class MCPServerManager: GuardrailRaisedException, HTTPException, ) as e: - verbose_logger.error(f"Guardrail blocked MCP tool call during result check: {e}") + verbose_logger.error("Guardrail blocked MCP tool call during result check: %s", e) raise e async def call_tool( @@ -5345,7 +5345,7 @@ class MCPServerManager: asyncio.create_task(self._initialize_tool_name_to_mcp_server_name_mapping()) except RuntimeError as e: # no running event loop verbose_logger.exception( - f"No running event loop - skipping tool name to MCP server name mapping initialization: {e}" + "No running event loop - skipping tool name to MCP server name mapping initialization: %s", e ) async def _initialize_tool_name_to_mcp_server_name_mapping(self): @@ -5364,12 +5364,12 @@ class MCPServerManager: # at startup we have none, so an upstream 401 is normal. # Swallow it so we keep mapping the remaining servers. verbose_logger.debug( - f"Skipping tool name mapping for server {server.name} due to upstream auth error: {e}" + "Skipping tool name mapping for server %s due to upstream auth error: %s", server.name, e ) continue except Exception as e: verbose_logger.warning( - f"Failed to get tools from server {server.name} during tool name mapping initialization: {e}" + "Failed to get tools from server %s during tool name mapping initialization: %s", server.name, e ) continue for tool in tools: @@ -5449,7 +5449,7 @@ class MCPServerManager: } ) db_mcp_servers = [LiteLLM_MCPServerTable.model_validate(r.model_dump()) for r in raw_rows] - verbose_logger.info(f"Found {len(db_mcp_servers)} MCP servers in database") + verbose_logger.info("Found %s MCP servers in database", len(db_mcp_servers)) previous_registry = self.registry new_registry: dict[str, MCPServer] = {} @@ -5481,7 +5481,7 @@ class MCPServerManager: alias=getattr(server, "alias", None), server_name=getattr(server, "server_name", None), ) - verbose_logger.debug(f"Building server from DB: {server.server_id} ({server.server_name})") + verbose_logger.debug("Building server from DB: %s (%s)", server.server_id, server.server_name) # raw_rows come straight from the DB, so their global env var # values (like credentials) are still encrypted here, unlike the # already-decrypted records add_server/update_server are handed. @@ -5776,7 +5776,7 @@ class MCPServerManager: server = self.get_mcp_server_by_id(server_id) if not server: - verbose_logger.warning(f"MCP Server {server_id} not found") + verbose_logger.warning("MCP Server %s not found", server_id) return LiteLLM_MCPServerTable( server_id=server_id, server_name=None, @@ -5929,7 +5929,7 @@ class MCPServerManager: for server_id in allowed_server_ids: server = self.get_mcp_server_by_id(server_id) if not server: - verbose_logger.warning(f"MCP Server {server_id} not found in registry") + verbose_logger.warning("MCP Server %s not found in registry", server_id) continue mcp_server_table = self._build_mcp_server_table(server) diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py index db2851c60aa..5b30b58c0e3 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py @@ -138,8 +138,9 @@ def get_base_url(spec: dict[str, Any], spec_path: str | None = None) -> str: base_domain = f"{parsed.scheme}://{parsed.netloc}" full_base_url = base_domain + server_url verbose_logger.info( - f"OpenAPI spec has relative server URL '{server_url}'. " - f"Deriving base from spec_path: {full_base_url}" + "OpenAPI spec has relative server URL '%s'. Deriving base from spec_path: %s", + server_url, + full_base_url, ) return full_base_url @@ -160,12 +161,12 @@ def get_base_url(spec: dict[str, Any], spec_path: str | None = None) -> str: ]: if spec_path.endswith(suffix): base_url = spec_path[: -len(suffix)] - verbose_logger.info(f"No server info in OpenAPI spec. Using derived base URL: {base_url}") + verbose_logger.info("No server info in OpenAPI spec. Using derived base URL: %s", base_url) return base_url if spec_path.split("/")[-1].endswith((".json", ".yaml", ".yml")): base_url = "/".join(spec_path.split("/")[:-1]) - verbose_logger.info(f"No server info in OpenAPI spec. Using derived base URL: {base_url}") + verbose_logger.info("No server info in OpenAPI spec. Using derived base URL: %s", base_url) return base_url return "" @@ -497,4 +498,4 @@ def register_tools_from_openapi(spec: dict[str, Any], base_url: str): input_schema=input_schema, handler=tool_func, ) - verbose_logger.debug(f"Registered tool: {tool_name}") + verbose_logger.debug("Registered tool: %s", tool_name) diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index d0458db51c6..57dfe6823b9 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -50,7 +50,7 @@ MCP_AVAILABLE: bool = True try: importlib.import_module("mcp") except ImportError as e: - verbose_logger.debug(f"MCP module not found: {e}") + verbose_logger.debug("MCP module not found: %s", e) MCP_AVAILABLE = False @@ -328,8 +328,10 @@ if MCP_AVAILABLE: return {"Authorization": f"Bearer {cred['access_token']}"} except Exception as e: verbose_logger.warning( - f"_get_user_oauth_extra_headers: failed to retrieve credential for " - f"user={user_id} server={server_id}: {e}" + "_get_user_oauth_extra_headers: failed to retrieve credential for user=%s server=%s: %s", + user_id, + server_id, + e, ) return None @@ -356,7 +358,7 @@ if MCP_AVAILABLE: creds = await list_user_oauth_credentials(prisma_client, user_id) return {c["server_id"]: c for c in creds if "server_id" in c} except Exception as e: - verbose_logger.warning(f"_prefetch_user_oauth_creds: failed to prefetch for user={user_id}: {e}") + verbose_logger.warning("_prefetch_user_oauth_creds: failed to prefetch for user=%s: %s", user_id, e) return {} def _create_tool_response_objects(tools, server: MCPServer): @@ -641,7 +643,7 @@ if MCP_AVAILABLE: raise except MCPServerListError as e: fault = classify_list_exception(e) - verbose_logger.info(f"Listing tools from {server.name} failed with a {fault.tag} fault") + verbose_logger.info("Listing tools from %s failed with a %s fault", server.name, fault.tag) raise HTTPException( status_code=list_fault_http_status(fault), detail={ @@ -650,7 +652,7 @@ if MCP_AVAILABLE: }, ) from e except Exception as e: - verbose_logger.exception(f"Error getting tools from {server.name}: {e}") + verbose_logger.exception("Error getting tools from %s: %s", server.name, e) return { "tools": [], "error": "server_error", @@ -862,7 +864,7 @@ if MCP_AVAILABLE: ) list_tools_result.extend(tools_result) except Exception as e: - verbose_logger.exception(f"Error getting tools from {server.name}: {e}") + verbose_logger.exception("Error getting tools from %s: %s", server.name, e) errors.append( f"{get_server_prefix(server)}: {classify_list_exception(e).tag}" if isinstance(e, (MCPServerListError, MCPUpstreamAuthError)) @@ -1052,7 +1054,7 @@ if MCP_AVAILABLE: }, ) except BlockedPiiEntityError as e: - verbose_logger.error(f"BlockedPiiEntityError in MCP tool call: {e}") + verbose_logger.error("BlockedPiiEntityError in MCP tool call: %s", e) raise HTTPException( status_code=400, detail={ @@ -1063,7 +1065,7 @@ if MCP_AVAILABLE: }, ) except GuardrailRaisedException as e: - verbose_logger.error(f"GuardrailRaisedException in MCP tool call: {e}") + verbose_logger.error("GuardrailRaisedException in MCP tool call: %s", e) raise HTTPException( status_code=400, detail={ @@ -1076,16 +1078,16 @@ if MCP_AVAILABLE: # A client-forwarded pass-through upstream 401 from either the direct or the virtual call # branch. Relay it as a 401 + WWW-Authenticate so the MCP client can re-run upstream OAuth, # and log at info: an expected caller-must-reauth signal, not an operator-actionable error. - verbose_logger.info(f"MCP tool call relaying upstream HTTP {e.status_code}") + verbose_logger.info("MCP tool call relaying upstream HTTP %s", e.status_code) raise _relay_upstream_auth_http_exception(e, request) except HTTPException as e: # Locally generated denials (tool/server permission, IP filtering, BYOK) stay at error level # so restriction probing keeps full monitoring visibility; the relayed upstream 401 above is # the only status demoted to info. - verbose_logger.error(f"HTTPException in MCP tool call: {e}") + verbose_logger.error("HTTPException in MCP tool call: %s", e) raise e except Exception as e: - verbose_logger.exception(f"Unexpected error in MCP tool call: {e}") + verbose_logger.exception("Unexpected error in MCP tool call: %s", e) raise HTTPException( status_code=500, detail={ diff --git a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py index a7cc9fe3ed0..e2c60a488a4 100644 --- a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py +++ b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py @@ -98,7 +98,7 @@ class SemanticMCPToolFilter: tools = await global_mcp_server_manager.get_tools_for_server(server_id) all_tools.extend(tools) except Exception as e: - verbose_logger.warning(f"Failed to fetch tools from server {server_id}: {e}") + verbose_logger.warning("Failed to fetch tools from server %s: %s", server_id, e) continue if not all_tools: @@ -106,11 +106,11 @@ class SemanticMCPToolFilter: self.tool_router = None return - verbose_logger.info(f"Fetched {len(all_tools)} tools from {len(registry)} MCP servers") + verbose_logger.info("Fetched %s tools from %s MCP servers", len(all_tools), len(registry)) self._build_router(all_tools) except Exception as e: - verbose_logger.error(f"Failed to build router from MCP registry: {e}") + verbose_logger.error("Failed to build router from MCP registry: %s", e) self.tool_router = None raise @@ -172,10 +172,10 @@ class SemanticMCPToolFilter: auto_sync="local", ) - verbose_logger.info(f"Built semantic router with {len(routes)} tools") + verbose_logger.info("Built semantic router with %s tools", len(routes)) except Exception as e: - verbose_logger.error(f"Failed to build semantic router: {e}") + verbose_logger.error("Failed to build semantic router: %s", e) self.tool_router = None if _is_context_window_error(e): self.context_window_error = str(e) @@ -254,7 +254,7 @@ class SemanticMCPToolFilter: self._tool_map.update(missing) verbose_logger.info( - f"Semantic tool filter indexed {len(routes)} request-time tools missing from the startup index" + "Semantic tool filter indexed %s request-time tools missing from the startup index", len(routes) ) async def filter_tools( @@ -321,7 +321,8 @@ class SemanticMCPToolFilter: except Exception as e: if _is_context_window_error(e): verbose_logger.error( - f"Semantic tool filter embedding exceeded its context window: {e}", + "Semantic tool filter embedding exceeded its context window: %s", + e, exc_info=True, ) raise SemanticToolFilterContextWindowError( @@ -329,7 +330,7 @@ class SemanticMCPToolFilter: stage="the user query or the MCP tool descriptions being indexed", original_error=str(e), ) from e - verbose_logger.error(f"Semantic tool filter failed: {e}", exc_info=True) + verbose_logger.error("Semantic tool filter failed: %s", e, exc_info=True) return available_tools def _extract_tool_names_from_matches(self, matches) -> list[str]: diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index a894413019e..6e7af04e747 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -153,7 +153,7 @@ try: "active_mcp_session", default=None ) except ImportError as e: - verbose_logger.debug(f"MCP module not found: {e}") + verbose_logger.debug("MCP module not found: %s", e) MCP_AVAILABLE = False # When MCP is not available, we set these to None at module level # All code using these types is inside `if MCP_AVAILABLE:` blocks @@ -657,7 +657,7 @@ if MCP_AVAILABLE: try: await _purge_expired_stateful_session_auth_contexts() except Exception as e: - verbose_logger.exception(f"Error cleaning up expired MCP stateful sessions: {e}") + verbose_logger.exception("Error cleaning up expired MCP stateful sessions: %s", e) async def initialize_session_managers(): """Initialize the session managers. Can be called from main app lifespan.""" @@ -713,7 +713,7 @@ if MCP_AVAILABLE: if _sse_session_manager_cm: await _sse_session_manager_cm.__aexit__(None, None, None) except Exception as e: - verbose_logger.exception(f"Error during session manager shutdown: {e}") + verbose_logger.exception("Error during session manager shutdown: %s", e) _session_manager_cm = None _session_manager_stateful_cm = None @@ -765,10 +765,11 @@ if MCP_AVAILABLE: raw_headers, _client_ip, ) = await get_or_extract_auth_context() - verbose_logger.debug(f"MCP list_tools - User API Key Auth from context: {user_api_key_auth}") - verbose_logger.debug(f"MCP list_tools - MCP servers from context: {mcp_servers}") + verbose_logger.debug("MCP list_tools - User API Key Auth from context: %s", user_api_key_auth) + verbose_logger.debug("MCP list_tools - MCP servers from context: %s", mcp_servers) verbose_logger.debug( - f"MCP list_tools - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" + "MCP list_tools - MCP server auth headers: %s", + list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None, ) if getattr( getattr(user_api_key_auth, "object_permission", None), @@ -795,7 +796,7 @@ if MCP_AVAILABLE: log_list_tools_to_spendlogs=True, list_tools_log_source="mcp_protocol", ) - verbose_logger.info(f"MCP list_tools - Successfully returned {len(listing.tools)} tools") + verbose_logger.info("MCP list_tools - Successfully returned %s tools", len(listing.tools)) if not listing.outcomes: return listing.tools outcome_meta = { @@ -805,7 +806,7 @@ if MCP_AVAILABLE: } return ListToolsResult.model_validate({"tools": listing.tools, "_meta": outcome_meta}) except Exception as e: - verbose_logger.exception(f"Error in list_tools endpoint: {e}") + verbose_logger.exception("Error in list_tools endpoint: %s", e) # Return empty list instead of failing completely # This prevents the HTTP stream from failing and allows the client to get a response return [] @@ -823,7 +824,7 @@ if MCP_AVAILABLE: try: host_ctx = host_server.request_context except Exception as e: - verbose_logger.warning(f"Could not capture host progress context: {e}") + verbose_logger.warning("Could not capture host progress context: %s", e) return None if not (host_ctx and hasattr(host_ctx, "meta") and host_ctx.meta): @@ -841,11 +842,11 @@ if MCP_AVAILABLE: progress=progress, total=total, ) - verbose_logger.debug(f"Forwarded progress {progress}/{total} to Host") + verbose_logger.debug("Forwarded progress %s/%s to Host", progress, total) except Exception as e: - verbose_logger.error(f"Failed to forward progress to Host: {e}") + verbose_logger.error("Failed to forward progress to Host: %s", e) - verbose_logger.debug(f"Host progressToken captured: {str(host_token)[:8]}...") + verbose_logger.debug("Host progressToken captured: %s...", str(host_token)[:8]) return forward_progress async def _build_virtual_call_logging_obj( @@ -1000,10 +1001,12 @@ if MCP_AVAILABLE: _client_ip, ) = await get_or_extract_auth_context() verbose_logger.debug( - f"MCP mcp_server_tool_call - user_api_key_auth={user_api_key_auth}, user_role={getattr(user_api_key_auth, 'user_role', 'N/A')}" + "MCP mcp_server_tool_call - user_api_key_auth=%s, user_role=%s", + user_api_key_auth, + getattr(user_api_key_auth, "user_role", "N/A"), ) - verbose_logger.debug(f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}") + verbose_logger.debug("MCP mcp_server_tool_call - User API Key Auth from context: %s", user_api_key_auth) try: # Inside this try so virtual-tool errors convert to isError @@ -1080,7 +1083,7 @@ if MCP_AVAILABLE: isError=True, ) except BlockedPiiEntityError as e: - verbose_logger.error(f"BlockedPiiEntityError in MCP tool call: {e}") + verbose_logger.error("BlockedPiiEntityError in MCP tool call: %s", e) return CallToolResult( content=[ TextContent( @@ -1091,13 +1094,13 @@ if MCP_AVAILABLE: isError=True, ) except GuardrailRaisedException as e: - verbose_logger.error(f"GuardrailRaisedException in MCP tool call: {e}") + verbose_logger.error("GuardrailRaisedException in MCP tool call: %s", e) return CallToolResult( content=[TextContent(text=f"Error: Guardrail violation - {e}", type="text")], isError=True, ) except HTTPException as e: - verbose_logger.error(f"HTTPException in MCP tool call: {e}") + verbose_logger.error("HTTPException in MCP tool call: %s", e) return CallToolResult( content=[TextContent(text=f"Error: {e.detail}", type="text")], isError=True, @@ -1108,7 +1111,7 @@ if MCP_AVAILABLE: # call path and the connect-time preemptive check do. Return an explicit isError # naming the upstream status (at info level, not a traceback) so the client still # learns it must re-authenticate upstream and expected pass-through 401s don't spam. - verbose_logger.info(f"Upstream auth failure calling MCP tool: HTTP {e.status_code}") + verbose_logger.info("Upstream auth failure calling MCP tool: HTTP %s", e.status_code) return CallToolResult( content=[ TextContent( @@ -1119,7 +1122,7 @@ if MCP_AVAILABLE: isError=True, ) except Exception as e: - verbose_logger.exception(f"MCP mcp_server_tool_call - error: {e}") + verbose_logger.exception("MCP mcp_server_tool_call - error: %s", e) return CallToolResult( content=[TextContent(text=f"Error: {e}", type="text")], isError=True, @@ -1155,10 +1158,11 @@ if MCP_AVAILABLE: raw_headers, _client_ip, ) = await get_or_extract_auth_context() - verbose_logger.debug(f"MCP list_prompts - User API Key Auth from context: {user_api_key_auth}") - verbose_logger.debug(f"MCP list_prompts - MCP servers from context: {mcp_servers}") + verbose_logger.debug("MCP list_prompts - User API Key Auth from context: %s", user_api_key_auth) + verbose_logger.debug("MCP list_prompts - MCP servers from context: %s", mcp_servers) verbose_logger.debug( - f"MCP list_prompts - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" + "MCP list_prompts - MCP server auth headers: %s", + list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None, ) # Get mcp_servers from context variable verbose_logger.debug("MCP list_prompts - Calling _list_prompts") @@ -1170,10 +1174,10 @@ if MCP_AVAILABLE: oauth2_headers=oauth2_headers, raw_headers=raw_headers, ) - verbose_logger.info(f"MCP list_prompts - Successfully returned {len(prompts)} prompts") + verbose_logger.info("MCP list_prompts - Successfully returned %s prompts", len(prompts)) return prompts except Exception as e: - verbose_logger.exception(f"Error in list_prompts endpoint: {e}") + verbose_logger.exception("Error in list_prompts endpoint: %s", e) # Return empty list instead of failing completely # This prevents the HTTP stream from failing and allows the client to get a response return [] @@ -1213,7 +1217,7 @@ if MCP_AVAILABLE: _client_ip, ) = await get_or_extract_auth_context() - verbose_logger.debug(f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}") + verbose_logger.debug("MCP mcp_server_tool_call - User API Key Auth from context: %s", user_api_key_auth) return await mcp_get_prompt( name=name, arguments=arguments, @@ -1248,10 +1252,11 @@ if MCP_AVAILABLE: raw_headers, _client_ip, ) = await get_or_extract_auth_context() - verbose_logger.debug(f"MCP list_resources - User API Key Auth from context: {user_api_key_auth}") - verbose_logger.debug(f"MCP list_resources - MCP servers from context: {mcp_servers}") + verbose_logger.debug("MCP list_resources - User API Key Auth from context: %s", user_api_key_auth) + verbose_logger.debug("MCP list_resources - MCP servers from context: %s", mcp_servers) verbose_logger.debug( - f"MCP list_resources - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" + "MCP list_resources - MCP server auth headers: %s", + list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None, ) resources = await _list_mcp_resources( @@ -1262,10 +1267,10 @@ if MCP_AVAILABLE: oauth2_headers=oauth2_headers, raw_headers=raw_headers, ) - verbose_logger.info(f"MCP list_resources - Successfully returned {len(resources)} resources") + verbose_logger.info("MCP list_resources - Successfully returned %s resources", len(resources)) return resources except Exception as e: - verbose_logger.exception(f"Error in list_resources endpoint: {e}") + verbose_logger.exception("Error in list_resources endpoint: %s", e) return [] finally: if _session_reset_token is not None: @@ -1291,10 +1296,11 @@ if MCP_AVAILABLE: raw_headers, _client_ip, ) = await get_or_extract_auth_context() - verbose_logger.debug(f"MCP list_resource_templates - User API Key Auth from context: {user_api_key_auth}") - verbose_logger.debug(f"MCP list_resource_templates - MCP servers from context: {mcp_servers}") + verbose_logger.debug("MCP list_resource_templates - User API Key Auth from context: %s", user_api_key_auth) + verbose_logger.debug("MCP list_resource_templates - MCP servers from context: %s", mcp_servers) verbose_logger.debug( - f"MCP list_resource_templates - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" + "MCP list_resource_templates - MCP server auth headers: %s", + list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None, ) resource_templates = await _list_mcp_resource_templates( @@ -1306,11 +1312,11 @@ if MCP_AVAILABLE: raw_headers=raw_headers, ) verbose_logger.info( - f"MCP list_resource_templates - Successfully returned {len(resource_templates)} resource templates" + "MCP list_resource_templates - Successfully returned %s resource templates", len(resource_templates) ) return resource_templates except Exception as e: - verbose_logger.exception(f"Error in list_resource_templates endpoint: {e}") + verbose_logger.exception("Error in list_resource_templates endpoint: %s", e) return [] finally: if _session_reset_token is not None: @@ -1400,7 +1406,7 @@ if MCP_AVAILABLE: if server_id == server.server_id: filtered_server[server.server_id] = server except Exception as e: - verbose_logger.debug(f"Could not resolve '{server_or_group}' as access group: {e}") + verbose_logger.debug("Could not resolve '%s' as access group: %s", server_or_group, e) if filtered_server: return list(filtered_server.values()) @@ -1659,7 +1665,7 @@ if MCP_AVAILABLE: creds = await list_user_oauth_credentials(prisma_client, user_id) return {c["server_id"]: c for c in creds if "server_id" in c} except Exception as e: - verbose_logger.warning(f"_prefetch_oauth_creds_for_user: failed to prefetch for user={user_id}: {e}") + verbose_logger.warning("_prefetch_oauth_creds_for_user: failed to prefetch for user=%s: %s", user_id, e) return {} def _prepare_mcp_server_headers( @@ -2023,7 +2029,10 @@ if MCP_AVAILABLE: filtered_tools = apply_tool_overrides(filtered_tools, server) verbose_logger.debug( - f"Successfully fetched {len(tools)} tools from server {server.name}, {len(filtered_tools)} after filtering" + "Successfully fetched %s tools from server %s, %s after filtering", + len(tools), + server.name, + len(filtered_tools), ) return filtered_tools, ServerListOk(tool_count=len(filtered_tools)) except MCPUpstreamAuthError as e: @@ -2033,10 +2042,10 @@ if MCP_AVAILABLE: # 401 + WWW-Authenticate (the MCP session manager serializes it as a JSON-RPC # error). Single-server routes surface it via the request-scope preemptive # check in _raise_preemptive_401_for_unauthenticated_servers instead. - verbose_logger.debug(f"MCP list_tools: omitting {server.name}; it needs upstream auth") + verbose_logger.debug("MCP list_tools: omitting %s; it needs upstream auth", server.name) return [], classify_list_exception(e) except Exception as e: - verbose_logger.exception(f"Error getting tools from server {server.name}: {e}") + verbose_logger.exception("Error getting tools from server %s: %s", server.name, e) return [], classify_list_exception(e) # Fetch tools from all servers in parallel @@ -2089,7 +2098,7 @@ if MCP_AVAILABLE: log_exc, ) - verbose_logger.info(f"Successfully fetched {len(all_tools)} tools total from all MCP servers") + verbose_logger.info("Successfully fetched %s tools total from all MCP servers", len(all_tools)) return AggregateToolListing(tools=all_tools, outcomes=server_outcomes) except Exception as e: @@ -2167,12 +2176,12 @@ if MCP_AVAILABLE: all_prompts.extend(prompts) - verbose_logger.debug(f"Successfully fetched {len(prompts)} prompts from server {server.name}") + verbose_logger.debug("Successfully fetched %s prompts from server %s", len(prompts), server.name) except Exception as e: - verbose_logger.exception(f"Error getting prompts from server {server.name}: {e}") + verbose_logger.exception("Error getting prompts from server %s: %s", server.name, e) # Continue with other servers instead of failing completely - verbose_logger.info(f"Successfully fetched {len(all_prompts)} prompts total from all MCP servers") + verbose_logger.info("Successfully fetched %s prompts total from all MCP servers", len(all_prompts)) return all_prompts @@ -2219,11 +2228,11 @@ if MCP_AVAILABLE: ) all_resources.extend(resources) - verbose_logger.debug(f"Successfully fetched {len(resources)} resources from server {server.name}") + verbose_logger.debug("Successfully fetched %s resources from server %s", len(resources), server.name) except Exception as e: - verbose_logger.exception(f"Error getting resources from server {server.name}: {e}") + verbose_logger.exception("Error getting resources from server %s: %s", server.name, e) - verbose_logger.info(f"Successfully fetched {len(all_resources)} resources total from all MCP servers") + verbose_logger.info("Successfully fetched %s resources total from all MCP servers", len(all_resources)) return all_resources @@ -2356,10 +2365,10 @@ if MCP_AVAILABLE: list_tools_log_source=list_tools_log_source, client_ip=client_ip, ) - verbose_logger.debug(f"Successfully fetched {len(listing.tools)} tools from managed MCP servers") + verbose_logger.debug("Successfully fetched %s tools from managed MCP servers", len(listing.tools)) return listing except Exception as e: - verbose_logger.exception(f"Error getting tools from managed MCP servers: {e}") + verbose_logger.exception("Error getting tools from managed MCP servers: %s", e) # Continue with an empty listing instead of failing completely return AggregateToolListing(tools=[], outcomes={}) @@ -2396,9 +2405,9 @@ if MCP_AVAILABLE: oauth2_headers=oauth2_headers, raw_headers=raw_headers, ) - verbose_logger.debug(f"Successfully fetched {len(managed_prompts)} prompts from managed MCP servers") + verbose_logger.debug("Successfully fetched %s prompts from managed MCP servers", len(managed_prompts)) except Exception as e: - verbose_logger.exception(f"Error getting tools from managed MCP servers: {e}") + verbose_logger.exception("Error getting tools from managed MCP servers: %s", e) # Continue with empty managed tools list instead of failing completely return managed_prompts @@ -2426,9 +2435,9 @@ if MCP_AVAILABLE: oauth2_headers=oauth2_headers, raw_headers=raw_headers, ) - verbose_logger.debug(f"Successfully fetched {len(managed_resources)} resources from managed MCP servers") + verbose_logger.debug("Successfully fetched %s resources from managed MCP servers", len(managed_resources)) except Exception as e: - verbose_logger.exception(f"Error getting resources from managed MCP servers: {e}") + verbose_logger.exception("Error getting resources from managed MCP servers: %s", e) return managed_resources @@ -2814,7 +2823,7 @@ if MCP_AVAILABLE: if isinstance(hook_result, dict) and "arguments" in hook_result: arguments = hook_result["arguments"] - verbose_logger.debug(f"Executing local registry tool: {name}") + verbose_logger.debug("Executing local registry tool: %s", name) # For BYOK servers the credential must be injected via a ContextVar # because the tool function has headers baked into its closure. # Pre-format the full Authorization header value using the server's @@ -3335,7 +3344,7 @@ if MCP_AVAILABLE: result = tool.handler(**arguments) return [TextContent(text=str(result), type="text")] except Exception as e: - verbose_logger.exception(f"Error executing local tool {name}: {e}") + verbose_logger.exception("Error executing local tool %s: %s", name, e) return [TextContent(text=f"Error: {e}", type="text")] def _get_mcp_servers_in_path(path: str) -> list[str] | None: @@ -3986,7 +3995,7 @@ if MCP_AVAILABLE: # to the appropriate response. return exc.response.status_code, exc.response.headers.get("www-authenticate") except Exception as exc: - verbose_logger.debug(f"_probe_upstream_auth: probe to {url} failed ({exc}), allowing request through") + verbose_logger.debug("_probe_upstream_auth: probe to %s failed (%s), allowing request through", url, exc) return 200, None async def _check_passthrough_upstream_auth( @@ -4124,9 +4133,9 @@ if MCP_AVAILABLE: # Extract client IP for MCP access control _client_ip = IPAddressUtils.get_mcp_client_ip(StarletteRequest(scope)) - verbose_logger.debug(f"MCP request mcp_servers (header/path): {mcp_servers}") + verbose_logger.debug("MCP request mcp_servers (header/path): %s", mcp_servers) verbose_logger.debug( - f"MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" + "MCP server auth headers: %s", list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None ) # Strip any client-supplied x-mcp-toolset-id to prevent forgery. @@ -4413,7 +4422,7 @@ if MCP_AVAILABLE: # 500 that surfaces as a cancelled tool call. raise _proxy_exception_to_http_exception(e) except Exception as e: - verbose_logger.exception(f"Error handling MCP request: {e}") + verbose_logger.exception("Error handling MCP request: %s", e) # Try to send a graceful error response for non-HTTP exceptions try: from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR @@ -4424,7 +4433,7 @@ if MCP_AVAILABLE: ) await error_response(scope, receive, send) except Exception as response_error: - verbose_logger.exception(f"Failed to send error response: {response_error}") + verbose_logger.exception("Failed to send error response: %s", response_error) # If we can't send a proper response, re-raise the original error raise e @@ -4445,9 +4454,9 @@ if MCP_AVAILABLE: # Extract client IP for MCP access control _sse_client_ip = IPAddressUtils.get_mcp_client_ip(StarletteRequest(scope)) - verbose_logger.debug(f"MCP request mcp_servers (header/path): {mcp_servers}") + verbose_logger.debug("MCP request mcp_servers (header/path): %s", mcp_servers) verbose_logger.debug( - f"MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" + "MCP server auth headers: %s", list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None ) # Strip any client-supplied x-mcp-toolset-id to prevent forgery. @@ -4524,7 +4533,7 @@ if MCP_AVAILABLE: # 500 that surfaces as a cancelled tool call. raise _proxy_exception_to_http_exception(e) except Exception as e: - verbose_logger.exception(f"Error handling MCP request: {e}") + verbose_logger.exception("Error handling MCP request: %s", e) # Try to send a graceful error response for non-HTTP exceptions try: # Send a proper HTTP error response instead of letting the exception bubble up @@ -4537,7 +4546,7 @@ if MCP_AVAILABLE: ) await error_response(scope, receive, send) except Exception as response_error: - verbose_logger.exception(f"Failed to send error response: {response_error}") + verbose_logger.exception("Failed to send error response: %s", response_error) # If we can't send a proper response, re-raise the original error raise e diff --git a/litellm/proxy/_experimental/mcp_server/sse_transport.py b/litellm/proxy/_experimental/mcp_server/sse_transport.py index 09863a7d391..fd90cb59d97 100644 --- a/litellm/proxy/_experimental/mcp_server/sse_transport.py +++ b/litellm/proxy/_experimental/mcp_server/sse_transport.py @@ -46,7 +46,7 @@ class SseServerTransport: super().__init__() self._endpoint = endpoint self._read_stream_writers = {} - verbose_logger.debug(f"SseServerTransport initialized with endpoint: {endpoint}") + verbose_logger.debug("SseServerTransport initialized with endpoint: %s", endpoint) @asynccontextmanager async def connect_sse(self, request: Request): @@ -67,7 +67,7 @@ class SseServerTransport: session_id = uuid4() session_uri = f"{quote(self._endpoint)}?session_id={session_id.hex}" self._read_stream_writers[session_id] = read_stream_writer - verbose_logger.debug(f"Created new session with ID: {session_id}") + verbose_logger.debug("Created new session with ID: %s", session_id) sse_stream_writer: MemoryObjectSendStream[dict[str, Any]] sse_stream_reader: MemoryObjectReceiveStream[dict[str, Any]] @@ -77,10 +77,10 @@ class SseServerTransport: verbose_logger.debug("Starting SSE writer") async with sse_stream_writer, write_stream_reader: await sse_stream_writer.send({"event": "endpoint", "data": session_uri}) - verbose_logger.debug(f"Sent endpoint event: {session_uri}") + verbose_logger.debug("Sent endpoint event: %s", session_uri) async for message in write_stream_reader: - verbose_logger.debug(f"Sending message via SSE: {message}") + verbose_logger.debug("Sending message via SSE: %s", message) await sse_stream_writer.send( { "event": "message", @@ -108,31 +108,31 @@ class SseServerTransport: try: session_id = UUID(hex=session_id_param) - verbose_logger.debug(f"Parsed session ID: {session_id}") + verbose_logger.debug("Parsed session ID: %s", session_id) except ValueError: - verbose_logger.warning(f"Received invalid session ID: {session_id_param}") + verbose_logger.warning("Received invalid session ID: %s", session_id_param) response = Response("Invalid session ID", status_code=400) return response writer = self._read_stream_writers.get(session_id) if not writer: - verbose_logger.warning(f"Could not find session for ID: {session_id}") + verbose_logger.warning("Could not find session for ID: %s", session_id) response = Response("Could not find session", status_code=404) return response json = await request.json() - verbose_logger.debug(f"Received JSON: {json}") + verbose_logger.debug("Received JSON: %s", json) try: message = types.JSONRPCMessage.model_validate(json) - verbose_logger.debug(f"Validated client message: {message}") + verbose_logger.debug("Validated client message: %s", message) except ValidationError as err: - verbose_logger.error(f"Failed to parse message: {err}") + verbose_logger.error("Failed to parse message: %s", err) response = Response("Could not parse message", status_code=400) await writer.send(err) return response - verbose_logger.debug(f"Sending message to writer: {message}") + verbose_logger.debug("Sending message to writer: %s", message) response = Response("Accepted", status_code=202) await writer.send(message) return response diff --git a/litellm/proxy/_experimental/mcp_server/tool_registry.py b/litellm/proxy/_experimental/mcp_server/tool_registry.py index 1ebeac9993a..f2ef94f412b 100644 --- a/litellm/proxy/_experimental/mcp_server/tool_registry.py +++ b/litellm/proxy/_experimental/mcp_server/tool_registry.py @@ -40,7 +40,7 @@ class MCPToolRegistry: input_schema=input_schema, handler=handler, ) - verbose_logger.debug(f"Registered tool: {name}") + verbose_logger.debug("Registered tool: %s", name) def get_tool(self, name: str) -> MCPTool | None: """ @@ -122,7 +122,7 @@ class MCPToolRegistry: handler = get_instance_fn(handler_name, config_file_path) if handler is None: - verbose_logger.warning(f"Warning: Could not find handler {handler_name} for tool {name}") + verbose_logger.warning("Warning: Could not find handler %s for tool %s", handler_name, name) continue # Register the tool diff --git a/litellm/proxy/_experimental/mcp_server/toolset_db.py b/litellm/proxy/_experimental/mcp_server/toolset_db.py index a321c40b9e2..26c830d5d50 100644 --- a/litellm/proxy/_experimental/mcp_server/toolset_db.py +++ b/litellm/proxy/_experimental/mcp_server/toolset_db.py @@ -55,7 +55,7 @@ async def list_mcp_toolsets( rows = await MCPToolsetRepository(prisma_client).table.find_many(where=where) return [_toolset_from_row(r) for r in rows] except Exception as e: - verbose_proxy_logger.warning(f"litellm.proxy._experimental.mcp_server.toolset_db::list_mcp_toolsets - {e}") + verbose_proxy_logger.warning("litellm.proxy._experimental.mcp_server.toolset_db::list_mcp_toolsets - %s", e) return [] diff --git a/litellm/proxy/_experimental/mcp_server/ui_session_utils.py b/litellm/proxy/_experimental/mcp_server/ui_session_utils.py index 14726cba3a7..0e750453be6 100644 --- a/litellm/proxy/_experimental/mcp_server/ui_session_utils.py +++ b/litellm/proxy/_experimental/mcp_server/ui_session_utils.py @@ -91,7 +91,7 @@ async def admitted_user_context(user_api_key_auth: UserAPIKeyAuth) -> UserAPIKey try: admitted = await MCPRequestHandler._reload_admitted_user(user_id) except HTTPException as e: - verbose_logger.warning(f"MCP dashboard session: admitted-subject reload failed for {user_id}: {e.detail}") + verbose_logger.warning("MCP dashboard session: admitted-subject reload failed for %s: %s", user_id, e.detail) return None return admitted.model_copy(update={"parent_otel_span": user_api_key_auth.parent_otel_span}) diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py index 6d48b31658b..c10b3d95cd3 100644 --- a/litellm/proxy/agent_endpoints/a2a_endpoints.py +++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py @@ -468,7 +468,7 @@ async def _handle_stream_message( obj = normalize_stream_event(obj, served_version, request_id=request_id) yield json.dumps(obj) + "\n" except Exception as e: - verbose_proxy_logger.exception(f"Error streaming A2A response: {e}") + verbose_proxy_logger.exception("Error streaming A2A response: %s", e) if ( use_proxy_hooks and proxy_logging_obj is not None @@ -561,13 +561,13 @@ async def get_agent_card( served_version = _served_version(agent, request) agent_card = normalize_agent_card(agent_card, served_version) - verbose_proxy_logger.debug(f"Returning agent card for '{agent_id}' with proxy URL: {proxy_url}") + verbose_proxy_logger.debug("Returning agent card for '%s' with proxy URL: %s", agent_id, proxy_url) return JSONResponse(content=agent_card) except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error getting agent card: {e}") + verbose_proxy_logger.exception("Error getting agent card: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -615,7 +615,7 @@ async def invoke_agent_a2a( body = await request.json() request_data = body - verbose_proxy_logger.debug(f"A2A request for agent '{agent_id}': {body}") + verbose_proxy_logger.debug("A2A request for agent '%s': %s", agent_id, body) # Validate JSON-RPC format if body.get("jsonrpc") != "2.0": @@ -690,7 +690,9 @@ async def invoke_agent_a2a( if not agent_url and not custom_llm_provider: return _jsonrpc_error(request_id, -32000, f"Agent '{agent_id}' has no URL configured", 500) - verbose_proxy_logger.info(f"Proxying A2A request to agent '{agent_id}' at {agent_url or 'completion-bridge'}") + verbose_proxy_logger.info( + "Proxying A2A request to agent '%s' at %s", agent_id, agent_url or "completion-bridge" + ) # Set up data dict for litellm processing if "metadata" not in body: @@ -965,7 +967,7 @@ async def invoke_agent_a2a( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error invoking agent: {e}") + verbose_proxy_logger.exception("Error invoking agent: %s", e) try: await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, diff --git a/litellm/proxy/agent_endpoints/a2a_routing.py b/litellm/proxy/agent_endpoints/a2a_routing.py index 0410f067560..6446a9ad221 100644 --- a/litellm/proxy/agent_endpoints/a2a_routing.py +++ b/litellm/proxy/agent_endpoints/a2a_routing.py @@ -46,7 +46,7 @@ async def route_a2a_agent_request( # Look up agent in registry agent = global_agent_registry.get_agent_by_name(agent_name) if agent is None: - verbose_proxy_logger.error(f"[A2A] Agent '{agent_name}' not found in registry") + verbose_proxy_logger.error("[A2A] Agent '%s' not found in registry", agent_name) route_name = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type) raise ProxyModelNotFoundError(route=route_name, model_name=model_name) @@ -68,12 +68,12 @@ async def route_a2a_agent_request( # Get API base URL from agent config if not agent.agent_card_params or "url" not in agent.agent_card_params: - verbose_proxy_logger.error(f"[A2A] Agent '{agent_name}' has no URL configured") + verbose_proxy_logger.error("[A2A] Agent '%s' has no URL configured", agent_name) route_name = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type) raise ProxyModelNotFoundError(route=route_name, model_name=model_name) # Inject API base and route to litellm data["api_base"] = agent.agent_card_params["url"] - verbose_proxy_logger.debug(f"[A2A] Routing {model_name} to {data['api_base']}") + verbose_proxy_logger.debug("[A2A] Routing %s to %s", model_name, data["api_base"]) return getattr(litellm, f"{route_type}")(**data) diff --git a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py index 6999228c83d..e0b8b24de97 100644 --- a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py +++ b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py @@ -59,7 +59,7 @@ class AgentRequestHandler: return list(set(allowed_agents)) except Exception as e: - verbose_logger.warning(f"Failed to get allowed agents: {e}") + verbose_logger.warning("Failed to get allowed agents: %s", e) return [] @staticmethod @@ -179,7 +179,7 @@ class AgentRequestHandler: return list(set(all_agents)) except Exception as e: - verbose_logger.warning(f"Failed to get allowed agents for key: {e}") + verbose_logger.warning("Failed to get allowed agents for key: %s", e) return [] @staticmethod @@ -255,7 +255,7 @@ class AgentRequestHandler: # litellm-dashboard is the default UI team and will never have agents; # skip noisy warnings for it. if user_api_key_auth.team_id != UI_TEAM_ID: - verbose_logger.warning(f"Failed to get allowed agents for team: {e}") + verbose_logger.warning("Failed to get allowed agents for team: %s", e) return [] @staticmethod @@ -285,7 +285,7 @@ class AgentRequestHandler: for agent in agents: agent_ids.add(agent.agent_id) except Exception as e: - verbose_logger.debug(f"Error getting agents from access groups: {e}") + verbose_logger.debug("Error getting agents from access groups: %s", e) return agent_ids @staticmethod @@ -310,7 +310,7 @@ class AgentRequestHandler: return list(agent_ids) except Exception as e: - verbose_logger.warning(f"Failed to get agents from access groups: {e}") + verbose_logger.warning("Failed to get agents from access groups: %s", e) return [] @staticmethod @@ -369,7 +369,7 @@ class AgentRequestHandler: return key_object_permission.agent_access_groups or [] except Exception as e: - verbose_logger.warning(f"Failed to get agent access groups for key: {e}") + verbose_logger.warning("Failed to get agent access groups for key: %s", e) return [] @staticmethod @@ -412,5 +412,5 @@ class AgentRequestHandler: return object_permissions.agent_access_groups or [] except Exception as e: - verbose_logger.warning(f"Failed to get agent access groups for team: {e}") + verbose_logger.warning("Failed to get agent access groups for team: %s", e) return [] diff --git a/litellm/proxy/agent_endpoints/endpoints.py b/litellm/proxy/agent_endpoints/endpoints.py index db5341dbe5a..b4c0675c5ef 100644 --- a/litellm/proxy/agent_endpoints/endpoints.py +++ b/litellm/proxy/agent_endpoints/endpoints.py @@ -316,7 +316,7 @@ async def get_agents( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.agent_endpoints.get_agents(): Exception occurred - {e}") + verbose_proxy_logger.exception("litellm.proxy.agent_endpoints.get_agents(): Exception occurred - %s", e) raise HTTPException(status_code=500, detail={"error": f"Internal server error: {e}"}) @@ -434,10 +434,10 @@ async def create_agent( # Also register in memory try: AGENT_REGISTRY.register_agent(agent_config=result) - verbose_proxy_logger.info(f"Successfully registered agent '{agent_name}' (ID: {agent_id}) in memory") + verbose_proxy_logger.info("Successfully registered agent '%s' (ID: %s) in memory", agent_name, agent_id) except Exception as reg_error: verbose_proxy_logger.warning( - f"Failed to register agent '{agent_name}' (ID: {agent_id}) in memory: {reg_error}" + "Failed to register agent '%s' (ID: %s) in memory: %s", agent_name, agent_id, reg_error ) return result @@ -445,7 +445,7 @@ async def create_agent( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error adding agent to db: {e}") + verbose_proxy_logger.exception("Error adding agent to db: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -529,7 +529,7 @@ async def get_agent_by_id( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error getting agent from db: {e}") + verbose_proxy_logger.exception("Error getting agent from db: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -624,14 +624,14 @@ async def update_agent( AGENT_REGISTRY.register_agent(agent_config=result) verbose_proxy_logger.info( - f"Successfully updated agent '{existing_agent.get('agent_name')}' (ID: {agent_id}) in memory" + "Successfully updated agent '%s' (ID: %s) in memory", existing_agent.get("agent_name"), agent_id ) return result except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error updating agent: {e}") + verbose_proxy_logger.exception("Error updating agent: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -727,14 +727,14 @@ async def patch_agent( AGENT_REGISTRY.register_agent(agent_config=result) verbose_proxy_logger.info( - f"Successfully updated agent '{existing_agent.get('agent_name')}' (ID: {agent_id}) in memory" + "Successfully updated agent '%s' (ID: %s) in memory", existing_agent.get("agent_name"), agent_id ) return result except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error updating agent: {e}") + verbose_proxy_logger.exception("Error updating agent: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -789,7 +789,7 @@ async def delete_agent( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error deleting agent: {e}") + verbose_proxy_logger.exception("Error deleting agent: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -884,7 +884,7 @@ async def make_agent_public( await proxy_config.save_config(new_config=config) verbose_proxy_logger.debug( - f"Updated public agent groups to: {litellm.public_agent_groups} by user: {user_api_key_dict.user_id}" + "Updated public agent groups to: %s by user: %s", litellm.public_agent_groups, user_api_key_dict.user_id ) return { @@ -895,7 +895,7 @@ async def make_agent_public( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error making agent public: {e}") + verbose_proxy_logger.exception("Error making agent public: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -988,7 +988,7 @@ async def make_agents_public( await proxy_config.save_config(new_config=config) verbose_proxy_logger.debug( - f"Updated public agent groups to: {litellm.public_agent_groups} by user: {user_api_key_dict.user_id}" + "Updated public agent groups to: %s by user: %s", litellm.public_agent_groups, user_api_key_dict.user_id ) return { @@ -999,7 +999,7 @@ async def make_agents_public( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error making agent public: {e}") + verbose_proxy_logger.exception("Error making agent public: %s", e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/litellm/proxy/agent_endpoints/model_list_helpers.py b/litellm/proxy/agent_endpoints/model_list_helpers.py index 56053f59c85..fc70ca75ae6 100644 --- a/litellm/proxy/agent_endpoints/model_list_helpers.py +++ b/litellm/proxy/agent_endpoints/model_list_helpers.py @@ -40,7 +40,7 @@ async def append_agents_to_model_group( ) ) except Exception as e: - verbose_proxy_logger.debug(f"Error appending agents to model_group/info: {e}") + verbose_proxy_logger.debug("Error appending agents to model_group/info: %s", e) return model_groups @@ -84,6 +84,6 @@ async def append_agents_to_model_info( } ) except Exception as e: - verbose_proxy_logger.debug(f"Error appending agents to v2/model/info: {e}") + verbose_proxy_logger.debug("Error appending agents to v2/model/info: %s", e) return models diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py index bf797b92850..a6c45cf736d 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py @@ -79,12 +79,12 @@ async def get_marketplace(): try: manifest = json.loads(plugin.manifest_json) except json.JSONDecodeError: - verbose_proxy_logger.warning(f"Plugin {plugin.name} has invalid manifest JSON, skipping") + verbose_proxy_logger.warning("Plugin %s has invalid manifest JSON, skipping", plugin.name) continue # Source must be specified for URL-based marketplaces if "source" not in manifest: - verbose_proxy_logger.warning(f"Plugin {plugin.name} has no source field, skipping") + verbose_proxy_logger.warning("Plugin %s has no source field, skipping", plugin.name) continue entry: dict[str, Any] = { @@ -118,7 +118,7 @@ async def get_marketplace(): except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error generating marketplace: {e}") + verbose_proxy_logger.exception("Error generating marketplace: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to generate marketplace: {e}"}, @@ -283,7 +283,7 @@ async def register_plugin( ) action = "created" - verbose_proxy_logger.info(f"Plugin {request.name} {action} successfully") + verbose_proxy_logger.info("Plugin %s %s successfully", request.name, action) return { "status": "success", @@ -301,7 +301,7 @@ async def register_plugin( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error registering plugin: {e}") + verbose_proxy_logger.exception("Error registering plugin: %s", e) raise HTTPException( status_code=500, detail={"error": f"Registration failed: {e}"}, @@ -368,7 +368,7 @@ async def list_plugins( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error listing plugins: {e}") + verbose_proxy_logger.exception("Error listing plugins: %s", e) raise HTTPException( status_code=500, detail={"error": str(e)}, @@ -425,7 +425,7 @@ async def get_plugin( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error getting plugin: {e}") + verbose_proxy_logger.exception("Error getting plugin: %s", e) raise HTTPException( status_code=500, detail={"error": str(e)}, @@ -462,13 +462,13 @@ async def enable_plugin( data={"enabled": True, "updated_at": datetime.now(timezone.utc)}, ) - verbose_proxy_logger.info(f"Plugin {plugin_name} enabled") + verbose_proxy_logger.info("Plugin %s enabled", plugin_name) return {"status": "success", "message": f"Plugin '{plugin_name}' enabled"} except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error enabling plugin: {e}") + verbose_proxy_logger.exception("Error enabling plugin: %s", e) raise HTTPException( status_code=500, detail={"error": str(e)}, @@ -505,13 +505,13 @@ async def disable_plugin( data={"enabled": False, "updated_at": datetime.now(timezone.utc)}, ) - verbose_proxy_logger.info(f"Plugin {plugin_name} disabled") + verbose_proxy_logger.info("Plugin %s disabled", plugin_name) return {"status": "success", "message": f"Plugin '{plugin_name}' disabled"} except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error disabling plugin: {e}") + verbose_proxy_logger.exception("Error disabling plugin: %s", e) raise HTTPException( status_code=500, detail={"error": str(e)}, @@ -545,13 +545,13 @@ async def delete_plugin( await ClaudeCodePluginRepository(prisma_client).table.delete(where={"name": plugin_name}) - verbose_proxy_logger.info(f"Plugin {plugin_name} deleted") + verbose_proxy_logger.info("Plugin %s deleted", plugin_name) return {"status": "success", "message": f"Plugin '{plugin_name}' deleted"} except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error deleting plugin: {e}") + verbose_proxy_logger.exception("Error deleting plugin: %s", e) raise HTTPException( status_code=500, detail={"error": str(e)}, diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 5535928dfac..397ee64d399 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -189,7 +189,7 @@ async def anthropic_response( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.anthropic_response(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.anthropic_response(): Exception occured - %s", e) # Extract model_id from request metadata (same as success path) litellm_metadata = data.get("litellm_metadata", {}) or {} @@ -301,7 +301,7 @@ async def count_tokens( detail=detail, ) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.anthropic_endpoints.count_tokens(): Exception occurred - {e}") + verbose_proxy_logger.exception("litellm.proxy.anthropic_endpoints.count_tokens(): Exception occurred - %s", e) raise HTTPException(status_code=500, detail={"error": f"Internal server error: {e}"}) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 6b1d845ec95..c186bbc15ee 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -136,8 +136,10 @@ def _log_budget_lookup_failure(entity: str, error: Exception) -> None: if any(x in err_str for x in ("column", "schema", "does not exist", "prisma", "migrate")): hint = " Run `prisma db push` or `prisma migrate deploy` to fix schema mismatches." verbose_proxy_logger.error( - f"Budget lookup failed for {entity}; cache will not be populated. " - f"Each request will hit the database. Error: {error}.{hint}" + "Budget lookup failed for %s; cache will not be populated. Each request will hit the database. Error: %s.%s", + entity, + error, + hint, ) @@ -192,7 +194,7 @@ def _is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None if model_group_info is None: # Model not found or no pricing info available # Conservative approach: assume it has cost - verbose_proxy_logger.debug(f"No model group info found for {model_name}, assuming it has cost") + verbose_proxy_logger.debug("No model group info found for %s, assuming it has cost", model_name) if zero_cost_cache is not None: zero_cost_cache[model_name] = False return False @@ -205,7 +207,10 @@ def _is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None # If costs are not explicitly configured (None), assume it has cost if input_cost is None or output_cost is None: verbose_proxy_logger.debug( - f"Model {model_name} has undefined cost (input: {input_cost}, output: {output_cost}), assuming it has cost" + "Model %s has undefined cost (input: %s, output: %s), assuming it has cost", + model_name, + input_cost, + output_cost, ) if zero_cost_cache is not None: zero_cost_cache[model_name] = False @@ -214,7 +219,7 @@ def _is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None # If either cost is non-zero, return False if input_cost > 0 or output_cost > 0: verbose_proxy_logger.debug( - f"Model {model_name} has non-zero cost (input: {input_cost}, output: {output_cost})" + "Model %s has non-zero cost (input: %s, output: %s)", model_name, input_cost, output_cost ) if zero_cost_cache is not None: zero_cost_cache[model_name] = False @@ -246,7 +251,7 @@ def _is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None except Exception as e: # If we can't determine the cost, assume it has cost (conservative approach) - verbose_proxy_logger.debug(f"Error checking cost for model {model_name}: {e}, assuming it has cost") + verbose_proxy_logger.debug("Error checking cost for model %s: %s, assuming it has cost", model_name, e) return False # All models checked have zero cost @@ -957,7 +962,7 @@ async def get_default_end_user_budget( if budget_record is None: verbose_proxy_logger.warning( - f"Default end user budget not found in database: {litellm.max_end_user_budget_id}" + "Default end user budget not found in database: %s", litellm.max_end_user_budget_id ) return None @@ -973,7 +978,7 @@ async def get_default_end_user_budget( return _budget_obj except Exception as e: - verbose_proxy_logger.error(f"Error fetching default end user budget: {e}") + verbose_proxy_logger.error("Error fetching default end user budget: %s", e) return None @@ -1013,7 +1018,7 @@ async def get_team_member_default_budget( budget_record = await BudgetRepository(prisma_client).table.find_unique(where={"budget_id": budget_id}) if budget_record is None: - verbose_proxy_logger.warning(f"Team-default member budget not found in database: {budget_id}") + verbose_proxy_logger.warning("Team-default member budget not found in database: %s", budget_id) return None await user_api_key_cache.async_set_cache( @@ -1025,7 +1030,7 @@ async def get_team_member_default_budget( return LiteLLM_BudgetTable.model_validate(budget_record.dict()) except Exception: - verbose_proxy_logger.exception(f"Error fetching team-default member budget {budget_id}") + verbose_proxy_logger.exception("Error fetching team-default member budget %s", budget_id) return None @@ -1066,7 +1071,7 @@ async def _apply_default_budget_to_end_user( # Apply default budget to end user object end_user_obj.litellm_budget_table = default_budget verbose_proxy_logger.debug( - f"Applied default budget {litellm.max_end_user_budget_id} to end user {end_user_obj.user_id}" + "Applied default budget %s to end user %s", litellm.max_end_user_budget_id, end_user_obj.user_id ) return end_user_obj @@ -1289,7 +1294,7 @@ async def _end_user_id_exists_in_db( if end_user_obj is not None: return True except Exception as e: - verbose_proxy_logger.debug(f"end_user validation: get_end_user_object lookup failed: {e}") + verbose_proxy_logger.debug("end_user validation: get_end_user_object lookup failed: %s", e) try: user_obj = await get_user_object( @@ -1305,7 +1310,7 @@ async def _end_user_id_exists_in_db( if user_obj is not None: return True except Exception as e: - verbose_proxy_logger.debug(f"end_user validation: get_user_object lookup failed: {e}") + verbose_proxy_logger.debug("end_user validation: get_user_object lookup failed: %s", e) return False @@ -1376,7 +1381,7 @@ async def get_tag_objects_batch( ) tag_objects[tag_name] = _tag_obj except Exception as e: - verbose_proxy_logger.debug(f"Error batch fetching tags from database: {e}") + verbose_proxy_logger.debug("Error batch fetching tags from database: %s", e) return tag_objects @@ -1974,7 +1979,10 @@ async def _get_team_object_from_user_api_key_cache( ) except Exception as e: verbose_proxy_logger.debug( - f"Failed to load object_permission for team {team_id} with object_permission_id={_response.object_permission_id}: {e}" + "Failed to load object_permission for team %s with object_permission_id=%s: %s", + team_id, + _response.object_permission_id, + e, ) # save the team object to cache @@ -2250,7 +2258,10 @@ async def get_team_object_by_alias( ) except Exception as e: verbose_proxy_logger.debug( - f"Failed to load object_permission for team {team_obj.team_id} with object_permission_id={team_obj.object_permission_id}: {e}" + "Failed to load object_permission for team %s with object_permission_id=%s: %s", + team_obj.team_id, + team_obj.object_permission_id, + e, ) # Cache the result by both alias and team_id @@ -2610,7 +2621,9 @@ async def get_key_object( ) except Exception as e: verbose_proxy_logger.debug( - f"Failed to load object_permission for key with object_permission_id={_response.object_permission_id}: {e}" + "Failed to load object_permission for key with object_permission_id=%s: %s", + _response.object_permission_id, + e, ) # save the key object to cache diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index 681647814e7..b79e01eda21 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -95,7 +95,9 @@ class UserAPIKeyAuthExceptionHandler: use_x_forwarded_for=general_settings.get("use_x_forwarded_for", False), ) verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - {e}\nRequester IP Address:{requester_ip}", + "litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - %s\nRequester IP Address:%s", + e, + requester_ip, extra={"requester_ip": requester_ip}, ) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index a03ed13180c..df4f4caa78f 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -529,10 +529,11 @@ async def pre_db_read_auth_checks( _allowed_routes = general_settings["allowed_routes"] if premium_user is not True: verbose_proxy_logger.error( - f"Trying to set allowed_routes. This is an Enterprise feature. {CommonProxyErrors.not_premium_user.value}" + "Trying to set allowed_routes. This is an Enterprise feature. %s", + CommonProxyErrors.not_premium_user.value, ) if route not in _allowed_routes: - verbose_proxy_logger.error(f"Route {route} not in allowed_routes={_allowed_routes}") + verbose_proxy_logger.error("Route %s not in allowed_routes=%s", route, _allowed_routes) raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=f"Access forbidden: Route {route} not allowed", @@ -582,7 +583,7 @@ def route_in_additonal_public_routes(current_route: str): return False except Exception as e: - verbose_proxy_logger.error(f"route_in_additonal_public_routes: {e}") + verbose_proxy_logger.error("route_in_additonal_public_routes: %s", e) return False @@ -619,7 +620,7 @@ def get_request_route(request: Request) -> str: return raw_path except Exception as e: verbose_proxy_logger.debug( - f"error on get_request_route: {e}, defaulting to request.url.path={request.url.path}" + "error on get_request_route: %s, defaulting to request.url.path=%s", e, request.url.path ) return str(request.url.path) @@ -639,7 +640,7 @@ def get_request_route_template(request: Request) -> str | None: template = getattr(route, "path", None) return template if isinstance(template, str) and template else None except Exception as e: - verbose_proxy_logger.debug(f"error on get_request_route_template: {e}") + verbose_proxy_logger.debug("error on get_request_route_template: %s", e) return None @@ -781,7 +782,8 @@ async def check_if_request_size_is_safe(request: Request) -> bool: # Check if premium user if premium_user is not True: verbose_proxy_logger.warning( - f"using max_request_size_mb - not checking - this is an enterprise only feature. {CommonProxyErrors.not_premium_user.value}" + "using max_request_size_mb - not checking - this is an enterprise only feature. %s", + CommonProxyErrors.not_premium_user.value, ) return True @@ -791,7 +793,7 @@ async def check_if_request_size_is_safe(request: Request) -> bool: if content_length: header_size = int(content_length) header_size_mb = bytes_to_mb(bytes_value=header_size) - verbose_proxy_logger.debug(f"content_length request size in MB={header_size_mb}") + verbose_proxy_logger.debug("content_length request size in MB=%s", header_size_mb) if header_size_mb > max_request_size_mb: raise ProxyException( @@ -806,7 +808,7 @@ async def check_if_request_size_is_safe(request: Request) -> bool: body_size = len(body) request_size_mb = bytes_to_mb(bytes_value=body_size) - verbose_proxy_logger.debug(f"request body request size in MB={request_size_mb}") + verbose_proxy_logger.debug("request body request size in MB=%s", request_size_mb) if request_size_mb > max_request_size_mb: raise ProxyException( message=f"Request size is too large. Request size is {request_size_mb} MB. Max size is {max_request_size_mb} MB", @@ -841,12 +843,13 @@ async def check_response_size_is_safe(response: Any) -> bool: # Check if premium user if premium_user is not True: verbose_proxy_logger.warning( - f"using max_response_size_mb - not checking - this is an enterprise only feature. {CommonProxyErrors.not_premium_user.value}" + "using max_response_size_mb - not checking - this is an enterprise only feature. %s", + CommonProxyErrors.not_premium_user.value, ) return True response_size_mb = bytes_to_mb(bytes_value=sys.getsizeof(response)) - verbose_proxy_logger.debug(f"response size in MB={response_size_mb}") + verbose_proxy_logger.debug("response size in MB=%s", response_size_mb) if response_size_mb > max_response_size_mb: raise ProxyException( message=f"Response size is too large. Response size is {response_size_mb} MB. Max size is {max_response_size_mb} MB", diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index cf4b47e3180..cb860f5df4e 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -379,8 +379,10 @@ class JWTHandler: if not team_id: return default_value verbose_proxy_logger.debug( - f"JWT Auth: team_id_jwt_field '{self.litellm_jwtauth.team_id_jwt_field}' " - f"returned a list {team_id}; using first element '{team_id[0]}' automatically." + "JWT Auth: team_id_jwt_field '%s' returned a list %s; using first element '%s' automatically.", + self.litellm_jwtauth.team_id_jwt_field, + team_id, + team_id[0], ) team_id = team_id[0] return team_id # type: ignore[return-value] @@ -614,7 +616,7 @@ class JWTHandler: if cached_jwks_uri is not None: return cached_jwks_uri - verbose_proxy_logger.debug(f"JWT Auth: Fetching OIDC discovery document from {url}") + verbose_proxy_logger.debug("JWT Auth: Fetching OIDC discovery document from %s", url) response = await self.http_handler.get(url) if response.status_code != 200: raise Exception( @@ -629,7 +631,7 @@ class JWTHandler: if not jwks_uri: raise Exception(f"JWT Auth: OIDC discovery document at {url} does not contain a 'jwks_uri' field.") - verbose_proxy_logger.debug(f"JWT Auth: Resolved OIDC discovery {url} -> jwks_uri={jwks_uri}") + verbose_proxy_logger.debug("JWT Auth: Resolved OIDC discovery %s -> jwks_uri=%s", url, jwks_uri) await self.user_api_key_cache.async_set_cache( key=cache_key, value=jwks_uri, @@ -655,7 +657,7 @@ class JWTHandler: try: response_json = response.json() except Exception as e: - verbose_proxy_logger.error(f"Error parsing response: {e}. Original Response: {response.text}") + verbose_proxy_logger.error("Error parsing response: %s. Original Response: %s", e, response.text) raise Exception(f"Error parsing response: {e}. Check server logs for original response.") if "keys" in response_json: @@ -749,7 +751,7 @@ class JWTHandler: verbose_proxy_logger.debug("Returning cached OIDC UserInfo") return cached_userinfo - verbose_proxy_logger.debug(f"Calling OIDC UserInfo endpoint: {self.litellm_jwtauth.oidc_userinfo_endpoint}") + verbose_proxy_logger.debug("Calling OIDC UserInfo endpoint: %s", self.litellm_jwtauth.oidc_userinfo_endpoint) try: # Call the UserInfo endpoint with the access token @@ -765,7 +767,7 @@ class JWTHandler: raise Exception(f"OIDC UserInfo endpoint returned status {response.status_code}: {response.text}") userinfo = response.json() - verbose_proxy_logger.debug(f"Received OIDC UserInfo: {userinfo}") + verbose_proxy_logger.debug("Received OIDC UserInfo: %s", userinfo) # Cache the userinfo response await self.user_api_key_cache.async_set_cache( @@ -777,7 +779,7 @@ class JWTHandler: return userinfo except Exception as e: - verbose_proxy_logger.error(f"Error fetching OIDC UserInfo: {e}") + verbose_proxy_logger.error("Error fetching OIDC UserInfo: %s", e) raise Exception(f"Failed to fetch OIDC UserInfo: {e}") _unscoped_jwt_warning_emitted = False @@ -1239,7 +1241,7 @@ class JWTAuthManager: return None, None if team_alias: - verbose_proxy_logger.info(f"JWT Auth: Resolving team by alias: '{team_alias}'") + verbose_proxy_logger.info("JWT Auth: Resolving team by alias: '%s'", team_alias) team_object = await get_team_object_by_alias( team_alias=team_alias, prisma_client=prisma_client, @@ -1250,7 +1252,7 @@ class JWTAuthManager: if team_object: individual_team_id = team_object.team_id verbose_proxy_logger.info( - f"JWT Auth: Resolved team_alias='{team_alias}' to team_id='{individual_team_id}'" + "JWT Auth: Resolved team_alias='%s' to team_id='%s'", team_alias, individual_team_id ) return individual_team_id, team_object @@ -1391,7 +1393,7 @@ class JWTAuthManager: is_allowed = False denied_auth_enforced_pass_through_route = True verbose_proxy_logger.debug( - f"JWT team route check: team_id={team_id}, route={route}, is_allowed={is_allowed}" + "JWT team route check: team_id=%s, route=%s, is_allowed=%s", team_id, route, is_allowed ) if is_allowed: return team_id, team_object @@ -1482,7 +1484,7 @@ class JWTAuthManager: else None ) elif org_alias: - verbose_proxy_logger.info(f"JWT Auth: Resolving org by alias: '{org_alias}'") + verbose_proxy_logger.info("JWT Auth: Resolving org by alias: '%s'", org_alias) org_object = await get_org_object_by_alias( org_alias=org_alias, prisma_client=prisma_client, @@ -1492,7 +1494,7 @@ class JWTAuthManager: ) if org_object: verbose_proxy_logger.info( - f"JWT Auth: Resolved org_alias='{org_alias}' to org_id='{org_object.organization_id}'" + "JWT Auth: Resolved org_alias='%s' to org_id='%s'", org_alias, org_object.organization_id ) # Check if email domain is allowed before attempting to get/create user @@ -1625,7 +1627,7 @@ class JWTAuthManager: detail=f"Team '{header_team_id}' from x-litellm-team-id header is not in your JWT's allowed teams. Allowed teams: {list(allowed_team_ids)}", ) - verbose_proxy_logger.debug(f"Using team_id from x-litellm-team-id header: {header_team_id}") + verbose_proxy_logger.debug("Using team_id from x-litellm-team-id header: %s", header_team_id) return header_team_id @staticmethod @@ -1666,11 +1668,13 @@ class JWTAuthManager: user_role=LitellmUserRoles.PROXY_ADMIN ), # [TODO]: expose an internal service role, for better tracking ) - verbose_proxy_logger.debug(f"Successfully added user {user_object.user_id} to team {team_object.team_id}") + verbose_proxy_logger.debug( + "Successfully added user %s to team %s", user_object.user_id, team_object.team_id + ) except ProxyException as e: if e.type == ProxyErrorTypes.team_member_already_in_team: verbose_proxy_logger.debug( - f"User {user_object.user_id} is already a member of team {team_object.team_id}" + "User %s is already a member of team %s", user_object.user_id, team_object.team_id ) return else: diff --git a/litellm/proxy/auth/litellm_license.py b/litellm/proxy/auth/litellm_license.py index 1f61ef7ea28..357ea284103 100644 --- a/litellm/proxy/auth/litellm_license.py +++ b/litellm/proxy/auth/litellm_license.py @@ -26,7 +26,7 @@ class LicenseCheck: def __init__(self) -> None: self.license_str = os.getenv("LITELLM_LICENSE", None) - verbose_proxy_logger.debug(f"License Str value - {self.license_str}") + verbose_proxy_logger.debug("License Str value - %s", self.license_str) self.http_handler = HTTPHandler(timeout=NON_LLM_CONNECTION_TIMEOUT) self._premium_check_logged = False self.public_key = None @@ -48,11 +48,13 @@ class LicenseCheck: else: self.public_key = None except Exception as e: - verbose_proxy_logger.error(f"Error reading public key: {e}") + verbose_proxy_logger.error("Error reading public key: %s", e) def _verify(self, license_str: str) -> bool: verbose_proxy_logger.debug( - f"litellm.proxy.auth.litellm_license.py::_verify - Checking license against {self.base_url}/verify_license - {license_str}" + "litellm.proxy.auth.litellm_license.py::_verify - Checking license against %s/verify_license - %s", + self.base_url, + license_str, ) url = f"{self.base_url}/verify_license/{license_str}" @@ -79,12 +81,14 @@ class LicenseCheck: assert isinstance(premium, bool) verbose_proxy_logger.debug( - f"litellm.proxy.auth.litellm_license.py::_verify - License={license_str} is premium={premium}" + "litellm.proxy.auth.litellm_license.py::_verify - License=%s is premium=%s", license_str, premium ) return premium except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.auth.litellm_license.py::_verify - Unable to verify License={license_str} via api. - {e}" + "litellm.proxy.auth.litellm_license.py::_verify - Unable to verify License=%s via api. - %s", + license_str, + e, ) return False @@ -96,7 +100,8 @@ class LicenseCheck: try: if not self._premium_check_logged: verbose_proxy_logger.debug( - f"litellm.proxy.auth.litellm_license.py::is_premium() - ENTERING 'IS_PREMIUM' - LiteLLM License={self.license_str}" + "litellm.proxy.auth.litellm_license.py::is_premium() - ENTERING 'IS_PREMIUM' - LiteLLM License=%s", + self.license_str, ) if self.license_str is None: @@ -104,7 +109,8 @@ class LicenseCheck: if not self._premium_check_logged: verbose_proxy_logger.debug( - f"litellm.proxy.auth.litellm_license.py::is_premium() - Updated 'self.license_str' - {self.license_str}" + "litellm.proxy.auth.litellm_license.py::is_premium() - Updated 'self.license_str' - %s", + self.license_str, ) self._premium_check_logged = True @@ -187,6 +193,7 @@ class LicenseCheck: except Exception as e: verbose_proxy_logger.debug( - f"litellm.proxy.auth.litellm_license.py::verify_license_without_api_request - Unable to verify License locally. - {e}" + "litellm.proxy.auth.litellm_license.py::verify_license_without_api_request - Unable to verify License locally. - %s", + e, ) return False diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 24875dae9ab..fa0abe71081 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -132,7 +132,7 @@ def get_key_models( # deduplicate while preserving order all_models = list(dict.fromkeys(all_models)) - verbose_proxy_logger.debug(f"ALL KEY MODELS - {len(all_models)}") + verbose_proxy_logger.debug("ALL KEY MODELS - %s", len(all_models)) return all_models @@ -173,7 +173,7 @@ def get_team_models( # deduplicate while preserving order all_models = list(dict.fromkeys(all_models)) - verbose_proxy_logger.debug(f"ALL TEAM MODELS - {len(all_models)}") + verbose_proxy_logger.debug("ALL TEAM MODELS - %s", len(all_models)) return all_models @@ -448,7 +448,7 @@ def get_all_fallbacks( elif fallback_type == "content_policy": fallbacks_config = getattr(llm_router, "content_policy_fallbacks", []) else: - verbose_proxy_logger.warning(f"Unknown fallback_type: {fallback_type}") + verbose_proxy_logger.warning("Unknown fallback_type: %s", fallback_type) return [] if not fallbacks_config: @@ -463,5 +463,5 @@ def get_all_fallbacks( return fallback_model_group except Exception as e: - verbose_proxy_logger.error(f"Error getting fallbacks for model {model}: {e}") + verbose_proxy_logger.error("Error getting fallbacks for model %s: %s", model, e) return [] diff --git a/litellm/proxy/auth/oauth2_proxy_hook.py b/litellm/proxy/auth/oauth2_proxy_hook.py index a7e34072712..9c0bf11a851 100644 --- a/litellm/proxy/auth/oauth2_proxy_hook.py +++ b/litellm/proxy/auth/oauth2_proxy_hook.py @@ -65,7 +65,7 @@ async def handle_oauth2_proxy_request(request: Request) -> UserAPIKeyAuth: ) oauth2_config_mappings: dict[str, str] = general_settings.get("oauth2_config_mappings") or {} - verbose_proxy_logger.debug(f"Oauth2 config mappings: {oauth2_config_mappings}") + verbose_proxy_logger.debug("Oauth2 config mappings: %s", oauth2_config_mappings) if not oauth2_config_mappings: raise ValueError("Oauth2 config mappings not found in general_settings") diff --git a/litellm/proxy/auth/resolvers/store.py b/litellm/proxy/auth/resolvers/store.py index 0702f47fa99..e15a5668701 100644 --- a/litellm/proxy/auth/resolvers/store.py +++ b/litellm/proxy/auth/resolvers/store.py @@ -132,7 +132,9 @@ class IdentityStore: ) except Exception as e: verbose_proxy_logger.debug( - f"Failed to load object_permission for key with object_permission_id={key.object_permission_id}: {e}" + "Failed to load object_permission for key with object_permission_id=%s: %s", + key.object_permission_id, + e, ) await _cache_key_object( diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 1e63d9746a4..6c485504a0f 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -258,7 +258,7 @@ class RouteChecks: # check if user can access this route query_params = request.query_params user_id = query_params.get("user_id") - verbose_proxy_logger.debug(f"user_id: {user_id} & valid_token.user_id: {valid_token.user_id}") + verbose_proxy_logger.debug("user_id: %s & valid_token.user_id: %s", user_id, valid_token.user_id) if user_id and user_id != valid_token.user_id: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, @@ -326,7 +326,8 @@ class RouteChecks: if "admin_only_routes" in general_settings: if premium_user is not True: verbose_proxy_logger.error( - f"Trying to use 'admin_only_routes' this is an Enterprise only feature. {CommonProxyErrors.not_premium_user.value}" + "Trying to use 'admin_only_routes' this is an Enterprise only feature. %s", + CommonProxyErrors.not_premium_user.value, ) return if route in general_settings["admin_only_routes"]: diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 286837c8909..b295a25ab9f 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -102,7 +102,7 @@ try: enterprise_custom_auth: Callable | None = _enterprise_custom_auth except ImportError as e: - verbose_proxy_logger.debug(f"Error in enterprise custom auth: {e}") + verbose_proxy_logger.debug("Error in enterprise custom auth: %s", e) enterprise_custom_auth = None user_api_key_service_logger_obj = ServiceLogging() # used for tracking latency on OTEL @@ -406,7 +406,7 @@ def _apply_budget_limits_to_end_user_params( if budget_info.model_max_budget is not None: end_user_params["end_user_model_max_budget"] = budget_info.model_max_budget - verbose_proxy_logger.debug(f"Applied budget limits to end user {end_user_id}") + verbose_proxy_logger.debug("Applied budget limits to end user %s", end_user_id) async def user_api_key_auth_websocket(websocket: WebSocket): @@ -865,7 +865,9 @@ async def _resolve_jwt_to_virtual_key( ) if claim_value is None: - verbose_proxy_logger.debug(f"JWT Key Mapping: Claim field '{virtual_key_claim_field}' not found in JWT claims.") + verbose_proxy_logger.debug( + "JWT Key Mapping: Claim field '%s' not found in JWT claims.", virtual_key_claim_field + ) # A missing claim is an unmapped client — apply the no-match policy # rather than returning early. Otherwise a caller can bypass REJECT # simply by presenting a JWT that omits the configured field. For @@ -1248,7 +1250,7 @@ async def _user_api_key_auth_builder( user_email=mapped_user_email, ) except Exception as e: - verbose_proxy_logger.debug(f"JWT mapped-key user_email backfill skipped: {e}") + verbose_proxy_logger.debug("JWT mapped-key user_email backfill skipped: %s", e) else: if mapped_user_obj is not None: valid_token.user_email = mapped_user_obj.user_email @@ -1390,7 +1392,7 @@ async def _user_api_key_auth_builder( skip_budget_checks = _is_model_cost_zero(model=model, llm_router=llm_router) if skip_budget_checks: - verbose_proxy_logger.info(f"Skipping all budget checks for zero-cost model: {model}") + verbose_proxy_logger.info("Skipping all budget checks for zero-cost model: %s", model) # Fetch project object for JWT path if project_id is set _jwt_project_obj = None @@ -1501,7 +1503,7 @@ async def _user_api_key_auth_builder( except Exception as e: if isinstance(e, litellm.BudgetExceededError): raise e - verbose_proxy_logger.debug(f"Unable to find user in db. Error - {e}") + verbose_proxy_logger.debug("Unable to find user in db. Error - %s", e) ### CHECK IF ADMIN ### # note: never string compare api keys, this is vulenerable to a time attack. Use secrets.compare_digest instead @@ -1749,7 +1751,8 @@ async def _user_api_key_auth_builder( ) except Exception as e: verbose_logger.debug( - f"litellm.proxy.auth.user_api_key_auth.py::user_api_key_auth() - Unable to get user from db/cache. Setting user_obj to None. Exception received - {e}" + "litellm.proxy.auth.user_api_key_auth.py::user_api_key_auth() - Unable to get user from db/cache. Setting user_obj to None. Exception received - %s", + e, ) user_obj = None @@ -1775,7 +1778,7 @@ async def _user_api_key_auth_builder( skip_budget_checks = _is_model_cost_zero(model=model, llm_router=llm_router) if skip_budget_checks: - verbose_proxy_logger.info(f"Skipping all budget checks for zero-cost model: {model}") + verbose_proxy_logger.info("Skipping all budget checks for zero-cost model: %s", model) # Check 3. Check if user is in their team budget if not skip_budget_checks and valid_token.team_member_spend is not None: @@ -1839,7 +1842,7 @@ async def _user_api_key_auth_builder( if expiry_time.tzinfo is None or expiry_time.tzinfo.utcoffset(expiry_time) is None: expiry_time = expiry_time.replace(tzinfo=timezone.utc) verbose_proxy_logger.debug( - f"Checking if token expired, expiry time {expiry_time} and current time {current_time}" + "Checking if token expired, expiry time %s and current time %s", expiry_time, current_time ) if expiry_time < current_time: # Token exists but is expired. @@ -2689,11 +2692,13 @@ def get_api_key_from_custom_header(request: Request, custom_litellm_key_header_n if custom_api_key: api_key = _get_bearer_token(api_key=custom_api_key) verbose_proxy_logger.debug( - f"Found custom API key using header: {custom_litellm_key_header_name}, setting api_key={abbreviate_api_key(api_key)}" + "Found custom API key using header: %s, setting api_key=%s", + custom_litellm_key_header_name, + abbreviate_api_key(api_key), ) else: verbose_proxy_logger.exception( - f"No LiteLLM Virtual Key pass. Please set header={custom_litellm_key_header_name}: Bearer " + "No LiteLLM Virtual Key pass. Please set header=%s: Bearer ", custom_litellm_key_header_name ) return api_key @@ -2774,7 +2779,7 @@ async def _lookup_end_user_and_apply_budget( except Exception as e: if isinstance(e, litellm.BudgetExceededError): raise e - verbose_proxy_logger.debug(f"Unable to find user in db. Error - {e}") + verbose_proxy_logger.debug("Unable to find user in db. Error - %s", e) return valid_token, end_user_object @@ -2796,7 +2801,7 @@ async def _enforce_key_and_fallback_model_access( if config != {}: model_list = config.get("model_list", []) new_model_list = model_list - verbose_proxy_logger.debug(f"\n new llm router model list {new_model_list}") + verbose_proxy_logger.debug("\n new llm router model list %s", new_model_list) elif isinstance(valid_token.models, list) and "all-team-models" in valid_token.models: pass else: diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index 0c2764db33f..0d815d370e7 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -120,7 +120,8 @@ async def create_batch( try: data = await _read_request_body(request=request) verbose_proxy_logger.debug( - f"Request received by LiteLLM:\n{json.dumps(data, indent=4)}", + "Request received by LiteLLM:\n%s", + json.dumps(data, indent=4), ) base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) ( @@ -225,8 +226,10 @@ async def create_batch( ) verbose_proxy_logger.debug( - f"Created batch using model: {model_from_file_id}, " - f"original_batch_id: {original_batch_id}, encoded: {encoded_batch_id}" + "Created batch using model: %s, original_batch_id: %s, encoded: %s", + model_from_file_id, + original_batch_id, + encoded_batch_id, ) response.input_file_id = input_file_id @@ -293,7 +296,7 @@ async def create_batch( encode_batch_response_ids(response, model=model_param) - verbose_proxy_logger.debug(f"Created batch using model: {model_param}") + verbose_proxy_logger.debug("Created batch using model: %s", model_param) else: # SCENARIO 3: Fallback to custom_llm_provider (uses env variables) apply_team_provider_credentials( @@ -340,7 +343,7 @@ async def create_batch( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.create_batch(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.create_batch(): Exception occured - %s", e) raise handle_exception_on_proxy(e) @@ -471,7 +474,7 @@ async def retrieve_batch( # If batch is still processing, sync with provider to get latest state if response is not None: verbose_proxy_logger.debug( - f"Batch {batch_id} is in non-terminal state {response.status}, syncing with provider" + "Batch %s is in non-terminal state %s, syncing with provider", batch_id, response.status ) # Retrieve from provider (for non-terminal states or if DB lookup failed) @@ -505,7 +508,7 @@ async def retrieve_batch( encode_batch_response_ids(response, model=model_from_id) verbose_proxy_logger.debug( - f"Retrieved batch using model: {model_from_id}, original_id: {original_batch_id}" + "Retrieved batch using model: %s, original_id: %s", model_from_id, original_batch_id ) elif litellm.enable_loadbalancing_on_batch_endpoints is True or unified_batch_id: @@ -592,7 +595,7 @@ async def retrieve_batch( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.retrieve_batch(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.retrieve_batch(): Exception occured - %s", e) raise handle_exception_on_proxy(e) @@ -641,7 +644,7 @@ async def list_batches( version, ) - verbose_proxy_logger.debug(f"GET /v1/batches after={after} limit={limit}") + verbose_proxy_logger.debug("GET /v1/batches after=%s limit=%s", after, limit) try: if llm_router is None: raise HTTPException( @@ -703,7 +706,7 @@ async def list_batches( for batch in response_data: encode_batch_response_ids(batch, model=model_param) - verbose_proxy_logger.debug(f"Listed batches using model: {model_param}") + verbose_proxy_logger.debug("Listed batches using model: %s", model_param) # SCENARIO 2 (alternative): target_model_names based routing elif target_model_names or data.get("target_model_names", None): @@ -773,7 +776,7 @@ async def list_batches( original_exception=e, request_data={"after": after, "limit": limit}, ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.retrieve_batch(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.retrieve_batch(): Exception occured - %s", e) raise handle_exception_on_proxy(e) @@ -886,7 +889,7 @@ async def cancel_batch( encode_batch_response_ids(response, model=model_from_id) verbose_proxy_logger.debug( - f"Cancelled batch using model: {model_from_id}, original_id: {original_batch_id}" + "Cancelled batch using model: %s, original_id: %s", model_from_id, original_batch_id ) # SCENARIO 2: target_model_names based routing @@ -982,7 +985,7 @@ async def cancel_batch( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.create_batch(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.create_batch(): Exception occured - %s", e) raise handle_exception_on_proxy(e) diff --git a/litellm/proxy/caching_routes.py b/litellm/proxy/caching_routes.py index 50b2f63e18a..d823fa2d532 100644 --- a/litellm/proxy/caching_routes.py +++ b/litellm/proxy/caching_routes.py @@ -43,7 +43,7 @@ def _extract_cache_params() -> dict[str, Any]: cleaned_params = HealthCheckCacheParams(**cache_params).model_dump() if cache_params else {} return masker.mask_dict(cleaned_params) except (AttributeError, TypeError) as e: - verbose_proxy_logger.debug(f"Error extracting cache params: {e}") + verbose_proxy_logger.debug("Error extracting cache params: %s", e) return {} @@ -173,7 +173,7 @@ def _get_redis_client_info(cache_instance) -> tuple[list, int]: client_list = cache_instance.client_list() return client_list, len(client_list) except Exception as e: - verbose_proxy_logger.warning(f"CLIENT LIST command failed (likely restricted on managed Redis): {e}") + verbose_proxy_logger.warning("CLIENT LIST command failed (likely restricted on managed Redis): %s", e) return ["CLIENT LIST command not available on this Redis instance"], -1 diff --git a/litellm/proxy/client/cli/interface.py b/litellm/proxy/client/cli/interface.py index 9f42c3b06bb..ea20cbb5ea4 100644 --- a/litellm/proxy/client/cli/interface.py +++ b/litellm/proxy/client/cli/interface.py @@ -20,7 +20,7 @@ def styled_prompt(): click.echo("\n" * 3) except Exception as e: # Fallback if we can't get terminal size - verbose_logger.debug(f"Error getting terminal size: {e}") + verbose_logger.debug("Error getting terminal size: %s", e) click.echo("\n" * 3) # ASCII box drawing characters diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index bc3bcd233f0..f9cad283166 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -439,7 +439,7 @@ async def _parse_event_data_for_error(event_line: str | bytes) -> int | None: error_code = int(error_code_raw) except ValueError: verbose_proxy_logger.warning( - f"Error code is a string but not a valid integer: {error_code_raw}" + "Error code is a string but not a valid integer: %s", error_code_raw ) # Not a valid integer string, treat as if no valid code was found for this check @@ -447,7 +447,7 @@ async def _parse_event_data_for_error(event_line: str | bytes) -> int | None: if error_code is not None and 100 <= error_code <= 599: return error_code elif error_code_raw is not None: # Log if original code was present but not valid - verbose_proxy_logger.warning(f"Error has invalid or non-convertible code: {error_code_raw}") + verbose_proxy_logger.warning("Error has invalid or non-convertible code: %s", error_code_raw) except (orjson.JSONDecodeError, json.JSONDecodeError): # not a known error chunk pass @@ -644,7 +644,8 @@ async def create_response( # Should return standard JSON error response instead of SSE format final_status_code = error_code_from_chunk verbose_proxy_logger.debug( - f"Error detected in first stream chunk. Returning JSON error response with status code: {final_status_code}" + "Error detected in first stream chunk. Returning JSON error response with status code: %s", + final_status_code, ) # Parse error content @@ -663,7 +664,7 @@ async def create_response( headers=headers, ) except Exception as e: - verbose_proxy_logger.debug(f"Error parsing first chunk value: {e}") + verbose_proxy_logger.debug("Error parsing first chunk value: %s", e) except _ClientDisconnectedBeforeFirstChunk: # Client vanished during the time-to-first-token wait; the upstream @@ -694,7 +695,7 @@ async def create_response( ) except Exception as e: # Unexpected error consuming first chunk. - verbose_proxy_logger.exception(f"Error consuming first chunk from generator: {e}") + verbose_proxy_logger.exception("Error consuming first chunk from generator: %s", e) # Preserve status code from HTTPException (e.g., guardrail blocks) error_status = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR) @@ -943,7 +944,7 @@ def _log_llm_api_exception(e: Exception) -> None: "litellm.proxy.proxy_server._handle_llm_api_exception(): client disconnected, upstream LLM request cancelled" ) return - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - %s", e) async def _cancel_llm_call_on_client_disconnect( @@ -1083,7 +1084,7 @@ class ProxyBaseLLMRequestProcessing: try: return {key: str(value) for key, value in headers.items() if value not in exclude_values} except Exception as e: - verbose_proxy_logger.error(f"Error setting custom headers: {e}") + verbose_proxy_logger.error("Error setting custom headers: %s", e) return {} @staticmethod @@ -2945,7 +2946,7 @@ class ProxyBaseLLMRequestProcessing: raise except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.async_data_generator(): Exception occured - {e}" + "litellm.proxy.proxy_server.async_data_generator(): Exception occured - %s", e ) transformed_exception = await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, @@ -2955,7 +2956,8 @@ class ProxyBaseLLMRequestProcessing: if transformed_exception is not None: e = transformed_exception verbose_proxy_logger.debug( - f"\033[1;31mAn error occurred: {e}\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`" + "\x1b[1;31mAn error occurred: %s\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`", + e, ) if isinstance(e, HTTPException): diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 1eca0eb768c..6325a27b7d1 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -61,7 +61,7 @@ def initialize_callbacks_on_proxy( ) from litellm.proxy.proxy_server import prisma_client - verbose_proxy_logger.debug(f"{blue_color_code}initializing callbacks={value} on proxy{reset_color_code}") + verbose_proxy_logger.debug("%sinitializing callbacks=%s on proxy%s", blue_color_code, value, reset_color_code) if isinstance(value, list): imported_list: list[Any] = [] for callback in value: # ["presidio", ] @@ -298,7 +298,7 @@ def initialize_callbacks_on_proxy( imported_list.append(callback) else: verbose_proxy_logger.debug( - f"{blue_color_code} attempting to import custom calback={callback} {reset_color_code}" + "%s attempting to import custom calback=%s %s", blue_color_code, callback, reset_color_code ) imported_list.append( get_instance_fn( @@ -322,7 +322,7 @@ def initialize_callbacks_on_proxy( config_file_path=config_file_path, ) ] - verbose_proxy_logger.debug(f"{blue_color_code} Initialized Callbacks - {litellm.callbacks} {reset_color_code}") + verbose_proxy_logger.debug("%s Initialized Callbacks - %s %s", blue_color_code, litellm.callbacks, reset_color_code) def get_model_group_from_litellm_kwargs(kwargs: dict) -> str | None: diff --git a/litellm/proxy/common_utils/custom_openapi_spec.py b/litellm/proxy/common_utils/custom_openapi_spec.py index a884eab462a..de4f9ceaa63 100644 --- a/litellm/proxy/common_utils/custom_openapi_spec.py +++ b/litellm/proxy/common_utils/custom_openapi_spec.py @@ -49,7 +49,7 @@ class CustomOpenAPISpec: except Exception as e: # FastAPI 0.120+ may fail schema generation for certain types (e.g., openai.Timeout) # Log the error and return None to skip schema generation for this model - verbose_proxy_logger.debug(f"Failed to generate schema for {model_class}: {e}") + verbose_proxy_logger.debug("Failed to generate schema for %s: %s", model_class, e) return None @staticmethod @@ -267,13 +267,13 @@ class CustomOpenAPISpec: openapi_schema, paths, f"#/components/schemas/{schema_name}" ) - verbose_proxy_logger.debug(f"Successfully added {schema_name} schema to OpenAPI spec") + verbose_proxy_logger.debug("Successfully added %s schema to OpenAPI spec", schema_name) else: - verbose_proxy_logger.debug(f"Could not get schema for {schema_name}") + verbose_proxy_logger.debug("Could not get schema for %s", schema_name) except Exception as e: # If schema addition fails, continue without it - verbose_proxy_logger.debug(f"Failed to add {operation_name} request schema: {e}") + verbose_proxy_logger.debug("Failed to add %s request schema: %s", operation_name, e) return openapi_schema @@ -302,7 +302,7 @@ class CustomOpenAPISpec: operation_name="chat completion", ) except ImportError as e: - verbose_proxy_logger.debug(f"Failed to import ProxyChatCompletionRequest: {e}") + verbose_proxy_logger.debug("Failed to import ProxyChatCompletionRequest: %s", e) return openapi_schema @staticmethod @@ -328,7 +328,7 @@ class CustomOpenAPISpec: operation_name="embedding", ) except ImportError as e: - verbose_proxy_logger.debug(f"Failed to import EmbeddingRequest: {e}") + verbose_proxy_logger.debug("Failed to import EmbeddingRequest: %s", e) return openapi_schema @staticmethod @@ -356,7 +356,7 @@ class CustomOpenAPISpec: operation_name="responses API", ) except ImportError as e: - verbose_proxy_logger.debug(f"Failed to import ResponsesAPIRequestParams: {e}") + verbose_proxy_logger.debug("Failed to import ResponsesAPIRequestParams: %s", e) return openapi_schema @staticmethod diff --git a/litellm/proxy/common_utils/debug_utils.py b/litellm/proxy/common_utils/debug_utils.py index 7d2b303a7ca..7cbf8d4527e 100644 --- a/litellm/proxy/common_utils/debug_utils.py +++ b/litellm/proxy/common_utils/debug_utils.py @@ -29,18 +29,21 @@ def configure_gc_thresholds(): thresholds = [int(x.strip()) for x in gc_threshold_env.split(",")] if len(thresholds) == 3: gc.set_threshold(*thresholds) - verbose_proxy_logger.info(f"GC thresholds set to: {thresholds}") + verbose_proxy_logger.info("GC thresholds set to: %s", thresholds) else: verbose_proxy_logger.warning( - f"GC threshold not set: {gc_threshold_env}. Expected format: 'gen0,gen1,gen2'" + "GC threshold not set: %s. Expected format: 'gen0,gen1,gen2'", gc_threshold_env ) except ValueError as e: - verbose_proxy_logger.warning(f"Failed to parse GC threshold: {gc_threshold_env}. Error: {e}") + verbose_proxy_logger.warning("Failed to parse GC threshold: %s. Error: %s", gc_threshold_env, e) # Log current thresholds current_thresholds = gc.get_threshold() verbose_proxy_logger.info( - f"Current GC thresholds: gen0={current_thresholds[0]}, gen1={current_thresholds[1]}, gen2={current_thresholds[2]}" + "Current GC thresholds: gen0=%s, gen1=%s, gen2=%s", + current_thresholds[0], + current_thresholds[1], + current_thresholds[2], ) @@ -425,12 +428,12 @@ def _get_cache_memory_stats(user_api_key_cache, llm_router, proxy_logging_obj, r ), } except Exception as e: - verbose_proxy_logger.debug(f"Error getting Redis pool info: {e}") + verbose_proxy_logger.debug("Error getting Redis pool info: %s", e) else: cache_stats["redis_usage_cache"] = {"enabled": False} except Exception as e: - verbose_proxy_logger.debug(f"Error calculating cache stats: {e}") + verbose_proxy_logger.debug("Error calculating cache stats: %s", e) cache_stats["error"] = str(e) return cache_stats @@ -496,7 +499,7 @@ def _get_router_memory_stats(llm_router) -> dict[str, Any]: else: litellm_router_memory = {"note": "Router not initialized"} except Exception as e: - verbose_proxy_logger.debug(f"Error getting router memory info: {e}") + verbose_proxy_logger.debug("Error getting router memory info: %s", e) litellm_router_memory = {"error": str(e)} return litellm_router_memory @@ -546,7 +549,7 @@ def _get_process_memory_info(worker_pid: int, include_process_info: bool) -> dic "error": "psutil not installed. Install with: pip install psutil", } except Exception as e: - verbose_proxy_logger.debug(f"Error getting process info: {e}") + verbose_proxy_logger.debug("Error getting process info: %s", e) return {"pid": worker_pid, "error": str(e)} @@ -649,10 +652,10 @@ async def configure_gc_thresholds_endpoint( try: gc.set_threshold(generation_0, generation_1, generation_2) verbose_proxy_logger.info( - f"GC thresholds updated from {old_thresholds} to ({generation_0}, {generation_1}, {generation_2})" + "GC thresholds updated from %s to (%s, %s, %s)", old_thresholds, generation_0, generation_1, generation_2 ) except Exception as e: - verbose_proxy_logger.error(f"Failed to set GC thresholds: {e}") + verbose_proxy_logger.error("Failed to set GC thresholds: %s", e) raise HTTPException(status_code=500, detail=f"Failed to set GC thresholds: {e}") # Get current object count to show immediate impact @@ -783,4 +786,4 @@ def init_verbose_loggers(): except Exception as e: import logging - logging.warning(f"Failed to init verbose loggers: {e}") + logging.warning("Failed to init verbose loggers: %s", e) diff --git a/litellm/proxy/common_utils/encrypt_decrypt_utils.py b/litellm/proxy/common_utils/encrypt_decrypt_utils.py index 651e59ef959..e288de6ec44 100644 --- a/litellm/proxy/common_utils/encrypt_decrypt_utils.py +++ b/litellm/proxy/common_utils/encrypt_decrypt_utils.py @@ -108,7 +108,7 @@ def encrypt_value_helper(value: str, new_encryption_key: str | None = None): return encrypted_value verbose_proxy_logger.debug( - f"Invalid value type passed to encrypt_value: {type(value)} for Value: {value}\n Value must be a string" + "Invalid value type passed to encrypt_value: %s for Value: %s\n Value must be a string", type(value), value ) # if it's not a string - do not encrypt it and return the value return value @@ -150,7 +150,7 @@ def decrypt_value_helper( verbose_proxy_logger.debug(error_message) return value if return_original_value else None - verbose_proxy_logger.debug(f"Unable to decrypt value for key: {key}, returning None") + verbose_proxy_logger.debug("Unable to decrypt value for key: %s, returning None", key) if return_original_value: return value else: diff --git a/litellm/proxy/common_utils/expired_ui_session_key_cleanup_manager.py b/litellm/proxy/common_utils/expired_ui_session_key_cleanup_manager.py index a8acc28d9de..cabd1ca84ef 100644 --- a/litellm/proxy/common_utils/expired_ui_session_key_cleanup_manager.py +++ b/litellm/proxy/common_utils/expired_ui_session_key_cleanup_manager.py @@ -101,7 +101,7 @@ class ExpiredUISessionKeyCleanupManager: e, ) return 0 - verbose_proxy_logger.error(f"Expired UI session key cleanup failed: {e}") + verbose_proxy_logger.error("Expired UI session key cleanup failed: %s", e) return 0 finally: if lock_acquired and self.pod_lock_manager and self.pod_lock_manager.redis_cache: diff --git a/litellm/proxy/common_utils/get_routes.py b/litellm/proxy/common_utils/get_routes.py index 4e3c908a8bb..4f320889fe6 100644 --- a/litellm/proxy/common_utils/get_routes.py +++ b/litellm/proxy/common_utils/get_routes.py @@ -70,5 +70,5 @@ class GetRoutes: else: return None except Exception: - verbose_logger.exception(f"Error getting endpoint name for route: {endpoint_function}") + verbose_logger.exception("Error getting endpoint name for route: %s", endpoint_function) return None diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 67212539cc4..f8cfb14326d 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -72,7 +72,7 @@ async def _read_request_body(request: Request | None) -> dict: # a later raw-body re-read sees the original payload — # banned-param checks must see the same body the handler # acts on. - verbose_proxy_logger.error(f"Invalid form payload: {e}") + verbose_proxy_logger.error("Invalid form payload: %s", e) raise ProxyException( message=f"Invalid form payload: {e}", type="invalid_request_error", @@ -98,7 +98,7 @@ async def _read_request_body(request: Request | None) -> dict: # Above the configured size, skip the repair and raise the 400 now. repair_limit_bytes = MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB * 1024 * 1024 if repair_limit_bytes > 0 and len(body) > repair_limit_bytes: - verbose_proxy_logger.error(f"Invalid JSON payload received: {e}") + verbose_proxy_logger.error("Invalid JSON payload received: %s", e) raise ProxyException( message=f"Invalid JSON payload: {e}", type="invalid_request_error", @@ -120,7 +120,7 @@ async def _read_request_body(request: Request | None) -> dict: parsed_body = json.loads(body_str) except json.JSONDecodeError: # If both orjson and json.loads fail, throw a proper error - verbose_proxy_logger.error(f"Invalid JSON payload received: {e}") + verbose_proxy_logger.error("Invalid JSON payload received: %s", e) raise ProxyException( message=f"Invalid JSON payload: {e}", type="invalid_request_error", @@ -134,11 +134,11 @@ async def _read_request_body(request: Request | None) -> dict: except (json.JSONDecodeError, orjson.JSONDecodeError, ProxyException) as e: # Re-raise ProxyException as-is - verbose_proxy_logger.error(f"Invalid JSON payload received: {e}") + verbose_proxy_logger.error("Invalid JSON payload received: %s", e) raise except Exception as e: # Catch unexpected errors to avoid crashes - verbose_proxy_logger.exception(f"Unexpected error reading request body - {e}") + verbose_proxy_logger.exception("Unexpected error reading request body - %s", e) return {} @@ -159,7 +159,7 @@ def _safe_get_request_query_params(request: Request | None) -> dict: return dict(request.query_params) return {} except Exception as e: - verbose_proxy_logger.debug(f"Unexpected error reading request query params - {e}") + verbose_proxy_logger.debug("Unexpected error reading request query params - %s", e) return {} @@ -172,7 +172,7 @@ def _safe_set_request_parsed_body( return request.scope["parsed_body"] = (tuple(parsed_body.keys()), parsed_body) except Exception as e: - verbose_proxy_logger.debug(f"Unexpected error setting request parsed body - {e}") + verbose_proxy_logger.debug("Unexpected error setting request parsed body - %s", e) def _safe_get_request_headers(request: Request | None) -> dict: @@ -190,11 +190,11 @@ def _safe_get_request_headers(request: Request | None) -> dict: if isinstance(cached, dict): return cached if cached is not None: - verbose_proxy_logger.debug(f"Unexpected cached request headers type - {type(cached)}") + verbose_proxy_logger.debug("Unexpected cached request headers type - %s", type(cached)) try: headers = dict(request.headers) except Exception as e: - verbose_proxy_logger.debug(f"Unexpected error reading request headers - {e}") + verbose_proxy_logger.debug("Unexpected error reading request headers - %s", e) headers = {} try: if state is not None: @@ -393,7 +393,7 @@ def extract_nested_form_metadata(form_data: dict[str, Any], prefix: str = "litel # Skip UploadFile objects - they should not be in metadata if isinstance(value, UploadFile): - verbose_proxy_logger.warning(f"Skipping UploadFile in metadata extraction for key: {key}") + verbose_proxy_logger.warning("Skipping UploadFile in metadata extraction for key: %s", key) continue # Extract the nested path from bracket notation @@ -406,7 +406,7 @@ def extract_nested_form_metadata(form_data: dict[str, Any], prefix: str = "litel parts = path_string.split("][") if not parts or not parts[0]: - verbose_proxy_logger.warning(f"Invalid metadata key format (empty path): {key}") + verbose_proxy_logger.warning("Invalid metadata key format (empty path): %s", key) continue # Navigate/create nested dictionary structure @@ -414,7 +414,7 @@ def extract_nested_form_metadata(form_data: dict[str, Any], prefix: str = "litel for part in parts[:-1]: if not isinstance(current, dict): verbose_proxy_logger.warning( - f"Cannot create nested path - intermediate value is not a dict at: {part}" + "Cannot create nested path - intermediate value is not a dict at: %s", part ) break current = current.setdefault(part, {}) @@ -423,10 +423,10 @@ def extract_nested_form_metadata(form_data: dict[str, Any], prefix: str = "litel if isinstance(current, dict): current[parts[-1]] = value else: - verbose_proxy_logger.warning(f"Cannot set value - parent is not a dict for key: {key}") + verbose_proxy_logger.warning("Cannot set value - parent is not a dict for key: %s", key) except Exception as e: - verbose_proxy_logger.error(f"Error parsing metadata key '{key}': {e}") + verbose_proxy_logger.error("Error parsing metadata key '%s': %s", key, e) continue return metadata @@ -505,7 +505,8 @@ def populate_request_with_path_params(request_data: dict, request: Request) -> d continue request_data.setdefault(key, value) verbose_proxy_logger.debug( - f"populate_request_with_path_params: Found path_params, vector_store_ids={request_data.get('vector_store_ids')}" + "populate_request_with_path_params: Found path_params, vector_store_ids=%s", + request_data.get("vector_store_ids"), ) return request_data @@ -533,7 +534,7 @@ def _add_vector_store_id_from_path(request_data: dict, request: Request) -> None if vector_store_match: vector_store_id = vector_store_match.group(1) verbose_proxy_logger.debug( - f"populate_request_with_path_params: Extracted vector_store_id={vector_store_id} from path={path}" + "populate_request_with_path_params: Extracted vector_store_id=%s from path=%s", vector_store_id, path ) request_data.setdefault("vector_store_id", vector_store_id) existing_ids = request_data.get("vector_store_ids") @@ -543,7 +544,8 @@ def _add_vector_store_id_from_path(request_data: dict, request: Request) -> None else: request_data["vector_store_ids"] = [vector_store_id] verbose_proxy_logger.debug( - f"populate_request_with_path_params: Updated request_data with vector_store_ids={request_data.get('vector_store_ids')}" + "populate_request_with_path_params: Updated request_data with vector_store_ids=%s", + request_data.get("vector_store_ids"), ) else: - verbose_proxy_logger.debug(f"populate_request_with_path_params: No vector_store_id present in path={path}") + verbose_proxy_logger.debug("populate_request_with_path_params: No vector_store_id present in path=%s", path) diff --git a/litellm/proxy/common_utils/key_rotation_manager.py b/litellm/proxy/common_utils/key_rotation_manager.py index 8e065a06979..a7e8f7aeb13 100644 --- a/litellm/proxy/common_utils/key_rotation_manager.py +++ b/litellm/proxy/common_utils/key_rotation_manager.py @@ -84,20 +84,20 @@ class KeyRotationManager: verbose_proxy_logger.debug("No keys are due for rotation at this time") return - verbose_proxy_logger.info(f"Found {len(keys_to_rotate)} keys due for rotation") + verbose_proxy_logger.info("Found %s keys due for rotation", len(keys_to_rotate)) # Rotate each key for key in keys_to_rotate: try: await self._rotate_key(key) key_identifier = key.key_name or (key.token[:8] + "..." if key.token else "unknown") - verbose_proxy_logger.info(f"Successfully rotated key: {key_identifier}") + verbose_proxy_logger.info("Successfully rotated key: %s", key_identifier) except Exception as e: key_identifier = key.key_name or (key.token[:8] + "..." if key.token else "unknown") - verbose_proxy_logger.error(f"Failed to rotate key {key_identifier}: {e}") + verbose_proxy_logger.error("Failed to rotate key %s: %s", key_identifier, e) except Exception as e: - verbose_proxy_logger.error(f"Key rotation process failed: {e}") + verbose_proxy_logger.error("Key rotation process failed: %s", e) finally: # Only release the lock if it was actually acquired if lock_acquired and self.pod_lock_manager and self.pod_lock_manager.redis_cache: diff --git a/litellm/proxy/common_utils/load_config_utils.py b/litellm/proxy/common_utils/load_config_utils.py index 56aedb76590..67b3baaf626 100644 --- a/litellm/proxy/common_utils/load_config_utils.py +++ b/litellm/proxy/common_utils/load_config_utils.py @@ -20,9 +20,9 @@ def get_file_contents_from_s3(bucket_name, object_key): aws_secret_access_key=credentials.secret_key, aws_session_token=credentials.token, # Optional, if using temporary credentials ) - verbose_proxy_logger.debug(f"Retrieving {object_key} from S3 bucket: {bucket_name}") + verbose_proxy_logger.debug("Retrieving %s from S3 bucket: %s", object_key, bucket_name) response = s3_client.get_object(Bucket=bucket_name, Key=object_key) - verbose_proxy_logger.debug(f"Response: {response}") + verbose_proxy_logger.debug("Response: %s", response) # Read the file contents and directly parse YAML file_contents = response["Body"].read().decode("utf-8") @@ -34,9 +34,9 @@ def get_file_contents_from_s3(bucket_name, object_key): except ImportError as e: # this is most likely if a user is not using the litellm docker container - verbose_proxy_logger.error(f"ImportError: {e}") + verbose_proxy_logger.error("ImportError: %s", e) except Exception as e: - verbose_proxy_logger.error(f"Error retrieving file contents: {e}") + verbose_proxy_logger.error("Error retrieving file contents: %s", e) return None @@ -57,7 +57,7 @@ async def get_config_file_contents_from_gcs(bucket_name, object_key): return config except Exception as e: - verbose_proxy_logger.error(f"Error retrieving file contents: {e}") + verbose_proxy_logger.error("Error retrieving file contents: %s", e) return None @@ -93,12 +93,12 @@ def download_python_file_from_s3( aws_session_token=credentials.token, ) - verbose_proxy_logger.debug(f"Downloading Python file {object_key} from S3 bucket: {bucket_name}") + verbose_proxy_logger.debug("Downloading Python file %s from S3 bucket: %s", object_key, bucket_name) response = s3_client.get_object(Bucket=bucket_name, Key=object_key) # Read the file contents file_contents = response["Body"].read().decode("utf-8") - verbose_proxy_logger.debug(f"File contents: {file_contents}") + verbose_proxy_logger.debug("File contents: %s", file_contents) # Ensure directory exists os.makedirs(os.path.dirname(local_file_path), exist_ok=True) @@ -107,14 +107,14 @@ def download_python_file_from_s3( with open(local_file_path, "w") as f: f.write(file_contents) - verbose_proxy_logger.debug(f"Python file downloaded successfully to {local_file_path}") + verbose_proxy_logger.debug("Python file downloaded successfully to %s", local_file_path) return True except ImportError as e: - verbose_proxy_logger.error(f"ImportError: {e}") + verbose_proxy_logger.error("ImportError: %s", e) return False except Exception as e: - verbose_proxy_logger.exception(f"Error downloading Python file: {e}") + verbose_proxy_logger.exception("Error downloading Python file: %s", e) return False @@ -154,11 +154,11 @@ async def download_python_file_from_gcs( with open(local_file_path, "w") as f: f.write(file_contents) - verbose_proxy_logger.debug(f"Python file downloaded successfully to {local_file_path}") + verbose_proxy_logger.debug("Python file downloaded successfully to %s", local_file_path) return True except Exception as e: - verbose_proxy_logger.exception(f"Error downloading Python file from GCS: {e}") + verbose_proxy_logger.exception("Error downloading Python file from GCS: %s", e) return False diff --git a/litellm/proxy/common_utils/openapi_schema_compat.py b/litellm/proxy/common_utils/openapi_schema_compat.py index 06a18524733..919a9fa3939 100644 --- a/litellm/proxy/common_utils/openapi_schema_compat.py +++ b/litellm/proxy/common_utils/openapi_schema_compat.py @@ -80,7 +80,7 @@ def get_openapi_schema_with_compat( except (ImportError, AttributeError) as e: # If patching fails, try normal generation with error handling - verbose_proxy_logger.debug(f"Could not patch Pydantic schema generation: {e}. Trying normal generation.") + verbose_proxy_logger.debug("Could not patch Pydantic schema generation: %s. Trying normal generation.", e) try: return get_openapi_func( title=title, @@ -97,7 +97,7 @@ def get_openapi_schema_with_compat( ): # If we still get the error, log it and return minimal schema verbose_proxy_logger.warning( - f"PydanticSchemaGenerationError during schema generation: {pydantic_error}" + "PydanticSchemaGenerationError during schema generation: %s", pydantic_error ) return { "openapi": "3.0.0", diff --git a/litellm/proxy/common_utils/performance_utils.py b/litellm/proxy/common_utils/performance_utils.py index 50de40480fd..09ef61b7116 100644 --- a/litellm/proxy/common_utils/performance_utils.py +++ b/litellm/proxy/common_utils/performance_utils.py @@ -55,7 +55,7 @@ def _start_profiling(profile_sampling_rate: float) -> None: if _profiler is None: _profiler = cProfile.Profile() _profiler.enable() - verbose_proxy_logger.info(f"Profiling started with sampling rate: {profile_sampling_rate}") + verbose_proxy_logger.info("Profiling started with sampling rate: %s", profile_sampling_rate) def _start_profiling_for_request(profile_sampling_rate: float) -> bool: @@ -77,9 +77,9 @@ def _save_stats(profile_file: PathLib) -> None: _profiler.dump_stats(str(profile_file)) # Re-enable profiler to continue profiling _profiler.enable() - verbose_proxy_logger.debug(f"Profiling stats saved to {profile_file}") + verbose_proxy_logger.debug("Profiling stats saved to %s", profile_file) except Exception as e: - verbose_proxy_logger.error(f"Error saving profiling stats: {e}") + verbose_proxy_logger.error("Error saving profiling stats: %s", e) # Make sure profiler is re-enabled even if there's an error try: _profiler.enable() @@ -178,7 +178,7 @@ def wrap_function_with_line_profiler(module: Any, function_name: str) -> bool: try: original_function = getattr(module, function_name, None) if original_function is None: - verbose_proxy_logger.warning(f"Function {function_name} not found in module {module.__name__}") + verbose_proxy_logger.warning("Function %s not found in module %s", function_name, module.__name__) return False # Store original function if not already wrapped @@ -189,10 +189,10 @@ def wrap_function_with_line_profiler(module: Any, function_name: str) -> bool: profiled_function = _line_profiler(original_function) setattr(module, function_name, profiled_function) - verbose_proxy_logger.info(f"Wrapped {module.__name__}.{function_name} with line_profiler") + verbose_proxy_logger.info("Wrapped %s.%s with line_profiler", module.__name__, function_name) return True except Exception as e: - verbose_proxy_logger.error(f"Error wrapping {function_name} with line_profiler: {e}") + verbose_proxy_logger.error("Error wrapping %s with line_profiler: %s", function_name, e) return False @@ -226,7 +226,7 @@ def wrap_function_directly(func: Callable) -> Callable: _line_profiler.add_function(func) profiled_function = _line_profiler(func) - verbose_proxy_logger.info(f"Wrapped function {func.__name__} with line_profiler") + verbose_proxy_logger.info("Wrapped function %s with line_profiler", func.__name__) return profiled_function @@ -251,7 +251,7 @@ def collect_line_profiler_stats(output_file: str | None = None) -> None: # Save to file output_path = PathLib(output_file) _line_profiler.dump_stats(str(output_path)) - verbose_proxy_logger.info(f"Line profiler stats saved to {output_path}") + verbose_proxy_logger.info("Line profiler stats saved to %s", output_path) else: # Print to stdout from io import StringIO @@ -261,7 +261,7 @@ def collect_line_profiler_stats(output_file: str | None = None) -> None: stats_output = stream.getvalue() verbose_proxy_logger.info("Line profiler stats:\n" + stats_output) except Exception as e: - verbose_proxy_logger.error(f"Error collecting line profiler stats: {e}") + verbose_proxy_logger.error("Error collecting line profiler stats: %s", e) def register_shutdown_handler(output_file: str | None = None) -> None: @@ -282,4 +282,4 @@ def register_shutdown_handler(output_file: str | None = None) -> None: collect_line_profiler_stats(output_file=output_file) atexit.register(shutdown_handler) - verbose_proxy_logger.debug(f"Registered line_profiler shutdown handler for {output_file}") + verbose_proxy_logger.debug("Registered line_profiler shutdown handler for %s", output_file) diff --git a/litellm/proxy/custom_prompt_management.py b/litellm/proxy/custom_prompt_management.py index 355edb69897..ff95e58a3b4 100644 --- a/litellm/proxy/custom_prompt_management.py +++ b/litellm/proxy/custom_prompt_management.py @@ -27,7 +27,10 @@ class X42PromptManagement(CustomPromptManagement): - non_default_params: dict - update with any optional params (e.g. temperature, max_tokens, etc.) to use (can be pulled from prompt management tool) """ verbose_logger.debug( - f"in async get chat completion prompt. Prompt ID: {prompt_id}, Prompt Variables: {prompt_variables}, Dynamic Callback Params: {dynamic_callback_params}" + "in async get chat completion prompt. Prompt ID: %s, Prompt Variables: %s, Dynamic Callback Params: %s", + prompt_id, + prompt_variables, + dynamic_callback_params, ) return model, messages, non_default_params diff --git a/litellm/proxy/db/check_migration.py b/litellm/proxy/db/check_migration.py index 5e53e118e45..62333324c90 100644 --- a/litellm/proxy/db/check_migration.py +++ b/litellm/proxy/db/check_migration.py @@ -97,5 +97,6 @@ def check_prisma_schema_diff(db_url: str | None = None) -> None: has_diff, message = check_prisma_schema_diff_helper(db_url) if has_diff: verbose_logger.exception( - f"🚨🚨🚨 prisma schema out of sync with db. Consider running these sql_commands to sync the two - {message}" + "🚨🚨🚨 prisma schema out of sync with db. Consider running these sql_commands to sync the two - %s", + message, ) diff --git a/litellm/proxy/db/create_views.py b/litellm/proxy/db/create_views.py index 3ced1589757..27d978f5b8c 100644 --- a/litellm/proxy/db/create_views.py +++ b/litellm/proxy/db/create_views.py @@ -238,7 +238,7 @@ async def should_create_missing_views(db: _db) -> bool: result = await db.query_raw(query=sql_query) - verbose_logger.debug(f"Estimated Row count of LiteLLM_SpendLogs = {result}") + verbose_logger.debug("Estimated Row count of LiteLLM_SpendLogs = %s", result) if ( result and isinstance(result, list) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index b24ed4b8282..9d494bf9402 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -163,7 +163,11 @@ class DBSpendUpdateWriter: try: verbose_proxy_logger.debug( - f"Enters prisma db call, response_cost: {response_cost}, token: {token}; user_id: {user_id}; team_id: {team_id}" + "Enters prisma db call, response_cost: %s, token: %s; user_id: %s; team_id: %s", + response_cost, + token, + user_id, + team_id, ) if ProxyUpdateSpend.disable_spend_updates() is True: return @@ -728,7 +732,7 @@ class DBSpendUpdateWriter: if isinstance(request_tags, str): tags = safe_json_loads(request_tags, default=[]) if not tags: - verbose_proxy_logger.debug(f"Failed to parse request_tags JSON: {request_tags}") + verbose_proxy_logger.debug("Failed to parse request_tags JSON: %s", request_tags) return elif isinstance(request_tags, list): tags = request_tags @@ -1120,7 +1124,7 @@ class DBSpendUpdateWriter: ### UPDATE USER TABLE ### user_list_transactions = db_spend_update_transactions["user_list_transactions"] - verbose_proxy_logger.debug(f"User Spend transactions: {user_list_transactions}") + verbose_proxy_logger.debug("User Spend transactions: %s", user_list_transactions) if user_list_transactions is not None and len(user_list_transactions.keys()) > 0: for i in range(n_retry_times + 1): start_time = time.time() @@ -1152,7 +1156,7 @@ class DBSpendUpdateWriter: ### UPDATE END-USER TABLE ### end_user_list_transactions = db_spend_update_transactions["end_user_list_transactions"] - verbose_proxy_logger.debug(f"End-User Spend transactions: {end_user_list_transactions}") + verbose_proxy_logger.debug("End-User Spend transactions: %s", end_user_list_transactions) if end_user_list_transactions is not None and len(end_user_list_transactions.keys()) > 0: await ProxyUpdateSpend.update_end_user_spend( n_retry_times=n_retry_times, @@ -1162,7 +1166,7 @@ class DBSpendUpdateWriter: ) ### UPDATE KEY TABLE ### key_list_transactions = db_spend_update_transactions["key_list_transactions"] - verbose_proxy_logger.debug(f"KEY Spend transactions: {key_list_transactions}") + verbose_proxy_logger.debug("KEY Spend transactions: %s", key_list_transactions) if key_list_transactions is not None and len(key_list_transactions.keys()) > 0: for i in range(n_retry_times + 1): start_time = time.time() @@ -1195,7 +1199,7 @@ class DBSpendUpdateWriter: ### UPDATE TEAM TABLE ### team_list_transactions = db_spend_update_transactions["team_list_transactions"] - verbose_proxy_logger.debug(f"Team Spend transactions: {team_list_transactions}") + verbose_proxy_logger.debug("Team Spend transactions: %s", team_list_transactions) if team_list_transactions is not None and len(team_list_transactions.keys()) > 0: for i in range(n_retry_times + 1): start_time = time.time() @@ -1204,7 +1208,9 @@ class DBSpendUpdateWriter: async with transaction.batch_() as batcher: # Sort by team_id for consistent lock ordering across pods to prevent deadlocks. for team_id, response_cost in sorted(team_list_transactions.items()): - verbose_proxy_logger.debug(f"Updating spend for team id={team_id} by {response_cost}") + verbose_proxy_logger.debug( + "Updating spend for team id=%s by %s", team_id, response_cost + ) batcher.litellm_teamtable.update_many( # 'update_many' prevents error from being raised if no row exists where={"team_id": team_id}, data={"spend": {"increment": response_cost}}, @@ -1226,7 +1232,7 @@ class DBSpendUpdateWriter: ### UPDATE TEAM Membership TABLE with spend ### team_member_list_transactions = db_spend_update_transactions["team_member_list_transactions"] - verbose_proxy_logger.debug(f"Team Membership Spend transactions: {team_member_list_transactions}") + verbose_proxy_logger.debug("Team Membership Spend transactions: %s", team_member_list_transactions) if team_member_list_transactions is not None and len(team_member_list_transactions.keys()) > 0: # Track which team memberships will be updated for cache invalidation team_memberships_to_invalidate: list[tuple[str, str]] = [] @@ -1280,12 +1286,12 @@ class DBSpendUpdateWriter: cache_key = f"team_membership:{user_id}:{team_id}" await user_api_key_cache.async_delete_cache(key=cache_key) verbose_proxy_logger.debug( - f"Invalidated team membership cache for user_id={user_id}, team_id={team_id}" + "Invalidated team membership cache for user_id=%s, team_id=%s", user_id, team_id ) ### UPDATE ORG TABLE ### org_list_transactions = db_spend_update_transactions["org_list_transactions"] - verbose_proxy_logger.debug(f"Org Spend transactions: {org_list_transactions}") + verbose_proxy_logger.debug("Org Spend transactions: %s", org_list_transactions) if org_list_transactions is not None and len(org_list_transactions.keys()) > 0: for i in range(n_retry_times + 1): start_time = time.time() @@ -1368,7 +1374,7 @@ class DBSpendUpdateWriter: """ from litellm.proxy.utils import _raise_failed_update_spend_exception - verbose_proxy_logger.debug(f"{entity_name} Spend transactions: {transactions}") + verbose_proxy_logger.debug("%s Spend transactions: %s", entity_name, transactions) if transactions is not None and len(transactions.keys()) > 0: for i in range(n_retry_times + 1): start_time = time.time() @@ -1378,7 +1384,11 @@ class DBSpendUpdateWriter: # Sort by entity_id for consistent lock ordering across pods to prevent deadlocks. for entity_id, response_cost in sorted(transactions.items()): verbose_proxy_logger.debug( - f"Updating spend for {entity_name} {where_field}={entity_id} by {response_cost}" + "Updating spend for %s %s=%s by %s", + entity_name, + where_field, + entity_id, + response_cost, ) getattr(batcher, table_accessor).update_many( where={where_field: entity_id}, @@ -1507,7 +1517,7 @@ class DBSpendUpdateWriter: from litellm.proxy.utils import _raise_failed_update_spend_exception verbose_proxy_logger.debug( - f"Daily {entity_type.capitalize()} Spend transactions: {len(daily_spend_transactions)}" + "Daily %s Spend transactions: %s", entity_type.capitalize(), len(daily_spend_transactions) ) BATCH_SIZE = 100 start_time = time.time() @@ -1541,7 +1551,7 @@ class DBSpendUpdateWriter: if len(transactions_to_process) == 0: verbose_proxy_logger.debug( - f"No new transactions to process for daily {entity_type} spend update" + "No new transactions to process for daily %s spend update", entity_type ) return @@ -1837,14 +1847,15 @@ class DBSpendUpdateWriter: raise ValueError(f"Invalid type: {type}") if not all(key in payload for key in expected_keys): verbose_proxy_logger.debug( - f"Missing expected keys: {expected_keys}, in payload, skipping from daily_user_spend_transactions" + "Missing expected keys: %s, in payload, skipping from daily_user_spend_transactions", expected_keys ) return None any_expected_keys = ["model", "mcp_namespaced_tool_name"] if not any(key in payload for key in any_expected_keys): verbose_proxy_logger.debug( - f"Missing any expected keys: {any_expected_keys}, in payload, skipping from daily_user_spend_transactions" + "Missing any expected keys: %s, in payload, skipping from daily_user_spend_transactions", + any_expected_keys, ) return None elif "mcp_namespaced_tool_name" in payload: @@ -1856,7 +1867,7 @@ class DBSpendUpdateWriter: return None request_status = prisma_client.get_request_status(payload) - verbose_proxy_logger.debug(f"Logged request status: {request_status}") + verbose_proxy_logger.debug("Logged request status: %s", request_status) _metadata: SpendLogsMetadata = json.loads(payload["metadata"]) usage_obj = _metadata.get("usage_object", {}) or {} if isinstance(payload["startTime"], datetime): @@ -1866,7 +1877,7 @@ class DBSpendUpdateWriter: date = payload["startTime"].split("T")[0] else: verbose_proxy_logger.debug( - f"Invalid start time: {payload['startTime']}, skipping from daily_user_spend_transactions" + "Invalid start time: %s, skipping from daily_user_spend_transactions", payload["startTime"] ) return None try: diff --git a/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py b/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py index 01f6a92485a..604cde4a88c 100644 --- a/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py +++ b/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py @@ -106,7 +106,7 @@ end ) return False except Exception as e: - verbose_proxy_logger.error(f"Error acquiring Redis lock for {cronjob_id}: {e}") + verbose_proxy_logger.error("Error acquiring Redis lock for %s: %s", cronjob_id, e) return False async def release_lock( @@ -148,7 +148,7 @@ end cronjob_id, ) except Exception as e: - verbose_proxy_logger.error(f"Error releasing Redis lock for {cronjob_id}: {e}") + verbose_proxy_logger.error("Error releasing Redis lock for %s: %s", cronjob_id, e) async def _compare_and_delete_lock(self, lock_key: str) -> int: """ diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py index d7c70bdb20c..6e5d83bc500 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py +++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py @@ -43,14 +43,14 @@ class SpendLogCleanup: pod_lock_manager = proxy_logging_obj.db_spend_update_writer.pod_lock_manager self.pod_lock_manager = pod_lock_manager - verbose_proxy_logger.info(f"SpendLogCleanup initialized with batch size: {self.batch_size}") + verbose_proxy_logger.info("SpendLogCleanup initialized with batch size: %s", self.batch_size) def _should_delete_spend_logs(self) -> bool: """ Determines if logs should be deleted based on the max retention period in settings. """ retention_setting = self.general_settings.get("maximum_spend_logs_retention_period") - verbose_proxy_logger.info(f"Checking retention setting: {retention_setting}") + verbose_proxy_logger.info("Checking retention setting: %s", retention_setting) if retention_setting is None: verbose_proxy_logger.info("No retention setting found") @@ -59,16 +59,16 @@ class SpendLogCleanup: try: if isinstance(retention_setting, int): verbose_proxy_logger.warning( - f"maximum_spend_logs_retention_period is an integer ({retention_setting}); treating as days. " - "Use a string like '3d' to be explicit." + "maximum_spend_logs_retention_period is an integer (%s); treating as days. Use a string like '3d' to be explicit.", + retention_setting, ) retention_setting = f"{retention_setting}d" self.retention_seconds = duration_in_seconds(retention_setting) - verbose_proxy_logger.info(f"Retention period set to {self.retention_seconds} seconds") + verbose_proxy_logger.info("Retention period set to %s seconds", self.retention_seconds) return True except ValueError as e: verbose_proxy_logger.warning( - f"Invalid maximum_spend_logs_retention_period value: {retention_setting}, error: {e}" + "Invalid maximum_spend_logs_retention_period value: %s, error: %s", retention_setting, e ) return False @@ -145,15 +145,16 @@ class SpendLogCleanup: deleted_count = deleted_result else: verbose_proxy_logger.error( - f"Unexpected execute_raw return type for {table_name} cleanup: {type(deleted_result)}; " - "aborting cleanup to avoid infinite loop" + "Unexpected execute_raw return type for %s cleanup: %s; aborting cleanup to avoid infinite loop", + table_name, + type(deleted_result), ) break - verbose_proxy_logger.info(f"Deleted {deleted_count} {table_name} rows in this batch") + verbose_proxy_logger.info("Deleted %s %s rows in this batch", deleted_count, table_name) if deleted_count == 0: - verbose_proxy_logger.info(f"No more {table_name} rows to delete. Total deleted: {total_deleted}") + verbose_proxy_logger.info("No more %s rows to delete. Total deleted: %s", table_name, total_deleted) break total_deleted += deleted_count @@ -192,7 +193,7 @@ class SpendLogCleanup: """ lock_acquired = False try: - verbose_proxy_logger.info(f"Cleanup job triggered at {datetime.now()}") + verbose_proxy_logger.info("Cleanup job triggered at %s", datetime.now()) if not self._should_delete_spend_logs(): return @@ -210,7 +211,7 @@ class SpendLogCleanup: or False ) verbose_proxy_logger.info( - f"Lock acquisition attempt: {'successful' if lock_acquired else 'failed'} at {datetime.now()}" + "Lock acquisition attempt: %s at %s", "successful" if lock_acquired else "failed", datetime.now() ) if not lock_acquired: @@ -218,7 +219,7 @@ class SpendLogCleanup: return cutoff_date = datetime.now(timezone.utc) - timedelta(seconds=float(self.retention_seconds)) - verbose_proxy_logger.info(f"Removing logs older than {cutoff_date.isoformat()}") + verbose_proxy_logger.info("Removing logs older than %s", cutoff_date.isoformat()) if self.general_settings.get( "use_spend_logs_partitioning", False @@ -235,13 +236,13 @@ class SpendLogCleanup: # or in a partition that spans the cutoff, so retention must # also delete those stragglers row-wise. total_deleted = await self._delete_old_logs(prisma_client, cutoff_date) - verbose_proxy_logger.info(f"Deleted {total_deleted} expired logs not covered by dropped partitions") + verbose_proxy_logger.info("Deleted %s expired logs not covered by dropped partitions", total_deleted) else: total_deleted = await self._delete_old_logs(prisma_client, cutoff_date) - verbose_proxy_logger.info(f"Deleted {total_deleted} logs") + verbose_proxy_logger.info("Deleted %s logs", total_deleted) index_deleted = await self._delete_old_tool_index_rows(prisma_client, cutoff_date) - verbose_proxy_logger.info(f"Deleted {index_deleted} expired tool index rows") + verbose_proxy_logger.info("Deleted %s expired tool index rows", index_deleted) except Exception as e: # .exception() captures the traceback; str(e) alone on a Prisma/DB diff --git a/litellm/proxy/db/dynamo_db.py b/litellm/proxy/db/dynamo_db.py index 6367c34341f..805d17b36ac 100644 --- a/litellm/proxy/db/dynamo_db.py +++ b/litellm/proxy/db/dynamo_db.py @@ -39,7 +39,7 @@ class DynamoDBWrapper(CustomDB): def set_env_vars_based_on_arn(self): if self.database_arguments.aws_role_name is None: return - verbose_proxy_logger.debug(f"DynamoDB: setting env vars based on arn={self.database_arguments.aws_role_name}") + verbose_proxy_logger.debug("DynamoDB: setting env vars based on arn=%s", self.database_arguments.aws_role_name) import os import boto3 @@ -63,7 +63,7 @@ class DynamoDBWrapper(CustomDB): aws_secret_access_key = assumed_role["Credentials"]["SecretAccessKey"] aws_session_token = assumed_role["Credentials"]["SessionToken"] - verbose_proxy_logger.debug(f"Got STS assumed Role, aws_access_key_id={aws_access_key_id}") + verbose_proxy_logger.debug("Got STS assumed Role, aws_access_key_id=%s", aws_access_key_id) # set these in the env so aiodynamo can use them os.environ["AWS_ACCESS_KEY_ID"] = aws_access_key_id os.environ["AWS_SECRET_ACCESS_KEY"] = aws_secret_access_key diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index 79f86d548c5..f12f29a03d0 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -398,7 +398,7 @@ class PrismaWrapper: return token_created + timedelta(seconds=expires_in) except Exception as e: - verbose_proxy_logger.debug(f"Failed to parse token expiration: {e}") + verbose_proxy_logger.debug("Failed to parse token expiration: %s", e) return None def _calculate_seconds_until_refresh(self) -> float: @@ -420,8 +420,8 @@ class PrismaWrapper: if expiration_time is None: # If we can't parse the token, use fallback interval verbose_proxy_logger.debug( - f"Could not parse token expiration, using fallback interval of " - f"{self.FALLBACK_REFRESH_INTERVAL_SECONDS}s" + "Could not parse token expiration, using fallback interval of %ss", + self.FALLBACK_REFRESH_INTERVAL_SECONDS, ) return self.FALLBACK_REFRESH_INTERVAL_SECONDS @@ -670,8 +670,9 @@ class PrismaWrapper: This is more efficient than polling, requiring only 1 wake-up per token cycle. """ verbose_proxy_logger.info( - f"{self._log_prefix}RDS IAM token refresh loop started. " - f"Tokens will be refreshed {self.TOKEN_REFRESH_BUFFER_SECONDS}s before expiration." + "%sRDS IAM token refresh loop started. Tokens will be refreshed %ss before expiration.", + self._log_prefix, + self.TOKEN_REFRESH_BUFFER_SECONDS, ) while True: @@ -695,8 +696,10 @@ class PrismaWrapper: break except Exception as e: verbose_proxy_logger.error( - f"{self._log_prefix}Error in RDS IAM token refresh loop: {e}. " - f"Retrying in {self.FALLBACK_REFRESH_INTERVAL_SECONDS}s..." + "%sError in RDS IAM token refresh loop: %s. Retrying in %ss...", + self._log_prefix, + e, + self.FALLBACK_REFRESH_INTERVAL_SECONDS, ) # On error, wait before retrying to avoid tight error loops try: @@ -874,7 +877,7 @@ class PrismaManager: try: from litellm_proxy_extras.utils import ProxyExtrasDBManager except ImportError as e: - verbose_proxy_logger.error(f"\033[1;31mLiteLLM: Failed to import proxy extras. Got {e}\033[0m") + verbose_proxy_logger.error("\x1b[1;31mLiteLLM: Failed to import proxy extras. Got %s\x1b[0m", e) return False prisma_dir = PrismaManager._get_prisma_dir() @@ -899,12 +902,12 @@ class PrismaManager: PrismaManager._apply_replica_identity_full_if_requested() return True except subprocess.TimeoutExpired: - verbose_proxy_logger.warning(f"Attempt {attempt + 1} timed out") + verbose_proxy_logger.warning("Attempt %s timed out", attempt + 1) time.sleep(random.randrange(5, 15)) except subprocess.CalledProcessError as e: attempts_left = 3 - attempt retry_msg = f" Retrying... ({attempts_left} attempts left)" if attempts_left > 0 else "" - verbose_proxy_logger.warning(f"The process failed to execute. Details: {e}.{retry_msg}") + verbose_proxy_logger.warning("The process failed to execute. Details: %s.%s", e, retry_msg) time.sleep(random.randrange(5, 15)) finally: os.chdir(original_dir) diff --git a/litellm/proxy/example_config_yaml/custom_guardrail.py b/litellm/proxy/example_config_yaml/custom_guardrail.py index a755390743e..d30ed839ce4 100644 --- a/litellm/proxy/example_config_yaml/custom_guardrail.py +++ b/litellm/proxy/example_config_yaml/custom_guardrail.py @@ -23,7 +23,7 @@ class GuardrailForLBTestingA(CustomGuardrail): call_type: CallTypesLiteral, ) -> Optional[Union[Exception, str, dict]]: guardrail_lb_call_count["A"] += 1 - verbose_proxy_logger.info(f"GuardrailForLBTestingA called. Total A calls: {guardrail_lb_call_count['A']}") + verbose_proxy_logger.info("GuardrailForLBTestingA called. Total A calls: %s", guardrail_lb_call_count["A"]) return data @@ -38,7 +38,7 @@ class GuardrailForLBTestingB(CustomGuardrail): call_type: CallTypesLiteral, ) -> Optional[Union[Exception, str, dict]]: guardrail_lb_call_count["B"] += 1 - verbose_proxy_logger.info(f"GuardrailForLBTestingB called. Total B calls: {guardrail_lb_call_count['B']}") + verbose_proxy_logger.info("GuardrailForLBTestingB called. Total B calls: %s", guardrail_lb_call_count["B"]) return data diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py index 4daab1caf96..3cdd57490c6 100644 --- a/litellm/proxy/fine_tuning_endpoints/endpoints.py +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -112,7 +112,8 @@ async def create_fine_tuning_job( # Convert Pydantic model to dict verbose_proxy_logger.debug( - f"Request received by LiteLLM:\n{json.dumps(data, indent=4)}", + "Request received by LiteLLM:\n%s", + json.dumps(data, indent=4), ) # Include original request and headers in the data @@ -199,7 +200,9 @@ async def create_fine_tuning_job( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.create_fine_tuning_job(): Exception occurred - {e}") + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.create_fine_tuning_job(): Exception occurred - %s", e + ) raise handle_exception_on_proxy(e) @@ -338,7 +341,7 @@ async def retrieve_fine_tuning_job( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.retrieve_fine_tuning_job(): Exception occurred - {e}" + "litellm.proxy.proxy_server.retrieve_fine_tuning_job(): Exception occurred - %s", e ) raise handle_exception_on_proxy(e) @@ -466,7 +469,7 @@ async def list_fine_tuning_jobs( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.list_fine_tuning_jobs(): Exception occurred - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.list_fine_tuning_jobs(): Exception occurred - %s", e) raise handle_exception_on_proxy(e) @@ -604,5 +607,7 @@ async def cancel_fine_tuning_job( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.cancel_fine_tuning_job(): Exception occurred - {e}") + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.cancel_fine_tuning_job(): Exception occurred - %s", e + ) raise handle_exception_on_proxy(e) diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index 12373b7fb97..baf44e5a291 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -313,7 +313,7 @@ async def list_guardrails_v2( return ListGuardrailsResponse(guardrails=guardrail_configs) except Exception as e: - verbose_proxy_logger.exception(f"Error getting guardrails from db: {e}") + verbose_proxy_logger.exception("Error getting guardrails from db: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -397,7 +397,7 @@ async def create_guardrail( try: IN_MEMORY_GUARDRAIL_HANDLER.initialize_guardrail(guardrail=cast(Guardrail, result), source="db") verbose_proxy_logger.info( - f"Immediate sync: Successfully initialized guardrail '{guardrail_name}' (ID: {guardrail_id})" + "Immediate sync: Successfully initialized guardrail '%s' (ID: %s)", guardrail_name, guardrail_id ) except (ValueError, TypeError) as init_error: # Configuration error — roll back the DB write so the guardrail isn't orphaned @@ -405,19 +405,22 @@ async def create_guardrail( try: await _delete_guardrail_row(prisma_client, where={"guardrail_id": guardrail_id}) except Exception as rollback_err: - verbose_proxy_logger.warning(f"Rollback failed for guardrail '{guardrail_id}': {rollback_err}") + verbose_proxy_logger.warning("Rollback failed for guardrail '%s': %s", guardrail_id, rollback_err) raise HTTPException( status_code=400, detail=f"Guardrail configuration error: {init_error}", ) except Exception as init_error: verbose_proxy_logger.warning( - f"Immediate sync: Failed to initialize guardrail '{guardrail_name}' (ID: {guardrail_id}) in memory: {init_error}" + "Immediate sync: Failed to initialize guardrail '%s' (ID: %s) in memory: %s", + guardrail_name, + guardrail_id, + init_error, ) return result except Exception as e: - verbose_proxy_logger.exception(f"Error adding guardrail to db: {e}") + verbose_proxy_logger.exception("Error adding guardrail to db: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -515,11 +518,14 @@ async def update_guardrail( guardrail_id=guardrail_id, guardrail=cast(Guardrail, result) ) verbose_proxy_logger.info( - f"Immediate sync: Successfully updated guardrail '{guardrail_name}' (ID: {guardrail_id})" + "Immediate sync: Successfully updated guardrail '%s' (ID: %s)", guardrail_name, guardrail_id ) except Exception as update_error: verbose_proxy_logger.warning( - f"Immediate sync: Failed to update '{guardrail_name}' (ID: {guardrail_id}) in memory: {update_error}" + "Immediate sync: Failed to update '%s' (ID: %s) in memory: %s", + guardrail_name, + guardrail_id, + update_error, ) return result @@ -587,11 +593,14 @@ async def delete_guardrail( guardrail_id=guardrail_id, ) verbose_proxy_logger.info( - f"Immediate sync: Successfully removed guardrail '{guardrail_name}' (ID: {guardrail_id}) from memory" + "Immediate sync: Successfully removed guardrail '%s' (ID: %s) from memory", guardrail_name, guardrail_id ) except Exception as delete_error: verbose_proxy_logger.warning( - f"Immediate sync: Failed to remove guardrail '{guardrail_name}' (ID: {guardrail_id}) from memory: {delete_error}" + "Immediate sync: Failed to remove guardrail '%s' (ID: %s) from memory: %s", + guardrail_name, + guardrail_id, + delete_error, ) return result @@ -1203,18 +1212,21 @@ async def patch_guardrail( guardrail=guardrail, ) verbose_proxy_logger.info( - f"Immediate sync: Successfully updated guardrail '{guardrail_name}' (ID: {guardrail_id})" + "Immediate sync: Successfully updated guardrail '%s' (ID: %s)", guardrail_name, guardrail_id ) except Exception as update_error: verbose_proxy_logger.warning( - f"Immediate sync: Failed to update '{guardrail_name}' (ID: {guardrail_id}) in memory: {update_error}" + "Immediate sync: Failed to update '%s' (ID: %s) in memory: %s", + guardrail_name, + guardrail_id, + update_error, ) return result except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.exception(f"Error updating guardrail: {e}") + verbose_proxy_logger.exception("Error updating guardrail: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -2126,7 +2138,7 @@ async def test_custom_code_guardrail( ) except Exception as e: - verbose_proxy_logger.exception(f"Error testing custom code guardrail: {e}") + verbose_proxy_logger.exception("Error testing custom code guardrail: %s", e) return TestCustomCodeGuardrailResponse( success=False, error=f"Unexpected error: {e}", diff --git a/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py b/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py index 076cdfc8ecd..d9bcef731bb 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py +++ b/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py @@ -123,7 +123,7 @@ class AimGuardrail(CustomGuardrail): elif action_type == "anonymize_action": return self._anonymize_request(res, data) else: - verbose_proxy_logger.error(f"Aim: {action_type} action") + verbose_proxy_logger.error("Aim: %s action", action_type) return data @staticmethod @@ -328,7 +328,7 @@ class AimGuardrail(CustomGuardrail): from litellm.proxy.proxy_server import StreamingCallbackError raise StreamingCallbackError(blocking_message) - verbose_proxy_logger.error(f"Unknown message received from AIM: {result}") + verbose_proxy_logger.error("Unknown message received from AIM: %s", result) return async def forward_the_stream_to_aim( diff --git a/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py b/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py index 3df4f230dbf..95fe649a565 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py +++ b/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py @@ -58,7 +58,7 @@ class AzureContentSafetyPromptShieldGuardrail(AzureGuardrailBase, CustomGuardrai **kwargs, ) - verbose_proxy_logger.debug(f"Initialized Azure Prompt Shield Guardrail: {guardrail_name}") + verbose_proxy_logger.debug("Initialized Azure Prompt Shield Guardrail: %s", guardrail_name) async def async_make_request(self, user_prompt: str) -> "AzurePromptShieldGuardrailResponse": """ @@ -127,7 +127,7 @@ class AzureContentSafetyPromptShieldGuardrail(AzureGuardrailBase, CustomGuardrai user_prompt = self.get_user_prompt(new_messages) if user_prompt: - verbose_proxy_logger.debug(f"Azure Prompt Shield: User prompt: {user_prompt}") + verbose_proxy_logger.debug("Azure Prompt Shield: User prompt: %s", user_prompt) await self.async_make_request( user_prompt=user_prompt, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/azure/text_moderation.py b/litellm/proxy/guardrails/guardrail_hooks/azure/text_moderation.py index 0b1faf99469..5355631def6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/azure/text_moderation.py +++ b/litellm/proxy/guardrails/guardrail_hooks/azure/text_moderation.py @@ -90,7 +90,7 @@ class AzureContentSafetyTextModerationGuardrail(AzureGuardrailBase, CustomGuardr self.severity_threshold = int(severity_threshold) if severity_threshold else None self.severity_threshold_by_category = severity_threshold_by_category - verbose_proxy_logger.info(f"Initialized Azure Text Moderation Guardrail: {guardrail_name}") + verbose_proxy_logger.info("Initialized Azure Text Moderation Guardrail: %s", guardrail_name) @staticmethod def get_config_model() -> type["GuardrailConfigModel"] | None: @@ -223,7 +223,7 @@ class AzureContentSafetyTextModerationGuardrail(AzureGuardrailBase, CustomGuardr user_prompt = self.get_user_prompt(new_messages) if user_prompt: - verbose_proxy_logger.info(f"Azure Text Moderation: User prompt: {user_prompt}") + verbose_proxy_logger.info("Azure Text Moderation: User prompt: %s", user_prompt) await self.async_make_request( text=user_prompt, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 5aae14b83e6..dab9a3d47f5 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -2157,7 +2157,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): # dict.get("texts", []) would return None if the key exists with a None value. texts = inputs.get("texts") or [] try: - verbose_proxy_logger.debug(f"Bedrock Guardrail: Applying guardrail to {len(texts)} text(s)") + verbose_proxy_logger.debug("Bedrock Guardrail: Applying guardrail to %s text(s)", len(texts)) if input_type == "request": incremental_result = await self._apply_incremental_request_scan( diff --git a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py index e0411de2db4..661e1d4c749 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py +++ b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py @@ -264,7 +264,7 @@ class CatoNetworksGuardrail(CustomGuardrail): elif action_type == "anonymize_action": return self._anonymize_request(res, data) else: - verbose_proxy_logger.error(f"Cato: {action_type} action") + verbose_proxy_logger.error("Cato: %s action", action_type) return data def _handle_block_action(self, analysis_result: Any, required_action: Any) -> None: @@ -555,7 +555,7 @@ class CatoNetworksGuardrail(CustomGuardrail): return if blocking_message := result.get("blocking_message"): raise StreamingCallbackError(blocking_message) - verbose_proxy_logger.error(f"Unknown message received from Cato: {result}") + verbose_proxy_logger.error("Unknown message received from Cato: %s", result) return finally: await self._cancel_background_task(sender) diff --git a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py index 600da6ecfc4..b93a1f99a3a 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py @@ -275,7 +275,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): # Pass relevant kwargs to the parent class super().__init__(guardrail_name=guardrail_name, **kwargs) verbose_proxy_logger.debug( - f"Initialized CrowdStrike AIDR Guardrail: name={guardrail_name}, api_base={self.api_base}" + "Initialized CrowdStrike AIDR Guardrail: name=%s, api_base=%s", guardrail_name, self.api_base ) async def _call_crowdstrike_aidr_guard( @@ -306,7 +306,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): } verbose_proxy_logger.debug( - f"CrowdStrike AIDR Guardrail ({hook_name}): Calling endpoint {endpoint} with payload: {payload}" + "CrowdStrike AIDR Guardrail (%s): Calling endpoint %s with payload: %s", hook_name, endpoint, payload ) response = await self.async_handler.post(url=endpoint, json=payload, headers=headers) @@ -317,7 +317,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): if result.blocked: verbose_proxy_logger.warning( - f"CrowdStrike AIDR Guardrail ({hook_name}): Request blocked. Response: {result}" + "CrowdStrike AIDR Guardrail (%s): Request blocked. Response: %s", hook_name, result ) raise HTTPException( status_code=400, # Bad Request, indicating violation @@ -327,7 +327,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): }, ) verbose_proxy_logger.debug( - f"CrowdStrike AIDR Guardrail ({hook_name}): Request passed. Response: {result.detectors}" + "CrowdStrike AIDR Guardrail (%s): Request passed. Response: %s", hook_name, result.detectors ) return result @@ -396,7 +396,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): input_type: Literal["request", "response"], logging_obj: Optional["LiteLLMLoggingObj"] = None, ) -> GenericGuardrailAPIInputs: - verbose_proxy_logger.debug(f"CrowdStrike AIDR Guardrail: Applying guardrail to {input_type}") + verbose_proxy_logger.debug("CrowdStrike AIDR Guardrail: Applying guardrail to %s", input_type) # Extract inputs texts = inputs.get("texts", []) diff --git a/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py index c7a036562f7..8b7c231b690 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py @@ -176,7 +176,7 @@ class CustomCodeGuardrail(CustomGuardrail): try: self._do_compile() - verbose_proxy_logger.debug(f"Custom code guardrail '{self.guardrail_name}' compiled successfully") + verbose_proxy_logger.debug("Custom code guardrail '%s' compiled successfully", self.guardrail_name) except SyntaxError as e: self._compile_error = f"Syntax error in custom code: {e}" @@ -254,7 +254,7 @@ class CustomCodeGuardrail(CustomGuardrail): # Pre-call block uses passthrough; must not wrap as execution error (500) raise except Exception as e: - verbose_proxy_logger.error(f"Custom code guardrail '{self.guardrail_name}' execution error: {e}") + verbose_proxy_logger.error("Custom code guardrail '%s' execution error: %s", self.guardrail_name, e) raise CustomCodeExecutionError( f"Custom code guardrail execution failed: {e}", details={ @@ -308,15 +308,16 @@ class CustomCodeGuardrail(CustomGuardrail): """ if not isinstance(result, dict): verbose_proxy_logger.warning( - f"Custom code guardrail '{self.guardrail_name}': " - f"Expected dict result, got {type(result).__name__}. Treating as allow." + "Custom code guardrail '%s': Expected dict result, got %s. Treating as allow.", + self.guardrail_name, + type(result).__name__, ) return inputs action = result.get("action", "allow") if action == "allow": - verbose_proxy_logger.debug(f"Custom code guardrail '{self.guardrail_name}': Allowing {input_type}") + verbose_proxy_logger.debug("Custom code guardrail '%s': Allowing %s", self.guardrail_name, input_type) return inputs elif action == "block": @@ -324,7 +325,7 @@ class CustomCodeGuardrail(CustomGuardrail): detection_info = result.get("detection_info", {}) verbose_proxy_logger.info( - f"Custom code guardrail '{self.guardrail_name}': Blocking {input_type} - {reason}" + "Custom code guardrail '%s': Blocking %s - %s", self.guardrail_name, input_type, reason ) is_output = input_type == "response" @@ -348,7 +349,7 @@ class CustomCodeGuardrail(CustomGuardrail): ) elif action == "modify": - verbose_proxy_logger.debug(f"Custom code guardrail '{self.guardrail_name}': Modifying {input_type}") + verbose_proxy_logger.debug("Custom code guardrail '%s': Modifying %s", self.guardrail_name, input_type) # Apply modifications modified_inputs = dict(inputs) @@ -366,7 +367,7 @@ class CustomCodeGuardrail(CustomGuardrail): else: verbose_proxy_logger.warning( - f"Custom code guardrail '{self.guardrail_name}': Unknown action '{action}'. Treating as allow." + "Custom code guardrail '%s': Unknown action '%s'. Treating as allow.", self.guardrail_name, action ) return inputs @@ -393,7 +394,7 @@ class CustomCodeGuardrail(CustomGuardrail): try: self.custom_code = new_code self._do_compile() - verbose_proxy_logger.info(f"Custom code guardrail '{self.guardrail_name}': Code updated successfully") + verbose_proxy_logger.info("Custom code guardrail '%s': Code updated successfully", self.guardrail_name) except SyntaxError as e: # Rollback on failure self.custom_code = old_code diff --git a/litellm/proxy/guardrails/guardrail_hooks/custom_code/primitives.py b/litellm/proxy/guardrails/guardrail_hooks/custom_code/primitives.py index 43a3671ad97..f77bd324462 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/custom_code/primitives.py +++ b/litellm/proxy/guardrails/guardrail_hooks/custom_code/primitives.py @@ -94,7 +94,7 @@ def regex_match(text: str, pattern: str, flags: int = 0) -> bool: try: return bool(re.search(pattern, text, flags)) except re.error as e: - verbose_proxy_logger.warning(f"Starlark regex_match error: {e}") + verbose_proxy_logger.warning("Starlark regex_match error: %s", e) return False @@ -113,7 +113,7 @@ def regex_match_all(text: str, pattern: str, flags: int = 0) -> bool: try: return bool(re.fullmatch(pattern, text, flags)) except re.error as e: - verbose_proxy_logger.warning(f"Starlark regex_match_all error: {e}") + verbose_proxy_logger.warning("Starlark regex_match_all error: %s", e) return False @@ -133,7 +133,7 @@ def regex_replace(text: str, pattern: str, replacement: str, flags: int = 0) -> try: return re.sub(pattern, replacement, text, flags=flags) except re.error as e: - verbose_proxy_logger.warning(f"Starlark regex_replace error: {e}") + verbose_proxy_logger.warning("Starlark regex_replace error: %s", e) return text @@ -152,7 +152,7 @@ def regex_find_all(text: str, pattern: str, flags: int = 0) -> list[str]: try: return re.findall(pattern, text, flags) except re.error as e: - verbose_proxy_logger.warning(f"Starlark regex_find_all error: {e}") + verbose_proxy_logger.warning("Starlark regex_find_all error: %s", e) return [] @@ -174,7 +174,7 @@ def json_parse(text: str) -> Any | None: try: return json.loads(text) except (json.JSONDecodeError, TypeError) as e: - verbose_proxy_logger.debug(f"Starlark json_parse error: {e}") + verbose_proxy_logger.debug("Starlark json_parse error: %s", e) return None @@ -191,7 +191,7 @@ def json_stringify(obj: Any) -> str: try: return json.dumps(obj) except (TypeError, ValueError) as e: - verbose_proxy_logger.warning(f"Starlark json_stringify error: {e}") + verbose_proxy_logger.warning("Starlark json_stringify error: %s", e) return "" @@ -222,7 +222,7 @@ def json_schema_valid(obj: Any, schema: dict[str, Any]) -> bool: return False raise except Exception as e: - verbose_proxy_logger.warning(f"Custom code json_schema_valid error: {e}") + verbose_proxy_logger.warning("Custom code json_schema_valid error: %s", e) return False @@ -473,16 +473,16 @@ async def http_request( return _http_success_response(response) except httpx.TimeoutException as e: - verbose_proxy_logger.warning(f"Custom code http_request timeout: {e}") + verbose_proxy_logger.warning("Custom code http_request timeout: %s", e) return _http_error_response(f"Request timeout after {timeout}s") except httpx.HTTPStatusError as e: # Return the response even for non-2xx status codes return _http_success_response(e.response) except httpx.RequestError as e: - verbose_proxy_logger.warning(f"Custom code http_request error: {e}") + verbose_proxy_logger.warning("Custom code http_request error: %s", e) return _http_error_response(f"Request failed: {e}") except Exception as e: - verbose_proxy_logger.warning(f"Custom code http_request unexpected error: {e}") + verbose_proxy_logger.warning("Custom code http_request unexpected error: %s", e) return _http_error_response(f"Unexpected error: {e}") diff --git a/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py b/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py index 73306ce1154..26f61418cae 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py +++ b/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py @@ -237,7 +237,7 @@ class HiddenlayerGuardrail(CustomGuardrail): response.raise_for_status() result = response.json() - verbose_proxy_logger.debug(f"Hiddenlayer reponse: {result}") + verbose_proxy_logger.debug("Hiddenlayer reponse: %s", result) return result except HTTPStatusError as e: @@ -261,7 +261,7 @@ class HiddenlayerGuardrail(CustomGuardrail): response.raise_for_status() result = response.json() - verbose_proxy_logger.debug(f"Hiddenlayer reponse: {result}") + verbose_proxy_logger.debug("Hiddenlayer reponse: %s", result) return result @staticmethod @@ -434,7 +434,7 @@ class HiddenlayerGuardrailV2(CustomGuardrail): ) response.raise_for_status() - verbose_proxy_logger.debug(f"Hiddenlayer reponse: {response}") + verbose_proxy_logger.debug("Hiddenlayer reponse: %s", response) return response except HTTPStatusError as e: @@ -457,7 +457,7 @@ class HiddenlayerGuardrailV2(CustomGuardrail): response.raise_for_status() - verbose_proxy_logger.debug(f"Hiddenlayer reponse: {response}") + verbose_proxy_logger.debug("Hiddenlayer reponse: %s", response) return response @staticmethod diff --git a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py index 90d131893c6..6b713e4d519 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py +++ b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py @@ -116,8 +116,10 @@ class LassoGuardrail(CustomGuardrail): self.api_base = api_base or os.getenv("LASSO_API_BASE") or "https://server.lasso.security/gateway/v3" verbose_proxy_logger.debug( - f"Lasso guardrail initialized: {kwargs.get('guardrail_name', 'unknown')}, " - f"event_hook: {kwargs.get('event_hook', 'unknown')}, mask: {self.mask}" + "Lasso guardrail initialized: %s, event_hook: %s, mask: %s", + kwargs.get("guardrail_name", "unknown"), + kwargs.get("event_hook", "unknown"), + self.mask, ) super().__init__(**kwargs) @@ -299,7 +301,7 @@ class LassoGuardrail(CustomGuardrail): except Exception as e: if isinstance(e, HTTPException): raise e - verbose_proxy_logger.error(f"Error in post-call Lasso masking: {e}") + verbose_proxy_logger.error("Error in post-call Lasso masking: %s", e) raise LassoGuardrailAPIError(f"Failed to apply post-call masking: {e}") else: # Use the same data for conversation_id consistency (no cache access needed) @@ -308,7 +310,7 @@ class LassoGuardrail(CustomGuardrail): else: verbose_proxy_logger.warning("No response messages found to validate") else: - verbose_proxy_logger.warning(f"Unexpected response type for post-call hook: {type(response)}") + verbose_proxy_logger.warning("Unexpected response type for post-call hook: %s", type(response)) return response @@ -353,7 +355,7 @@ class LassoGuardrail(CustomGuardrail): if cached_conversation_id: return cached_conversation_id except Exception as e: - verbose_proxy_logger.warning(f"Cache retrieval failed: {e}") + verbose_proxy_logger.warning("Cache retrieval failed: %s", e) # Generate new conversation_id and store in cache generated_id = self._generate_ulid() @@ -361,7 +363,7 @@ class LassoGuardrail(CustomGuardrail): try: cache.set_cache(cache_key, generated_id, ttl=3600) # Cache for 1 hour except Exception as e: - verbose_proxy_logger.warning(f"Cache storage failed: {e}") + verbose_proxy_logger.warning("Cache storage failed: %s", e) return generated_id @@ -599,7 +601,8 @@ class LassoGuardrail(CustomGuardrail): # Log error with context verbose_proxy_logger.error( - f"Error calling Lasso API: {error}", + "Error calling Lasso API: %s", + error, extra={ "guardrail_name": getattr(self, "guardrail_name", "unknown"), "message_type": message_type, @@ -810,7 +813,7 @@ class LassoGuardrail(CustomGuardrail): ) -> LassoResponse: """Call the Lasso API and return the response.""" url = api_url or f"{self.api_base}/classify" - verbose_proxy_logger.debug(f"Calling Lasso API with messageType: {payload.get('messageType')}") + verbose_proxy_logger.debug("Calling Lasso API with messageType: %s", payload.get("messageType")) response = await self.async_handler.post( url=url, headers=headers, @@ -848,7 +851,7 @@ class LassoGuardrail(CustomGuardrail): """ if response and response.get("violations_detected") is True: violated_deputies = self._parse_violated_deputies(response) - verbose_proxy_logger.warning(f"Lasso guardrail detected violations: {violated_deputies}") + verbose_proxy_logger.warning("Lasso guardrail detected violations: %s", violated_deputies) # Check if any findings have "BLOCK" action blocking_violations = self._check_for_blocking_actions(response) @@ -866,7 +869,7 @@ class LassoGuardrail(CustomGuardrail): else: # Continue with warning for non-blocking violations (e.g., AUTO_MASKING) verbose_proxy_logger.info( - f"Non-blocking Lasso violations detected, continuing with warning: {violated_deputies}" + "Non-blocking Lasso violations detected, continuing with warning: %s", violated_deputies ) def _check_for_blocking_actions(self, response: LassoResponse) -> list[str]: @@ -955,7 +958,7 @@ class LassoGuardrail(CustomGuardrail): if msg.content and apply_text and text_cursor < len(masked_text): msg.content = masked_text[text_cursor] text_cursor += 1 - verbose_proxy_logger.debug(f"Applied masked text content to choice {text_cursor}") + verbose_proxy_logger.debug("Applied masked text content to choice %s", text_cursor) for call in getattr(msg, "tool_calls", None) or []: call_id = self._get_field(call, "id") @@ -970,7 +973,7 @@ class LassoGuardrail(CustomGuardrail): func = getattr(call, "function", None) if func: func.arguments = json.dumps(masked_input) - verbose_proxy_logger.debug(f"Applied masked tool_call arguments for call_id={call_id}") + verbose_proxy_logger.debug("Applied masked tool_call arguments for call_id=%s", call_id) @staticmethod def get_config_model() -> type["GuardrailConfigModel"] | None: diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py index c6900c38cbf..c36b5849fb6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py @@ -121,7 +121,7 @@ class CategoryConfig: try: self.phrase_patterns.append((p, re.compile(p, re.IGNORECASE))) except re.error: - verbose_proxy_logger.warning(f"Invalid phrase pattern in {category_name}: {p}") + verbose_proxy_logger.warning("Invalid phrase pattern in %s: %s", category_name, p) class ContentFilterGuardrail(CustomGuardrail): @@ -226,8 +226,8 @@ class ContentFilterGuardrail(CustomGuardrail): p["action"] == ContentFilterAction.MASK for p in self.compiled_patterns ): verbose_proxy_logger.warning( - f"ContentFilterGuardrail '{self.guardrail_name}': 'during_call' mode with 'MASK' action is unstable due to race conditions. " - "Use 'pre_call' mode for reliable request masking." + "ContentFilterGuardrail '%s': 'during_call' mode with 'MASK' action is unstable due to race conditions. Use 'pre_call' mode for reliable request masking.", + self.guardrail_name, ) # Load blocked words - always initialize as dict @@ -238,7 +238,7 @@ class ContentFilterGuardrail(CustomGuardrail): # Defensive check: ensure blocked_words is a dict (not a list) if not isinstance(self.blocked_words, dict): verbose_proxy_logger.error( - f"blocked_words is not a dict, got {type(self.blocked_words)}. Resetting to empty dict." + "blocked_words is not a dict, got %s. Resetting to empty dict.", type(self.blocked_words) ) self.blocked_words = {} @@ -247,11 +247,12 @@ class ContentFilterGuardrail(CustomGuardrail): self._load_blocked_words_file(blocked_words_file) verbose_proxy_logger.debug( - f"ContentFilterGuardrail initialized with {len(self.compiled_patterns)} patterns " - f"and {len(self.blocked_words)} blocked words" + "ContentFilterGuardrail initialized with %s patterns and %s blocked words", + len(self.compiled_patterns), + len(self.blocked_words), ) verbose_proxy_logger.debug( - f"Loaded {len(self.loaded_categories)} categories with {len(self.category_keywords)} keywords" + "Loaded %s categories with %s keywords", len(self.loaded_categories), len(self.category_keywords) ) def _init_competitor_intent_checker(self, competitor_intent_config: dict[str, Any]) -> None: @@ -406,7 +407,7 @@ class ContentFilterGuardrail(CustomGuardrail): # Prevent path traversal via category_name (e.g. "../../etc/passwd") if not re.match(r"^[a-zA-Z0-9_\-]+$", category_name): - verbose_proxy_logger.warning(f"Category name '{category_name}' contains invalid characters, skipping") + verbose_proxy_logger.warning("Category name '%s' contains invalid characters, skipping", category_name) continue enabled = cat_config.get("enabled", True) @@ -417,7 +418,7 @@ class ContentFilterGuardrail(CustomGuardrail): custom_file = cat_config.get("category_file") if not enabled: - verbose_proxy_logger.debug(f"Category {category_name} is disabled, skipping") + verbose_proxy_logger.debug("Category %s is disabled, skipping", category_name) continue # Load category file (custom or default) @@ -425,7 +426,9 @@ class ContentFilterGuardrail(CustomGuardrail): try: category_file_path = self._resolve_category_file_path(custom_file) except ValueError as e: - verbose_proxy_logger.warning(f"Category {category_name}: invalid category_file path, skipping. {e}") + verbose_proxy_logger.warning( + "Category %s: invalid category_file path, skipping. %s", category_name, e + ) continue else: # Try .yaml first, then .json (e.g. harm_toxic_abuse.json) @@ -439,7 +442,7 @@ class ContentFilterGuardrail(CustomGuardrail): category_file_path = yaml_path # will trigger "not found" below if not os.path.exists(category_file_path): - verbose_proxy_logger.warning(f"Category file not found: {category_file_path}, skipping") + verbose_proxy_logger.warning("Category file not found: %s, skipping", category_file_path) continue try: @@ -487,13 +490,14 @@ class ContentFilterGuardrail(CustomGuardrail): ) verbose_proxy_logger.info( - f"Loaded category {category_name}: " - f"{len(category_config_obj.keywords)} keywords, " - f"{len(category_config_obj.always_block_keywords)} always-block keywords, " - f"conditional: {bool(category_config_obj.identifier_words)}" + "Loaded category %s: %s keywords, %s always-block keywords, conditional: %s", + category_name, + len(category_config_obj.keywords), + len(category_config_obj.always_block_keywords), + bool(category_config_obj.identifier_words), ) except Exception as e: - verbose_proxy_logger.error(f"Error loading category {category_name}: {e}") + verbose_proxy_logger.error("Error loading category %s: %s", category_name, e) def _load_conditional_category( self, @@ -534,9 +538,12 @@ class ContentFilterGuardrail(CustomGuardrail): inherit_file_path = inherit_json_path else: verbose_proxy_logger.warning( - f"Category {category_name}: inherit_from '{inherit_from}' file not found at {categories_dir}" + "Category %s: inherit_from '%s' file not found at %s", + category_name, + inherit_from, + categories_dir, ) - verbose_proxy_logger.debug(f"Tried paths: {inherit_yaml_path}, {inherit_json_path}") + verbose_proxy_logger.debug("Tried paths: %s, %s", inherit_yaml_path, inherit_json_path) if inherit_file_path: # Load the inherited category @@ -583,7 +590,7 @@ class ContentFilterGuardrail(CustomGuardrail): verbose_proxy_logger.info(log_msg) except Exception as e: - verbose_proxy_logger.error(f"Error loading conditional category for {category_name}: {e}") + verbose_proxy_logger.error("Error loading conditional category for %s: %s", category_name, e) def _load_category_file(self, file_path: str) -> CategoryConfig: """ @@ -708,9 +715,9 @@ class ContentFilterGuardrail(CustomGuardrail): "allow_word_numbers": bool(extra_config.get("allow_word_numbers")), } ) - verbose_proxy_logger.debug(f"Added pattern: {pattern_name} with action {pattern_config.action}") + verbose_proxy_logger.debug("Added pattern: %s with action %s", pattern_name, pattern_config.action) except Exception as e: - verbose_proxy_logger.error(f"Error adding pattern {pattern_config}: {e}") + verbose_proxy_logger.error("Error adding pattern %s: %s", pattern_config, e) raise def _load_blocked_words_file(self, file_path: str) -> None: @@ -737,7 +744,7 @@ class ContentFilterGuardrail(CustomGuardrail): for word_data in data["blocked_words"]: if not isinstance(word_data, dict) or "keyword" not in word_data or "action" not in word_data: - verbose_proxy_logger.warning(f"Skipping invalid word entry: {word_data}") + verbose_proxy_logger.warning("Skipping invalid word entry: %s", word_data) continue keyword = word_data["keyword"].lower() @@ -746,7 +753,7 @@ class ContentFilterGuardrail(CustomGuardrail): self.blocked_words[keyword] = (action, description) - verbose_proxy_logger.info(f"Loaded {len(data['blocked_words'])} blocked words from {file_path}") + verbose_proxy_logger.info("Loaded %s blocked words from %s", len(data["blocked_words"]), file_path) except FileNotFoundError: raise FileNotFoundError(f"Blocked words file not found: {file_path}") except Exception as e: @@ -889,7 +896,7 @@ class ContentFilterGuardrail(CustomGuardrail): matched_text = text[start:end] pattern_name = pattern_entry["pattern_name"] action = pattern_entry["action"] - verbose_proxy_logger.debug(f"Pattern '{pattern_name}' matched: {matched_text[:20]}...") + verbose_proxy_logger.debug("Pattern '%s' matched: %s...", pattern_name, matched_text[:20]) return (matched_text, pattern_name, action) return None @@ -933,7 +940,7 @@ class ContentFilterGuardrail(CustomGuardrail): for exception in category_obj.exceptions: if exception in text_lower: verbose_proxy_logger.debug( - f"Category exception '{exception}' found for {category_name}, skipping" + "Category exception '%s' found for %s, skipping", exception, category_name ) exception_found = True break @@ -975,7 +982,7 @@ class ContentFilterGuardrail(CustomGuardrail): if block_word_found: matched_phrase = f"{identifier_found} + {block_word_found}" verbose_proxy_logger.warning( - f"Conditional match in {category_name}: '{matched_phrase}' in sentence" + "Conditional match in %s: '%s' in sentence", category_name, matched_phrase ) return (matched_phrase, category_name, severity, action) @@ -1020,7 +1027,7 @@ class ContentFilterGuardrail(CustomGuardrail): for pattern_str, pattern in config.phrase_patterns: if pattern.search(text): - verbose_proxy_logger.warning(f"Phrase pattern match in {category_name}: '{pattern_str}'") + verbose_proxy_logger.warning("Phrase pattern match in %s: '%s'", category_name, pattern_str) return ( f"phrase: {pattern_str}", category_name, @@ -1048,7 +1055,7 @@ class ContentFilterGuardrail(CustomGuardrail): # Check exceptions first — they take precedence over always-block keywords too. for exception in exceptions: if exception in text_lower: - verbose_proxy_logger.debug(f"Exception phrase '{exception}' found, skipping category keyword check") + verbose_proxy_logger.debug("Exception phrase '%s' found, skipping category keyword check", exception) return None # Always-block keywords are checked after exceptions. @@ -1064,7 +1071,7 @@ class ContentFilterGuardrail(CustomGuardrail): keyword_pattern = r"\b" + keyword_pattern_str + r"\b" keyword_found = bool(re.search(keyword_pattern, text_lower)) if keyword_found: - verbose_proxy_logger.debug(f"Always-block keyword '{keyword}' found in category '{category}'") + verbose_proxy_logger.debug("Always-block keyword '%s' found in category '%s'", keyword, category) return (keyword, category, severity, action) # Check category keywords @@ -1095,7 +1102,7 @@ class ContentFilterGuardrail(CustomGuardrail): for exception in category_obj.exceptions: if exception in text_lower: verbose_proxy_logger.debug( - f"Category exception '{exception}' found for keyword '{keyword}', skipping" + "Category exception '%s' found for keyword '%s', skipping", exception, keyword ) exception_found = True break @@ -1103,7 +1110,7 @@ class ContentFilterGuardrail(CustomGuardrail): continue verbose_proxy_logger.debug( - f"Category keyword '{keyword}' found in category '{category}' with severity {severity}" + "Category keyword '%s' found in category '%s' with severity %s", keyword, category, severity ) return (keyword, category, severity, action) return None @@ -1140,7 +1147,7 @@ class ContentFilterGuardrail(CustomGuardrail): text_lower = text.lower() for keyword, (action, description) in self.blocked_words.items(): if keyword in text_lower: - verbose_proxy_logger.debug(f"Blocked word '{keyword}' found with action {action}") + verbose_proxy_logger.debug("Blocked word '%s' found with action %s", keyword, action) return (keyword, action, description) return None @@ -1179,7 +1186,9 @@ class ContentFilterGuardrail(CustomGuardrail): ) elif action == ContentFilterAction.MASK: verbose_proxy_logger.warning( - f"Conditional match '{matched_phrase}' from {category_name} detected but MASK action not supported for conditional categories" + "Conditional match '%s' from %s detected but MASK action not supported for conditional categories", + matched_phrase, + category_name, ) def _handle_category_keyword_match( @@ -1223,7 +1232,7 @@ class ContentFilterGuardrail(CustomGuardrail): flags=re.IGNORECASE, ) verbose_proxy_logger.info( - f"Masked category keyword '{keyword}' from {category_name} (severity: {severity})" + "Masked category keyword '%s' from %s (severity: %s)", keyword, category_name, severity ) return text @@ -1255,7 +1264,7 @@ class ContentFilterGuardrail(CustomGuardrail): elif action == ContentFilterAction.MASK: redaction_tag = self.pattern_redaction_format.format(pattern_name=pattern_name.upper()) text = self._mask_spans(text, spans, redaction_tag) - verbose_proxy_logger.info(f"Masked all {pattern_name} matches in content") + verbose_proxy_logger.info("Masked all %s matches in content", pattern_name) return text @@ -1268,7 +1277,7 @@ class ContentFilterGuardrail(CustomGuardrail): detections: list[ContentFilterDetection] | None, ) -> str: """Handle blocked word match detection and action.""" - verbose_proxy_logger.debug(f"Blocked word '{keyword}' found with action {action}") + verbose_proxy_logger.debug("Blocked word '%s' found with action %s", keyword, action) if detections is not None: blocked_word_detection: BlockedWordDetection = { @@ -1300,7 +1309,7 @@ class ContentFilterGuardrail(CustomGuardrail): text, flags=re.IGNORECASE, ) - verbose_proxy_logger.info(f"Masked keyword '{keyword}' in content") + verbose_proxy_logger.info("Masked keyword '%s' in content", keyword) return text @@ -1427,14 +1436,14 @@ class ContentFilterGuardrail(CustomGuardrail): message = getattr(choice, "message", None) if message and getattr(message, "content", None): image_description = message.content - verbose_proxy_logger.debug(f"Image description: {image_description}") + verbose_proxy_logger.debug("Image description: %s", image_description) descriptions.append(image_description) else: verbose_proxy_logger.warning("No image description found") # Apply content filtering to image descriptions verbose_proxy_logger.debug( - f"ContentFilterGuardrail: Applying guardrail to {len(descriptions)} image description(s)" + "ContentFilterGuardrail: Applying guardrail to %s image description(s)", len(descriptions) ) for description in descriptions: # This will raise HTTPException if BLOCK action is triggered @@ -1780,7 +1789,7 @@ class ContentFilterGuardrail(CustomGuardrail): await self._process_images(images, detections) # Process texts - verbose_proxy_logger.debug(f"ContentFilterGuardrail: Applying guardrail to {len(texts)} text(s)") + verbose_proxy_logger.debug("ContentFilterGuardrail: Applying guardrail to %s text(s)", len(texts)) processed_texts = [] for text in texts: @@ -1852,7 +1861,7 @@ class ContentFilterGuardrail(CustomGuardrail): exception_str: str = "" verbose_proxy_logger.info( - f"ContentFilterGuardrail: Starting robust streaming masking for model {request_data.get('model')}" + "ContentFilterGuardrail: Starting robust streaming masking for model %s", request_data.get("model") ) try: @@ -1897,7 +1906,7 @@ class ContentFilterGuardrail(CustomGuardrail): latest_detections_by_choice[choice_index] = choice_detections raise except Exception as e: - verbose_proxy_logger.error(f"ContentFilterGuardrail: Error in masking: {e}") + verbose_proxy_logger.error("ContentFilterGuardrail: Error in masking: %s", e) masked_text = text_to_scan # Fallback to current text # Determine how much can be safely yielded diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py index 8292f575c74..6b1d7f6a93e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py @@ -168,7 +168,7 @@ def get_available_content_categories() -> list[dict[str, str]]: # Skip files that can't be loaded but log the error for debugging from litellm._logging import verbose_proxy_logger - verbose_proxy_logger.warning(f"Failed to load category file {filename}: {e}") + verbose_proxy_logger.warning("Failed to load category file %s: %s", filename, e) continue elif filename.endswith(".json"): # JSON category files (e.g. harm_toxic_abuse.json) - no YAML header, use filename diff --git a/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py index afbc67f2abb..725719048d7 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py @@ -216,7 +216,9 @@ class LLMAsAJudgeGuardrail(CustomGuardrail): try: judge_result = await self._run_judge(messages, response_text) except Exception as judge_err: - verbose_logger.warning(f"llm_as_a_judge guardrail: judge call failed, failing open. Error: {judge_err}") + verbose_logger.warning( + "llm_as_a_judge guardrail: judge call failed, failing open. Error: %s", judge_err + ) status = "guardrail_failed_to_respond" return inputs @@ -263,7 +265,7 @@ class LLMAsAJudgeGuardrail(CustomGuardrail): except HTTPException: raise except Exception as e: - verbose_logger.warning(f"llm_as_a_judge guardrail unexpected error: {e}") + verbose_logger.warning("llm_as_a_judge guardrail unexpected error: %s", e) return inputs finally: self.add_standard_logging_guardrail_information_to_request_data( diff --git a/litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission/mcp_end_user_permission.py b/litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission/mcp_end_user_permission.py index 1760d01e247..08bd79a4a0c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission/mcp_end_user_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission/mcp_end_user_permission.py @@ -80,7 +80,7 @@ class MCPEndUserPermissionGuardrail(CustomGuardrail): if allowed_mcp_servers is None: return inputs # No restrictions → pass through unchanged - verbose_proxy_logger.debug(f"MCP guardrail: end user restricted to MCP servers: {allowed_mcp_servers}") + verbose_proxy_logger.debug("MCP guardrail: end user restricted to MCP servers: %s", allowed_mcp_servers) filtered_tools = [] removed_tools = [] @@ -97,13 +97,14 @@ class MCPEndUserPermissionGuardrail(CustomGuardrail): else: removed_tools.append(tool_name) verbose_proxy_logger.warning( - f"MCP guardrail: removing tool '{tool_name}' " - f"(server: '{server_name}') — not in end user's allowed servers" + "MCP guardrail: removing tool '%s' (server: '%s') — not in end user's allowed servers", + tool_name, + server_name, ) if removed_tools: verbose_proxy_logger.debug( - f"MCP guardrail: removed {len(removed_tools)} unauthorized MCP tool(s): {removed_tools}" + "MCP guardrail: removed %s unauthorized MCP tool(s): %s", len(removed_tools), removed_tools ) inputs["tools"] = filtered_tools @@ -162,7 +163,7 @@ class MCPEndUserPermissionGuardrail(CustomGuardrail): route="/mcp", ) except Exception as e: - verbose_proxy_logger.warning(f"MCP guardrail: failed to fetch end_user_object for '{end_user_id}': {e}") + verbose_proxy_logger.warning("MCP guardrail: failed to fetch end_user_object for '%s': %s", end_user_id, e) return None # ------------------------------------------------------------------ diff --git a/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py b/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py index b2f91083cc0..a1980134166 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py +++ b/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py @@ -163,7 +163,7 @@ class NomaGuardrail(CustomGuardrail): try: asyncio.create_task(coro) except Exception as e: - verbose_proxy_logger.error(f"Failed to create background Noma task: {e}") + verbose_proxy_logger.error("Failed to create background Noma task: %s", e) async def _process_user_message_check( self, @@ -233,7 +233,7 @@ class NomaGuardrail(CustomGuardrail): if anonymized_content: # Replace the user message content with anonymized version self._replace_user_message_content(request_data, anonymized_content) - verbose_proxy_logger.debug(f"Noma guardrail anonymized user message: {anonymized_content}") + verbose_proxy_logger.debug("Noma guardrail anonymized user message: %s", anonymized_content) return anonymized_content await self._check_verdict(USER_ROLE, json.dumps(input_items), response_json) @@ -309,7 +309,7 @@ class NomaGuardrail(CustomGuardrail): if anonymized_content: # Replace the LLM response content with anonymized version self._replace_llm_response_content(response, anonymized_content) - verbose_proxy_logger.debug(f"Noma guardrail anonymized LLM response: {anonymized_content}") + verbose_proxy_logger.debug("Noma guardrail anonymized LLM response: %s", anonymized_content) return anonymized_content await self._check_verdict(ASSISTANT_ROLE, content, response_json) @@ -348,7 +348,7 @@ class NomaGuardrail(CustomGuardrail): return "guardrail_failed_to_respond" except Exception as e: - verbose_proxy_logger.error(f"Error determining NOMA guardrail status: {e}") + verbose_proxy_logger.error("Error determining NOMA guardrail status: %s", e) return "guardrail_failed_to_respond" def _should_only_sensitive_data_failed(self, classification_obj: dict) -> bool: @@ -513,7 +513,7 @@ class NomaGuardrail(CustomGuardrail): try: await self._process_user_message_check(request_data, user_auth) except Exception as e: - verbose_proxy_logger.error(f"Noma background user message check failed: {e}") + verbose_proxy_logger.error("Noma background user message check failed: %s", e) async def _check_llm_response_background( self, @@ -525,7 +525,7 @@ class NomaGuardrail(CustomGuardrail): try: await self._process_llm_response_check(request_data, response, user_auth) except Exception as e: - verbose_proxy_logger.error(f"Noma background response check failed: {e}") + verbose_proxy_logger.error("Noma background response check failed: %s", e) async def _handle_verdict_background( self, @@ -547,7 +547,7 @@ class NomaGuardrail(CustomGuardrail): msg = f"Noma guardrail allowed {type} message: {message}" verbose_proxy_logger.info(msg) except Exception as e: - verbose_proxy_logger.error(f"Noma background verdict handling failed: {e}") + verbose_proxy_logger.error("Noma background verdict handling failed: %s", e) async def async_pre_call_hook( self, @@ -570,7 +570,7 @@ class NomaGuardrail(CustomGuardrail): try: self._create_background_noma_check(self._check_user_message_background(data, user_api_key_dict)) except Exception as e: - verbose_proxy_logger.error(f"Failed to start background Noma pre-call check: {e}") + verbose_proxy_logger.error("Failed to start background Noma pre-call check: %s", e) return data try: @@ -594,7 +594,7 @@ class NomaGuardrail(CustomGuardrail): event_type=GuardrailEventHooks.pre_call, ) - verbose_proxy_logger.error(f"Noma pre-call hook failed: {e}") + verbose_proxy_logger.error("Noma pre-call hook failed: %s", e) if self.block_failures: raise @@ -618,7 +618,7 @@ class NomaGuardrail(CustomGuardrail): try: self._create_background_noma_check(self._check_user_message_background(data, user_api_key_dict)) except Exception as e: - verbose_proxy_logger.error(f"Failed to start background Noma moderation check: {e}") + verbose_proxy_logger.error("Failed to start background Noma moderation check: %s", e) return data try: @@ -642,7 +642,7 @@ class NomaGuardrail(CustomGuardrail): event_type=GuardrailEventHooks.during_call, ) - verbose_proxy_logger.error(f"Noma moderation hook failed: {e}") + verbose_proxy_logger.error("Noma moderation hook failed: %s", e) if self.block_failures: raise @@ -665,7 +665,7 @@ class NomaGuardrail(CustomGuardrail): self._check_llm_response_background(data, response, user_api_key_dict) ) except Exception as e: - verbose_proxy_logger.error(f"Failed to start background Noma post-call check: {e}") + verbose_proxy_logger.error("Failed to start background Noma post-call check: %s", e) return response try: @@ -689,7 +689,7 @@ class NomaGuardrail(CustomGuardrail): event_type=GuardrailEventHooks.post_call, ) - verbose_proxy_logger.error(f"Noma post-call hook failed: {e}") + verbose_proxy_logger.error("Noma post-call hook failed: %s", e) if self.block_failures: raise return response @@ -828,7 +828,7 @@ class NomaGuardrail(CustomGuardrail): except Exception as e: if self.block_failures: raise - verbose_proxy_logger.error(f"Noma streaming post-call hook failed: {e}") + verbose_proxy_logger.error("Noma streaming post-call hook failed: %s", e) for chunk in all_chunks: yield chunk return diff --git a/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py b/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py index 37ce84b8e6e..7f6787015f6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py +++ b/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py @@ -59,7 +59,7 @@ class OnyxGuardrail(CustomGuardrail): raise ValueError("ONYX_API_KEY environment variable is not set") self.optional_params = kwargs super().__init__(**kwargs) - verbose_proxy_logger.info(f"OnyxGuard initialized with server: {self.api_base}") + verbose_proxy_logger.info("OnyxGuard initialized with server: %s", self.api_base) async def _validate_with_guard_server( self, @@ -87,7 +87,7 @@ class OnyxGuardrail(CustomGuardrail): detection_message = "Unknown violation" if "violated_rules" in result: detection_message = ", ".join(result["violated_rules"]) - verbose_proxy_logger.warning(f"Request blocked by Onyx Guard. Violations: {detection_message}.") + verbose_proxy_logger.warning("Request blocked by Onyx Guard. Violations: %s.", detection_message) raise HTTPException( status_code=400, detail=f"Request blocked by Onyx Guard. Violations: {detection_message}.", @@ -118,7 +118,8 @@ class OnyxGuardrail(CustomGuardrail): payload = parsed.get("response", {}) except Exception as e: verbose_proxy_logger.error( - f"Error in converting request_data to ModelResponse: {e}", + "Error in converting request_data to ModelResponse: %s", + e, extra={ "conversation_id": conversation_id, "input_type": input_type, @@ -133,7 +134,8 @@ class OnyxGuardrail(CustomGuardrail): raise e except Exception as e: verbose_proxy_logger.error( - f"Error in apply_guardrail guard: {e}", + "Error in apply_guardrail guard: %s", + e, extra={"conversation_id": conversation_id, "input_type": input_type}, ) return inputs diff --git a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py index 5ff04864e75..31d777ff089 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py +++ b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py @@ -87,7 +87,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): ) verbose_proxy_logger.debug( - f"Initialized OpenAI Moderation Guardrail: {guardrail_name} with model: {self.model}" + "Initialized OpenAI Moderation Guardrail: %s with model: %s", guardrail_name, self.model ) def _get_api_key(self) -> str | None: diff --git a/litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py b/litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py index 5c707153873..09e24cca9d8 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py +++ b/litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py @@ -94,7 +94,10 @@ class PangeaHandler(CustomGuardrail): **kwargs, ) verbose_proxy_logger.debug( - f"Initialized Pangea Guardrail: name={guardrail_name}, recipe={pangea_input_recipe}, api_base={self.api_base}" + "Initialized Pangea Guardrail: name=%s, recipe=%s, api_base=%s", + guardrail_name, + pangea_input_recipe, + self.api_base, ) async def _call_pangea_ai_guard(self, api: str, payload: dict, hook_name: str) -> dict: @@ -125,7 +128,7 @@ class PangeaHandler(CustomGuardrail): } verbose_proxy_logger.debug( - f"Pangea Guardrail ({hook_name}): Calling endpoint {endpoint} with payload: {payload}" + "Pangea Guardrail (%s): Calling endpoint %s with payload: %s", hook_name, endpoint, payload ) response = await self.async_handler.post(url=endpoint, json=payload, headers=headers) @@ -134,7 +137,7 @@ class PangeaHandler(CustomGuardrail): result = response.json() if result.get("result", {}).get("blocked"): - verbose_proxy_logger.warning(f"Pangea Guardrail ({hook_name}): Request blocked. Response: {result}") + verbose_proxy_logger.warning("Pangea Guardrail (%s): Request blocked. Response: %s", hook_name, result) raise HTTPException( status_code=400, # Bad Request, indicating violation detail={ @@ -143,7 +146,7 @@ class PangeaHandler(CustomGuardrail): }, ) verbose_proxy_logger.debug( - f"Pangea Guardrail ({hook_name}): Request passed. Response: {result.get('result', {}).get('detectors')}" + "Pangea Guardrail (%s): Request passed. Response: %s", hook_name, result.get("result", {}).get("detectors") ) return result @@ -195,7 +198,7 @@ class PangeaHandler(CustomGuardrail): event_type = GuardrailEventHooks.pre_call if self.should_run_guardrail(data=data, event_type=event_type) is not True: verbose_proxy_logger.debug( - f"Pangea Guardrail (async_pre_call_hook): Guardrail is disabled {self.guardrail_name}." + "Pangea Guardrail (async_pre_call_hook): Guardrail is disabled %s.", self.guardrail_name ) return data @@ -286,7 +289,7 @@ class PangeaHandler(CustomGuardrail): event_type = GuardrailEventHooks.post_call if self.should_run_guardrail(data=data, event_type=event_type) is not True: verbose_proxy_logger.debug( - f"Pangea Guardrail (async_pre_call_hook): Guardrail is disabled {self.guardrail_name}." + "Pangea Guardrail (async_pre_call_hook): Guardrail is disabled %s.", self.guardrail_name ) return data try: diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index 782ffef61cf..64831c8161f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -122,10 +122,8 @@ class PanwPrismaAirsHandler(CustomGuardrail): # Warn if no profile is configured (user must have API key with linked profile) if not self.profile_name: verbose_proxy_logger.warning( - f"PANW Prisma AIRS Guardrail '{guardrail_name}': No profile_name configured. " - f"Ensure your API key has a linked profile in Strata Cloud Manager, " - f"or provide 'profile_name'/'profile_id' via config or per-request metadata. " - f"Requests will fail if the API key is not linked to a profile." + "PANW Prisma AIRS Guardrail '%s': No profile_name configured. Ensure your API key has a linked profile in Strata Cloud Manager, or provide 'profile_name'/'profile_id' via config or per-request metadata. Requests will fail if the API key is not linked to a profile.", + guardrail_name, ) self.fallback_on_error = fallback_on_error @@ -143,15 +141,18 @@ class PanwPrismaAirsHandler(CustomGuardrail): if self.fallback_on_error == "allow": verbose_proxy_logger.warning( - f"PANW Prisma AIRS Guardrail '{guardrail_name}': fallback_on_error='allow' - " - f"requests will proceed without scanning when API is unavailable." + "PANW Prisma AIRS Guardrail '%s': fallback_on_error='allow' - requests will proceed without scanning when API is unavailable.", + guardrail_name, ) verbose_proxy_logger.info( - f"Initialized PANW Prisma AIRS Guardrail: {guardrail_name} " - f"(profile={self.profile_name or 'API-key-linked'}, " - f"mask_request={self.mask_request_content}, mask_response={self.mask_response_content}, " - f"fallback_on_error={self.fallback_on_error}, timeout={self.timeout})" + "Initialized PANW Prisma AIRS Guardrail: %s (profile=%s, mask_request=%s, mask_response=%s, fallback_on_error=%s, timeout=%s)", + guardrail_name, + self.profile_name or "API-key-linked", + self.mask_request_content, + self.mask_response_content, + self.fallback_on_error, + self.timeout, ) # MCP event → base-call compatibility map. @@ -231,7 +232,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): return " ".join(text_parts) if text_parts else "" except (AttributeError, IndexError) as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS: Error extracting response text: {e}") + verbose_proxy_logger.error("PANW Prisma AIRS: Error extracting response text: %s", e) return "" async def _call_panw_api( @@ -355,7 +356,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): # Validate response format if "action" not in result: - verbose_proxy_logger.error(f"PANW Prisma AIRS: Invalid API response format: {result}") + verbose_proxy_logger.error("PANW Prisma AIRS: Invalid API response format: %s", result) return {"action": "block", "category": "api_error"} # Check for profile-related errors from PANW API @@ -365,14 +366,14 @@ class PanwPrismaAirsHandler(CustomGuardrail): "not found" in error_msg or "required" in error_msg or "invalid" in error_msg ): verbose_proxy_logger.error( - f"PANW Prisma AIRS: Profile configuration error. " - f"Ensure your API key has a linked profile in Strata Cloud Manager, " - f"or provide 'profile_name' or 'profile_id' in config/metadata. " - f"PANW API response: {result}" + "PANW Prisma AIRS: Profile configuration error. Ensure your API key has a linked profile in Strata Cloud Manager, or provide 'profile_name' or 'profile_id' in config/metadata. PANW API response: %s", + result, ) verbose_proxy_logger.debug( - f"PANW Prisma AIRS: Scan result - Action: {result.get('action')}, Category: {result.get('category', 'unknown')}" + "PANW Prisma AIRS: Scan result - Action: %s, Category: %s", + result.get("action"), + result.get("category", "unknown"), ) return result @@ -406,8 +407,8 @@ class PanwPrismaAirsHandler(CustomGuardrail): if status in (401, 403) or is_profile_error: verbose_proxy_logger.error( - f"PANW Prisma AIRS: Authentication/config error (HTTP {status}). " - f"Check API key and profile configuration." + "PANW Prisma AIRS: Authentication/config error (HTTP %s). Check API key and profile configuration.", + status, ) return { "action": "block", @@ -416,7 +417,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): } elif status == 429 or status >= 500: # Transient: rate-limit and server errors — safe to fail-open - verbose_proxy_logger.error(f"PANW Prisma AIRS: API error (HTTP {status}): {error_body[:500]}") + verbose_proxy_logger.error("PANW Prisma AIRS: API error (HTTP %s): %s", status, error_body[:500]) return { "action": "block", "category": f"http_{status}_error", @@ -425,7 +426,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): else: # Permanent 4xx client errors (400, 404, etc.) — must not bypass scanning if status != 400: # 400 already logged with diagnostics above - verbose_proxy_logger.error(f"PANW Prisma AIRS: API error (HTTP {status}): {error_body[:500]}") + verbose_proxy_logger.error("PANW Prisma AIRS: API error (HTTP %s): %s", status, error_body[:500]) return { "action": "block", "category": f"http_{status}_error", @@ -433,7 +434,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): } except httpx.TimeoutException as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS: Timeout error: {e}") + verbose_proxy_logger.error("PANW Prisma AIRS: Timeout error: %s", e) return { "action": "block", "category": "timeout_error", @@ -441,7 +442,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): } except httpx.RequestError as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS: Network/request error: {e}") + verbose_proxy_logger.error("PANW Prisma AIRS: Network/request error: %s", e) return { "action": "block", "category": "network_error", @@ -449,7 +450,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): } except Exception as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS: Unexpected error: {e}") + verbose_proxy_logger.error("PANW Prisma AIRS: Unexpected error: %s", e) return {"action": "block", "category": "api_error", "_is_transient": True} @staticmethod @@ -713,8 +714,9 @@ class PanwPrismaAirsHandler(CustomGuardrail): if scan_result.get("_is_transient") and self.fallback_on_error == "allow": verbose_proxy_logger.warning( - f"PANW Prisma AIRS: Allowing {'response' if is_response else 'request'} " - f"without scanning (fallback_on_error='allow', error: {category})" + "PANW Prisma AIRS: Allowing %s without scanning (fallback_on_error='allow', error: %s)", + "response" if is_response else "request", + category, ) add_guardrail_to_applied_guardrails_header( request_data=data, guardrail_name=f"{self.guardrail_name}:unscanned" @@ -915,7 +917,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): litellm_metadata = data.setdefault("litellm_metadata", {}) if litellm_metadata.get(scan_key): - verbose_proxy_logger.debug(f"PANW Prisma AIRS: Skipping duplicate {scan_type}-call scan") + verbose_proxy_logger.debug("PANW Prisma AIRS: Skipping duplicate %s-call scan", scan_type) return True # Already scanned litellm_metadata[scan_key] = True @@ -1030,9 +1032,9 @@ class PanwPrismaAirsHandler(CustomGuardrail): data["messages"] = self._apply_masking_to_messages(messages, masked_text) elif "prompt" in data: data["prompt"] = masked_text - verbose_proxy_logger.info(f"PANW Prisma AIRS: Prompt allowed with masking (Category: {category})") + verbose_proxy_logger.info("PANW Prisma AIRS: Prompt allowed with masking (Category: %s)", category) else: - verbose_proxy_logger.info(f"PANW Prisma AIRS: Prompt allowed (Category: {category})") + verbose_proxy_logger.info("PANW Prisma AIRS: Prompt allowed (Category: %s)", category) add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=self.guardrail_name) return None @@ -1050,13 +1052,13 @@ class PanwPrismaAirsHandler(CustomGuardrail): # Block the request error_detail = self._build_error_detail(scan_result, is_response=False) - verbose_proxy_logger.warning(f"PANW Prisma AIRS: {error_detail['error']['message']}") + verbose_proxy_logger.warning("PANW Prisma AIRS: %s", error_detail["error"]["message"]) raise HTTPException(status_code=400, detail=error_detail) except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS scan failed: {e}") + verbose_proxy_logger.error("PANW Prisma AIRS scan failed: %s", e) raise HTTPException( status_code=500, detail={ @@ -1147,9 +1149,11 @@ class PanwPrismaAirsHandler(CustomGuardrail): if action == "allow": if masked_text: self._apply_masking_to_response(response, masked_text) - verbose_proxy_logger.info(f"PANW Prisma AIRS: Response allowed with masking (Category: {category})") + verbose_proxy_logger.info( + "PANW Prisma AIRS: Response allowed with masking (Category: %s)", category + ) else: - verbose_proxy_logger.info(f"PANW Prisma AIRS: Response allowed (Category: {category})") + verbose_proxy_logger.info("PANW Prisma AIRS: Response allowed (Category: %s)", category) add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=self.guardrail_name) return response @@ -1164,13 +1168,13 @@ class PanwPrismaAirsHandler(CustomGuardrail): # Block the response error_detail = self._build_error_detail(scan_result, is_response=True) - verbose_proxy_logger.warning(f"PANW Prisma AIRS: {error_detail['error']['message']}") + verbose_proxy_logger.warning("PANW Prisma AIRS: %s", error_detail["error"]["message"]) raise HTTPException(status_code=400, detail=error_detail) except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS scan failed: {e}") + verbose_proxy_logger.error("PANW Prisma AIRS scan failed: %s", e) raise HTTPException( status_code=500, detail={ @@ -1229,10 +1233,10 @@ class PanwPrismaAirsHandler(CustomGuardrail): self._apply_masking_to_response(assembled_model_response, masked_text) content_was_modified = True verbose_proxy_logger.info( - f"PANW Prisma AIRS: Streaming response allowed with masking (Category: {category})" + "PANW Prisma AIRS: Streaming response allowed with masking (Category: %s)", category ) else: - verbose_proxy_logger.info(f"PANW Prisma AIRS: Streaming response allowed (Category: {category})") + verbose_proxy_logger.info("PANW Prisma AIRS: Streaming response allowed (Category: %s)", category) elif masked_text and self.mask_response_content: self._apply_masking_to_response(assembled_model_response, masked_text) content_was_modified = True @@ -1241,7 +1245,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): ) else: error_detail = self._build_error_detail(scan_result, is_response=True) - verbose_proxy_logger.warning(f"PANW Prisma AIRS: {error_detail['error']['message']}") + verbose_proxy_logger.warning("PANW Prisma AIRS: %s", error_detail["error"]["message"]) raise HTTPException(status_code=400, detail=error_detail) return content_was_modified, assembled_model_response, scan_result @@ -1366,7 +1370,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): error_obj["code"] = e.status_code yield f"data: {json.dumps({'error': error_obj})}\n\n" except Exception as e: - verbose_proxy_logger.error(f"PANW Prisma AIRS streaming error: {e}") + verbose_proxy_logger.error("PANW Prisma AIRS streaming error: %s", e) yield f"data: {json.dumps({'error': {'message': 'Security scan failed - streaming response blocked for safety', 'type': 'guardrail_scan_error', 'code': 500, 'guardrail': self.guardrail_name}})}\n\n" async def _scan_tool_calls_for_guardrail( diff --git a/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py b/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py index 77767c8c61b..058b0a2f23c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py +++ b/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py @@ -209,10 +209,10 @@ class PillarGuardrail(CustomGuardrail): self.on_flagged_action = action else: if action: - verbose_proxy_logger.warning(f"Invalid action '{action}', using default") + verbose_proxy_logger.warning("Invalid action '%s', using default", action) self.on_flagged_action = self.DEFAULT_ON_FLAGGED_ACTION - verbose_proxy_logger.debug(f"Pillar Guardrail: Initialized with on_flagged_action: {self.on_flagged_action}") + verbose_proxy_logger.debug("Pillar Guardrail: Initialized with on_flagged_action: %s", self.on_flagged_action) self.async_mode = self._resolve_bool_config( provided_value=async_mode, @@ -246,11 +246,11 @@ class PillarGuardrail(CustomGuardrail): else: if action: verbose_proxy_logger.warning( - f"Invalid fallback action '{action}', using default '{self.DEFAULT_FALLBACK_ACTION}'" + "Invalid fallback action '%s', using default '%s'", action, self.DEFAULT_FALLBACK_ACTION ) self.fallback_on_error = self.DEFAULT_FALLBACK_ACTION - verbose_proxy_logger.debug(f"Pillar Guardrail: Initialized with fallback_on_error: {self.fallback_on_error}") + verbose_proxy_logger.debug("Pillar Guardrail: Initialized with fallback_on_error: %s", self.fallback_on_error) # Set timeout with graceful fallback on invalid configuration if timeout is not None: @@ -260,8 +260,9 @@ class PillarGuardrail(CustomGuardrail): self.timeout = float(os.environ.get("PILLAR_TIMEOUT", str(self.DEFAULT_TIMEOUT))) except (ValueError, TypeError): verbose_proxy_logger.warning( - f"Pillar Guardrail: Invalid PILLAR_TIMEOUT value '{os.environ.get('PILLAR_TIMEOUT')}', " - f"falling back to default {self.DEFAULT_TIMEOUT}s" + "Pillar Guardrail: Invalid PILLAR_TIMEOUT value '%s', falling back to default %ss", + os.environ.get("PILLAR_TIMEOUT"), + self.DEFAULT_TIMEOUT, ) self.timeout = self.DEFAULT_TIMEOUT @@ -311,7 +312,7 @@ class PillarGuardrail(CustomGuardrail): """ event_type = GuardrailEventHooks.pre_call if self.should_run_guardrail(data=data, event_type=event_type) is not True: - verbose_proxy_logger.debug(f"Pillar Guardrail: Pre-call scanning disabled for {self.guardrail_name}") + verbose_proxy_logger.debug("Pillar Guardrail: Pre-call scanning disabled for %s", self.guardrail_name) return data verbose_proxy_logger.debug("Pillar Guardrail: Pre-call hook") @@ -354,7 +355,7 @@ class PillarGuardrail(CustomGuardrail): """ event_type = GuardrailEventHooks.during_call if self.should_run_guardrail(data=data, event_type=event_type) is not True: - verbose_proxy_logger.debug(f"Pillar Guardrail: During-call scanning disabled for {self.guardrail_name}") + verbose_proxy_logger.debug("Pillar Guardrail: During-call scanning disabled for %s", self.guardrail_name) return data verbose_proxy_logger.debug("Pillar Guardrail: During-call moderation hook") @@ -388,7 +389,7 @@ class PillarGuardrail(CustomGuardrail): """ event_type = GuardrailEventHooks.post_call if self.should_run_guardrail(data=data, event_type=event_type) is not True: - verbose_proxy_logger.debug(f"Pillar Guardrail: Post-call scanning disabled for {self.guardrail_name}") + verbose_proxy_logger.debug("Pillar Guardrail: Post-call scanning disabled for %s", self.guardrail_name) return response verbose_proxy_logger.debug("Pillar Guardrail: Post-call hook") @@ -457,7 +458,7 @@ class PillarGuardrail(CustomGuardrail): raise e # Handle API communication errors based on fallback_on_error setting - verbose_proxy_logger.error(f"Pillar Guardrail: API communication failed - {e}") + verbose_proxy_logger.error("Pillar Guardrail: API communication failed - %s", e) return self._handle_api_error(e, data) @@ -677,8 +678,11 @@ class PillarGuardrail(CustomGuardrail): payload["provider"] = provider verbose_proxy_logger.debug( - f"Pillar Guardrail: Request context - user={user_id}, session={session_id}, " - f"model={model}, provider={provider}" + "Pillar Guardrail: Request context - user=%s, session=%s, model=%s, provider=%s", + user_id, + session_id, + model, + provider, ) return payload @@ -694,7 +698,7 @@ class PillarGuardrail(CustomGuardrail): Pillar API response as dictionary """ verbose_proxy_logger.debug( - f"Pillar Guardrail: Scanning {len(payload.get('messages', []))} messages for security threats" + "Pillar Guardrail: Scanning %s messages for security threats", len(payload.get("messages", [])) ) response = await self.async_handler.post( url=f"{self.api_base}/api/v1/protect", @@ -707,7 +711,7 @@ class PillarGuardrail(CustomGuardrail): flagged = res.get("flagged") session_id = res.get("session_id") - verbose_proxy_logger.debug(f"Pillar Guardrail: Analysis complete - flagged={flagged}, session={session_id}") + verbose_proxy_logger.debug("Pillar Guardrail: Analysis complete - flagged=%s, session=%s", flagged, session_id) return res def _process_pillar_response(self, pillar_response: dict[str, Any], original_data: dict) -> None: @@ -739,7 +743,7 @@ class PillarGuardrail(CustomGuardrail): # Store session_id from Pillar response for potential reuse pillar_session_id = pillar_response.get("session_id") if pillar_session_id: - verbose_proxy_logger.debug(f"Pillar Guardrail: Received session_id from server: {pillar_session_id}") + verbose_proxy_logger.debug("Pillar Guardrail: Received session_id from server: %s", pillar_session_id) # Store in request metadata for use in subsequent hooks if "pillar_session_id" not in metadata_store: metadata_store["pillar_session_id"] = pillar_session_id diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 7a38c4087c6..56f08fe44ca 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -740,7 +740,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): elif isinstance(content, list) and content_idx_optional is not None: messages[msg_idx]["content"][content_idx_optional]["text"] = r - verbose_proxy_logger.debug(f"Presidio PII Masking: Redacted pii message: {data['messages']}") + verbose_proxy_logger.debug("Presidio PII Masking: Redacted pii message: %s", data["messages"]) data["messages"] = messages return data except Exception as e: @@ -832,7 +832,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): elif isinstance(content, list) and content_idx_optional is not None: messages[msg_idx]["content"][content_idx_optional]["text"] = r - verbose_proxy_logger.debug(f"Presidio PII Masking: Redacted pii message: {messages}") + verbose_proxy_logger.debug("Presidio PII Masking: Redacted pii message: %s", messages) kwargs["messages"] = messages return kwargs, result @@ -847,7 +847,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): Output parse the response object to replace the masked tokens with user sent values """ verbose_proxy_logger.debug( - f"PII Masking Args: self.output_parse_pii={self.output_parse_pii}; type of response={type(response)}" + "PII Masking Args: self.output_parse_pii=%s; type of response=%s", self.output_parse_pii, type(response) ) if self.apply_to_output is True: @@ -1124,7 +1124,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): yield mock_response_stream except Exception as e: - verbose_proxy_logger.error(f"Error masking streaming PII output: {e}") + verbose_proxy_logger.error("Error masking streaming PII output: %s", e) for chunk in all_chunks: yield chunk @@ -1253,7 +1253,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): yield mock_response_stream except Exception as e: - verbose_proxy_logger.error(f"Error in PII streaming processing: {e}") + verbose_proxy_logger.error("Error in PII streaming processing: %s", e) for chunk in remaining_chunks: yield chunk diff --git a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py index 0f3a817b12c..a816ef3846e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py +++ b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py @@ -326,7 +326,7 @@ class PromptSecurityGuardrail(CustomGuardrail): except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error processing image: {e}") + verbose_proxy_logger.error("Error processing image: %s", e) @staticmethod def _resolve_key_alias_from_request_data(request_data: dict) -> str | None: @@ -481,7 +481,7 @@ class PromptSecurityGuardrail(CustomGuardrail): except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error sanitizing image file: {e}") + verbose_proxy_logger.error("Error sanitizing image file: %s", e) raise HTTPException(status_code=500, detail=f"File sanitization failed: {e}") async def _process_document_item(self, item: dict, user_api_key_alias: str | None) -> dict: @@ -520,7 +520,7 @@ class PromptSecurityGuardrail(CustomGuardrail): extension = mime_type.split("/")[-1] filename = f"document.{extension}" - verbose_proxy_logger.info(f"Sanitizing document: {filename}") + verbose_proxy_logger.info("Sanitizing document: %s", filename) sanitization_result = await self.sanitize_file_content( file_data, filename, user_api_key_alias=user_api_key_alias @@ -554,7 +554,7 @@ class PromptSecurityGuardrail(CustomGuardrail): except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error sanitizing document: {e}") + verbose_proxy_logger.error("Error sanitizing document: %s", e) raise HTTPException(status_code=500, detail=f"Document sanitization failed: {e}") async def process_message_files(self, messages: list, user_api_key_alias: str | None = None) -> list: diff --git a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py index fe2cc40074f..c93cc3f36f3 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py +++ b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py @@ -344,7 +344,7 @@ class QualifireGuardrail(CustomGuardrail): ) url = f"{self.qualifire_api_base}/api/evaluation/evaluate" - verbose_proxy_logger.debug(f"Qualifire Guardrail: Making request to {url}") + verbose_proxy_logger.debug("Qualifire Guardrail: Making request to %s", url) # Make the API request response = await self.async_handler.post( @@ -373,8 +373,8 @@ class QualifireGuardrail(CustomGuardrail): if is_flagged: if on_flagged == "monitor": verbose_proxy_logger.warning( - "Qualifire Guardrail: Monitoring mode - violation detected but allowing request. " - f"Response: {qualifire_response}" + "Qualifire Guardrail: Monitoring mode - violation detected but allowing request. Response: %s", + qualifire_response, ) else: # Block the request @@ -389,7 +389,7 @@ class QualifireGuardrail(CustomGuardrail): except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Qualifire Guardrail error: {e}") + verbose_proxy_logger.exception("Qualifire Guardrail error: %s", e) raise @log_guardrail_information diff --git a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py index c30b2b0910c..056489de70f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py +++ b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py @@ -112,7 +112,7 @@ class SemanticGuardRouteLoader: ) ) - verbose_logger.info(f"SemanticGuard: built {len(routes)} routes") + verbose_logger.info("SemanticGuard: built %s routes", len(routes)) return routes @classmethod diff --git a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/semantic_guard.py b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/semantic_guard.py index f57827d03c9..1657485ed78 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/semantic_guard.py +++ b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/semantic_guard.py @@ -89,8 +89,11 @@ class SemanticGuardrail(CustomGuardrail): self.route_count = len(routes) verbose_logger.info( - f"SemanticGuardrail '{guardrail_name}' initialized with {self.route_count} routes, " - f"embedding_model={embedding_model}, threshold={similarity_threshold}" + "SemanticGuardrail '%s' initialized with %s routes, embedding_model=%s, threshold=%s", + guardrail_name, + self.route_count, + embedding_model, + similarity_threshold, ) @classmethod @@ -219,7 +222,7 @@ def _handle_match( } verbose_logger.warning( - f"SemanticGuard match: route={route_name}, score={similarity_score}, action={guardrail.on_flagged_action}" + "SemanticGuard match: route=%s, score=%s, action=%s", route_name, similarity_score, guardrail.on_flagged_action ) if guardrail.on_flagged_action == "passthrough": diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py index b15e5b61243..b3cfa0ab4d9 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -227,7 +227,7 @@ class ToolPermissionGuardrail(CustomGuardrail): Returns: Tuple of (is_allowed, rule_id, message) """ - verbose_proxy_logger.debug(f"Checking permission for tool: {tool_name or tool_type}") + verbose_proxy_logger.debug("Checking permission for tool: %s", tool_name or tool_type) # Check each rule in order for rule in self.rules: @@ -539,7 +539,7 @@ class ToolPermissionGuardrail(CustomGuardrail): if not denied_tool_names: return data - verbose_proxy_logger.info(f"Blocking {len(denied_tool_names)} unauthorized tool uses") + verbose_proxy_logger.info("Blocking %s unauthorized tool uses", len(denied_tool_names)) # Create a mapping of tool_use_id to error result error_tool_names = set() @@ -606,7 +606,7 @@ class ToolPermissionGuardrail(CustomGuardrail): if not denied_tools: return - verbose_proxy_logger.info(f"Blocking {len(denied_tools)} unauthorized tool uses") + verbose_proxy_logger.info("Blocking %s unauthorized tool uses", len(denied_tools)) # Create a mapping of tool_use_id to error result error_results = {} @@ -680,7 +680,7 @@ class ToolPermissionGuardrail(CustomGuardrail): is_allowed, _, message = self._check_tool_permission(tool_name, tool_type) if not is_allowed and message is not None: - verbose_proxy_logger.warning(f"Tool Permission Guardrail: {message}") + verbose_proxy_logger.warning("Tool Permission Guardrail: %s", message) if self.on_disallowed_action == "block": raise HTTPException( status_code=400, @@ -730,7 +730,7 @@ class ToolPermissionGuardrail(CustomGuardrail): verbose_proxy_logger.debug("Tool Permission Guardrail: No tool uses found") return response - verbose_proxy_logger.debug(f"Tool Permission Guardrail: Found {len(tool_calls)} tool calls") + verbose_proxy_logger.debug("Tool Permission Guardrail: Found %s tool calls", len(tool_calls)) # Check permissions for each tool use denied_tools = [] @@ -738,7 +738,7 @@ class ToolPermissionGuardrail(CustomGuardrail): is_allowed, rule_id, message = self._get_permission_for_tool_call(tool_call) if not is_allowed and message is not None: - verbose_proxy_logger.warning(f"Tool Permission Guardrail: {message}") + verbose_proxy_logger.warning("Tool Permission Guardrail: %s", message) if self.on_disallowed_action == "block": raise GuardrailRaisedException( @@ -809,7 +809,7 @@ class ToolPermissionGuardrail(CustomGuardrail): yield chunk return - verbose_proxy_logger.debug(f"Tool Permission Guardrail: Found {len(tool_calls)} tool calls") + verbose_proxy_logger.debug("Tool Permission Guardrail: Found %s tool calls", len(tool_calls)) # Check permissions for each tool use denied_tools = [] @@ -817,7 +817,7 @@ class ToolPermissionGuardrail(CustomGuardrail): is_allowed, rule_id, message = self._get_permission_for_tool_call(tool_call) if not is_allowed and message is not None: - verbose_proxy_logger.warning(f"Tool Permission Guardrail: {message}") + verbose_proxy_logger.warning("Tool Permission Guardrail: %s", message) if self.on_disallowed_action == "block": raise GuardrailRaisedException( diff --git a/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py b/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py index 9e16e9d5786..f28fd7975f2 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py +++ b/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py @@ -70,9 +70,10 @@ class ZscalerAIGuard(CustomGuardrail): ) verbose_proxy_logger.debug( - f"""send_user_api_key_alias: {self.send_user_api_key_alias}, - send_user_api_key_user_id:{self.send_user_api_key_user_id}, - send_user_api_key_team_id:{self.send_user_api_key_team_id}""" + "send_user_api_key_alias: %s, \n send_user_api_key_user_id:%s, \n send_user_api_key_team_id:%s", + self.send_user_api_key_alias, + self.send_user_api_key_user_id, + self.send_user_api_key_team_id, ) super().__init__(**kwargs) @@ -144,7 +145,7 @@ class ZscalerAIGuard(CustomGuardrail): texts = inputs.get("texts", []) try: - verbose_proxy_logger.debug(f"ZscalerAIGuard: Checking {len(texts)} text(s)") + verbose_proxy_logger.debug("ZscalerAIGuard: Checking %s text(s)", len(texts)) metadata = request_data.get("metadata", {}) user_api_key_metadata = metadata.get("user_api_key_metadata", {}) or {} @@ -166,7 +167,7 @@ class ZscalerAIGuard(CustomGuardrail): ) ) ) - verbose_proxy_logger.info(f"policy_id applied: {policy_id}") + verbose_proxy_logger.info("policy_id applied: %s", policy_id) kwargs = {} if self.send_user_api_key_alias: @@ -179,11 +180,11 @@ class ZscalerAIGuard(CustomGuardrail): kwargs["user_api_key_user_id"] = ( self._resolve_metadata_value(request_data, "user_api_key_user_id") or "N/A" ) - verbose_proxy_logger.debug(f"inside apply_guardrail kwargs: {kwargs}") + verbose_proxy_logger.debug("inside apply_guardrail kwargs: %s", kwargs) zscaler_ai_guard_result = None direction = "OUT" if input_type == "response" else "IN" - verbose_proxy_logger.debug(f"direction: {direction}") + verbose_proxy_logger.debug("direction: %s", direction) # Concatenate all texts and send to Zscaler AI Guard if texts: concatenated_text = " ".join(texts) @@ -195,7 +196,7 @@ class ZscalerAIGuard(CustomGuardrail): content=concatenated_text, **kwargs, ) - verbose_proxy_logger.debug(f"response from zscaler ai guards: {zscaler_ai_guard_result}") + verbose_proxy_logger.debug("response from zscaler ai guards: %s", zscaler_ai_guard_result) if zscaler_ai_guard_result and zscaler_ai_guard_result.get("action") == "BLOCK": blocking_info = zscaler_ai_guard_result.get("zscaler_ai_guard_response") error_message = f"Content blocked by Zscaler AI Guard: {self.extract_blocking_info(blocking_info)}" @@ -241,9 +242,9 @@ class ZscalerAIGuard(CustomGuardrail): } extra_headers = headers.copy() if self.send_user_api_key_alias: - verbose_proxy_logger.debug(f"kwargs: {kwargs}") + verbose_proxy_logger.debug("kwargs: %s", kwargs) user_api_key_alias = kwargs.get("user_api_key_alias", "N/A") - verbose_proxy_logger.debug(f"kwargs user_api_key_alias: {user_api_key_alias}") + verbose_proxy_logger.debug("kwargs user_api_key_alias: %s", user_api_key_alias) extra_headers.update({"user-api-key-alias": user_api_key_alias}) if self.send_user_api_key_team_id: @@ -254,7 +255,7 @@ class ZscalerAIGuard(CustomGuardrail): user_api_key_user_id = kwargs.get("user_api_key_user_id", "N/A") extra_headers.update({"user-api-key-user-id": user_api_key_user_id}) - verbose_proxy_logger.debug(f"extra_headers: {extra_headers}") + verbose_proxy_logger.debug("extra_headers: %s", extra_headers) return extra_headers async def _send_request(self, url, headers, data): @@ -279,7 +280,7 @@ class ZscalerAIGuard(CustomGuardrail): if response.status_code >= 500: # Server error verbose_proxy_logger.error( - f"Zscaler AI Guard service is unavailable (Status: {response.status_code}). Blocking request." + "Zscaler AI Guard service is unavailable (Status: %s). Blocking request.", response.status_code ) user_facing_error = self._create_user_facing_error(f"Service is unavailable (HTTP {response.status_code})") raise HTTPException(status_code=500, detail=user_facing_error) @@ -289,11 +290,11 @@ class ZscalerAIGuard(CustomGuardrail): statusCode_in_response = json_response.get("statusCode", None) if statusCode_in_response == 200: guardrail_result = json_response.get("action", None) - verbose_proxy_logger.info(f"Zscaler AI Guard response: {json_response}") + verbose_proxy_logger.info("Zscaler AI Guard response: %s", json_response) if guardrail_result == "BLOCK": verbose_proxy_logger.info( - f"Violated Zscaler AI Guard guardrail policy. zscaler_ai_guard_response: {json_response}" + "Violated Zscaler AI Guard guardrail policy. zscaler_ai_guard_response: %s", json_response ) return { "action": "BLOCK", @@ -301,7 +302,7 @@ class ZscalerAIGuard(CustomGuardrail): } elif guardrail_result == "ALLOW" or guardrail_result == "DETECT": verbose_proxy_logger.debug( - f"{direction} is allowed by Zscaler AI Guard. guardrail_result: {guardrail_result}" + "%s is allowed by Zscaler AI Guard. guardrail_result: %s", direction, guardrail_result ) return { "action": "ALLOW", @@ -310,7 +311,7 @@ class ZscalerAIGuard(CustomGuardrail): } else: verbose_proxy_logger.error( - f"Action field in response is {guardrail_result}, expecting 'ALLOW', 'BLOCK' or 'DETECT'" + "Action field in response is %s, expecting 'ALLOW', 'BLOCK' or 'DETECT'", guardrail_result ) user_facing_error = self._create_user_facing_error( f"Action field in response is {guardrail_result}, expecting 'ALLOW', 'BLOCK' or 'DETECT'" @@ -318,13 +319,13 @@ class ZscalerAIGuard(CustomGuardrail): raise HTTPException(status_code=500, detail=user_facing_error) else: errorMsg = json_response.get("errorMsg", None) - verbose_proxy_logger.error(f"statusCode in response: {statusCode_in_response}, errorMsg: {errorMsg}") + verbose_proxy_logger.error("statusCode in response: %s, errorMsg: %s", statusCode_in_response, errorMsg) user_facing_error = self._create_user_facing_error( f"statusCode in response: {statusCode_in_response}, errorMsg: {errorMsg}" ) raise HTTPException(status_code=500, detail=user_facing_error) else: - verbose_proxy_logger.error(f"Zscaler AI Guard status_code - {response.status_code}") + verbose_proxy_logger.error("Zscaler AI Guard status_code - %s", response.status_code) user_facing_error = self._create_user_facing_error(f"Response status code: {response.status_code}") raise HTTPException(status_code=response.status_code, detail=user_facing_error) @@ -350,7 +351,7 @@ class ZscalerAIGuard(CustomGuardrail): response = await self._send_request(zscaler_ai_guard_url, extra_headers, data) return self._handle_response(response, direction) except Exception as e: - verbose_proxy_logger.error(f"{e}. Blocking request.") + verbose_proxy_logger.error("%s. Blocking request.", e) user_facing_error = self._create_user_facing_error(f"{e}") raise HTTPException(status_code=500, detail=user_facing_error) diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index b0e16c0ed2e..bb8787ce72d 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -115,7 +115,7 @@ def get_guardrail_initializer_from_hooks(): module_path = f"litellm.proxy.guardrails.guardrail_hooks.{item}" try: # Import the module - verbose_proxy_logger.debug(f"Discovering guardrails in: {module_path}") + verbose_proxy_logger.debug("Discovering guardrails in: %s", module_path) module = importlib.import_module(module_path) @@ -125,7 +125,7 @@ def get_guardrail_initializer_from_hooks(): if isinstance(registry, dict): discovered_initializers.update(registry) verbose_proxy_logger.debug( - f"Found guardrail_initializer_registry in {module_path}: {list(registry.keys())}" + "Found guardrail_initializer_registry in %s: %s", module_path, list(registry.keys()) ) # Check for standalone initialize_guardrail function (fallback for directory-based guardrails) @@ -133,21 +133,23 @@ def get_guardrail_initializer_from_hooks(): # For directories with just initialize_guardrail, use the directory name as the key initialize_fn = getattr(module, "initialize_guardrail") discovered_initializers[item] = initialize_fn - verbose_proxy_logger.debug(f"Found initialize_guardrail function in {module_path}") + verbose_proxy_logger.debug("Found initialize_guardrail function in %s", module_path) except ImportError as e: - verbose_proxy_logger.error(f"Could not import {module_path}: {e}") + verbose_proxy_logger.error("Could not import %s: %s", module_path, e) continue except Exception as e: - verbose_proxy_logger.error(f"Error processing {module_path}: {e}") + verbose_proxy_logger.error("Error processing %s: %s", module_path, e) continue verbose_proxy_logger.debug( - f"Discovered {len(discovered_initializers)} guardrail initializers: {list(discovered_initializers.keys())}" + "Discovered %s guardrail initializers: %s", + len(discovered_initializers), + list(discovered_initializers.keys()), ) except Exception as e: - verbose_proxy_logger.error(f"Error discovering guardrail initializers: {e}") + verbose_proxy_logger.error("Error discovering guardrail initializers: %s", e) return discovered_initializers @@ -194,7 +196,7 @@ def get_guardrail_class_from_hooks(): try: # Import the module - verbose_proxy_logger.debug(f"Discovering guardrails in: {module_path}") + verbose_proxy_logger.debug("Discovering guardrails in: %s", module_path) module = importlib.import_module(module_path) @@ -205,14 +207,14 @@ def get_guardrail_class_from_hooks(): discovered_classes.update(registry) except ImportError as e: - verbose_proxy_logger.debug(f"Could not import {module_path}: {e}") + verbose_proxy_logger.debug("Could not import %s: %s", module_path, e) continue except Exception as e: - verbose_proxy_logger.exception(f"Error processing {module_path}: {e}") + verbose_proxy_logger.exception("Error processing %s: %s", module_path, e) continue except Exception as e: - verbose_proxy_logger.error(f"Error discovering guardrail initializers: {e}") + verbose_proxy_logger.error("Error discovering guardrail initializers: %s", e) return discovered_classes @@ -686,8 +688,8 @@ class InMemoryGuardrailHandler: return LitellmParams(**params).model_dump() except ValidationError as e: verbose_proxy_logger.warning( - f"Could not normalize guardrail litellm_params for comparison; " - f"treating the guardrail as changed. Error: {e}" + "Could not normalize guardrail litellm_params for comparison; treating the guardrail as changed. Error: %s", + e, ) return params return params @@ -723,7 +725,7 @@ class InMemoryGuardrailHandler: # Log differences if any found if changed_fields: - verbose_proxy_logger.debug(f"Guardrail params changed. Differences: {changed_fields}") + verbose_proxy_logger.debug("Guardrail params changed. Differences: %s", changed_fields) # Return True if any fields changed return len(changed_fields) > 0 @@ -763,7 +765,7 @@ class InMemoryGuardrailHandler: if self._has_guardrail_params_changed(guardrail_id, guardrail): guardrail_name = guardrail.get("guardrail_name", "Unknown") verbose_proxy_logger.info( - f"Guardrail '{guardrail_name}' (ID: {guardrail_id}) params changed, re-initializing..." + "Guardrail '%s' (ID: %s) params changed, re-initializing...", guardrail_name, guardrail_id ) return self.reinitialize_guardrail( guardrail=guardrail, diff --git a/litellm/proxy/guardrails/init_guardrails.py b/litellm/proxy/guardrails/init_guardrails.py index 036ee5dca78..183afb941cb 100644 --- a/litellm/proxy/guardrails/init_guardrails.py +++ b/litellm/proxy/guardrails/init_guardrails.py @@ -86,7 +86,7 @@ def _populate_router_guardrail_list(guardrail_list: list[Guardrail]) -> None: router_guardrail_list.append(router_guardrail) llm_router.guardrail_list = router_guardrail_list - verbose_proxy_logger.debug(f"Populated router guardrail_list with {len(router_guardrail_list)} guardrails") + verbose_proxy_logger.debug("Populated router guardrail_list with %s guardrails", len(router_guardrail_list)) ### LEGACY IMPLEMENTATION ### @@ -97,7 +97,7 @@ def initialize_guardrails( litellm_settings: dict, ) -> dict[str, GuardrailItem]: try: - verbose_proxy_logger.debug(f"validating guardrails passed {guardrails_config}") + verbose_proxy_logger.debug("validating guardrails passed %s", guardrails_config) global all_guardrails for item in guardrails_config: """ @@ -141,5 +141,5 @@ def initialize_guardrails( return litellm.guardrail_name_config_map except Exception as e: - verbose_proxy_logger.exception(f"error initializing guardrails {e}") + verbose_proxy_logger.exception("error initializing guardrails %s", e) raise e diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index c3bce5e8370..40b986f39e6 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -425,7 +425,7 @@ async def health_services_endpoint( } except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.health_services_endpoint(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.health_services_endpoint(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -504,7 +504,7 @@ async def _save_health_check_to_db( checked_by=user_id, ) except Exception as db_error: - verbose_proxy_logger.warning(f"Failed to save health check to database for model {model_name}: {db_error}") + verbose_proxy_logger.warning("Failed to save health check to database for model %s: %s", model_name, db_error) # Continue execution - don't let database save failure break health checks @@ -708,7 +708,7 @@ async def _save_background_health_checks_to_db( checked_by, ) except Exception as db_error: - verbose_proxy_logger.warning(f"Failed to save background health checks to database: {db_error}") + verbose_proxy_logger.warning("Failed to save background health checks to database: %s", db_error) # Continue execution - don't let database save failure break health checks @@ -882,7 +882,7 @@ def _health_endpoint_resolve_target_model_name( try: deployment = llm_router.get_deployment(model_id=model_id) except Exception as e: - verbose_proxy_logger.error(f"Error getting deployment for model_id {model_id}: {e}") + verbose_proxy_logger.error("Error getting deployment for model_id %s: %s", model_id, e) raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail={"error": f"Model with ID {model_id} not found"}, @@ -1069,7 +1069,7 @@ async def health_endpoint( ) return _post_process(router_result) except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.py::health_endpoint(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.py::health_endpoint(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) raise e @@ -1107,7 +1107,7 @@ async def health_check_history_endpoint( "offset": offset, } except Exception as e: - verbose_proxy_logger.error(f"Error getting health check history: {e}") + verbose_proxy_logger.error("Error getting health check history: %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to retrieve health check history: {e}"}, @@ -1139,7 +1139,7 @@ async def latest_health_checks_endpoint( "total_models": len(checks_data), } except Exception as e: - verbose_proxy_logger.error(f"Error getting latest health checks: {e}") + verbose_proxy_logger.error("Error getting latest health checks: %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to retrieve latest health checks: {e}"}, @@ -1182,7 +1182,7 @@ async def shared_health_check_status_endpoint( health_status = await shared_health_manager.get_health_check_status() return {"shared_health_check_enabled": True, "status": health_status} except Exception as e: - verbose_proxy_logger.error(f"Error getting shared health check status: {e}") + verbose_proxy_logger.error("Error getting shared health check status: %s", e) raise HTTPException( status_code=fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to retrieve shared health check status: {e}"}, @@ -1850,7 +1850,7 @@ async def test_model_connection( loaded_model_info = dict(deployments[0].get("model_info") or {}) except Exception as e: verbose_proxy_logger.debug( - f"Could not find model {model_name} in router: {e}. Proceeding with request params only." + "Could not find model %s in router: %s. Proceeding with request params only.", model_name, e ) # Merge: config params (from proxy config) as base, request params override @@ -1897,7 +1897,7 @@ async def test_model_connection( except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.debug(f"litellm.proxy.health_endpoints.test_model_connection(): Exception occurred - {e}") + verbose_proxy_logger.debug("litellm.proxy.health_endpoints.test_model_connection(): Exception occurred - %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to test connection: {e}"}, diff --git a/litellm/proxy/hooks/azure_content_safety.py b/litellm/proxy/hooks/azure_content_safety.py index 3c7713b2819..8195cc87a1f 100644 --- a/litellm/proxy/hooks/azure_content_safety.py +++ b/litellm/proxy/hooks/azure_content_safety.py @@ -123,7 +123,7 @@ class _PROXY_AzureContentSafety( raise e except Exception as e: verbose_proxy_logger.error( - f"litellm.proxy.hooks.azure_content_safety.py::async_pre_call_hook(): Exception occured - {e}" + "litellm.proxy.hooks.azure_content_safety.py::async_pre_call_hook(): Exception occured - %s", e ) verbose_proxy_logger.debug(traceback.format_exc()) diff --git a/litellm/proxy/hooks/batch_rate_limiter.py b/litellm/proxy/hooks/batch_rate_limiter.py index ce4ff2cb370..afaf0ebf392 100644 --- a/litellm/proxy/hooks/batch_rate_limiter.py +++ b/litellm/proxy/hooks/batch_rate_limiter.py @@ -256,7 +256,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): if skip_providers: batch_provider = self._resolve_batch_provider(self._get_batch_routing_model(data)) if batch_provider and batch_provider in skip_providers: - verbose_proxy_logger.debug(f"Skipping batch input file processing for provider={batch_provider}") + verbose_proxy_logger.debug("Skipping batch input file processing for provider=%s", batch_provider) return True, None descriptors = self._create_batch_rate_limit_descriptors( @@ -592,15 +592,15 @@ class _PROXY_BatchRateLimiter(CustomLogger): # in the access log instead of getting buried in error noise. if e.status_code == 403: verbose_proxy_logger.warning( - f"Batch rejected: caller not authorized for a model named in {file_id}: {e.detail}" + "Batch rejected: caller not authorized for a model named in %s: %s", file_id, e.detail ) else: verbose_proxy_logger.error( - f"Batch input file rejected for {file_id}: status={e.status_code} detail={e.detail}" + "Batch input file rejected for %s: status=%s detail=%s", file_id, e.status_code, e.detail ) raise except Exception as e: - verbose_proxy_logger.error(f"Error counting input file usage for {file_id}: {e}") + verbose_proxy_logger.error("Error counting input file usage for %s: %s", file_id, e) raise async def _enforce_batch_file_model_access( @@ -791,7 +791,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): # Only handle batch creation if call_type != "acreate_batch": verbose_proxy_logger.debug( - f"Batch rate limiter: Not handling batch creation rate limiting for call type: {call_type}" + "Batch rate limiter: Not handling batch creation rate limiting for call type: %s", call_type ) return data @@ -814,7 +814,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): custom_llm_provider = data.get("custom_llm_provider", "openai") # Count tokens and requests from input file - verbose_proxy_logger.debug(f"Counting tokens from batch input file: {input_file_id}") + verbose_proxy_logger.debug("Counting tokens from batch input file: %s", input_file_id) batch_usage = await self.count_input_file_usage( file_id=input_file_id, custom_llm_provider=custom_llm_provider, @@ -823,7 +823,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): ) verbose_proxy_logger.debug( - f"Batch input file usage - Tokens: {batch_usage.total_tokens}, Requests: {batch_usage.request_count}" + "Batch input file usage - Tokens: %s, Requests: %s", batch_usage.total_tokens, batch_usage.request_count ) # Store batch usage in data for later reference @@ -846,6 +846,6 @@ class _PROXY_BatchRateLimiter(CustomLogger): # Re-raise HTTP exceptions (rate limit exceeded) raise except Exception as e: - verbose_proxy_logger.error(f"Error in batch rate limiting: {e}", exc_info=True) + verbose_proxy_logger.error("Error in batch rate limiting: %s", e, exc_info=True) # Don't block the request if rate limiting fails return data diff --git a/litellm/proxy/hooks/batch_redis_get.py b/litellm/proxy/hooks/batch_redis_get.py index 377cd8d3d45..e2e7c1a3d27 100644 --- a/litellm/proxy/hooks/batch_redis_get.py +++ b/litellm/proxy/hooks/batch_redis_get.py @@ -84,7 +84,7 @@ class _PROXY_BatchRedisRequests(CustomLogger): raise e except Exception as e: verbose_proxy_logger.error( - f"litellm.proxy.hooks.batch_redis_get.py::async_pre_call_hook(): Exception occured - {e}" + "litellm.proxy.hooks.batch_redis_get.py::async_pre_call_hook(): Exception occured - %s", e ) verbose_proxy_logger.debug(traceback.format_exc()) diff --git a/litellm/proxy/hooks/cache_control_check.py b/litellm/proxy/hooks/cache_control_check.py index f2a0f06b95b..7f486200c2d 100644 --- a/litellm/proxy/hooks/cache_control_check.py +++ b/litellm/proxy/hooks/cache_control_check.py @@ -52,5 +52,5 @@ class _PROXY_CacheControlCheck(CustomLogger): raise e except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.cache_control_check.py::async_pre_call_hook(): Exception occured - {e}" + "litellm.proxy.hooks.cache_control_check.py::async_pre_call_hook(): Exception occured - %s", e ) diff --git a/litellm/proxy/hooks/dynamic_rate_limiter.py b/litellm/proxy/hooks/dynamic_rate_limiter.py index 5d890b6787c..c0adcfd1b75 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter.py @@ -68,7 +68,7 @@ class DynamicRateLimiterCache: await self.cache.async_set_cache_sadd(key=key_name, value=value, ttl=self.ttl) except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.hooks.dynamic_rate_limiter.py::async_set_cache_sadd(): Exception occured - {e}" + "litellm.proxy.hooks.dynamic_rate_limiter.py::async_set_cache_sadd(): Exception occured - %s", e ) raise e @@ -106,7 +106,9 @@ class _PROXY_DynamicRateLimitHandler(CustomLogger): weight: float = 1 if litellm.priority_reservation is None or priority not in litellm.priority_reservation: verbose_proxy_logger.error( - f"Priority Reservation not set. priority={priority}, but litellm.priority_reservation is {litellm.priority_reservation}." + "Priority Reservation not set. priority=%s, but litellm.priority_reservation is %s.", + priority, + litellm.priority_reservation, ) elif priority is not None and litellm.priority_reservation is not None: if os.getenv("LITELLM_LICENSE", None) is None: @@ -172,7 +174,7 @@ class _PROXY_DynamicRateLimitHandler(CustomLogger): ) except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.hooks.dynamic_rate_limiter.py::check_available_usage: Exception occurred - {e}" + "litellm.proxy.hooks.dynamic_rate_limiter.py::check_available_usage: Exception occurred - %s", e ) return None, None, None, None, None @@ -263,6 +265,6 @@ class _PROXY_DynamicRateLimitHandler(CustomLogger): ) except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.hooks.dynamic_rate_limiter.py::async_post_call_success_hook(): Exception occured - {e}" + "litellm.proxy.hooks.dynamic_rate_limiter.py::async_post_call_success_hook(): Exception occured - %s", e ) return response diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index 773abed1785..326a8e01407 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -173,7 +173,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): if total_weight > 1.0: normalized = {k: v / total_weight for k, v in weights.items()} - verbose_proxy_logger.debug(f"Normalized over-allocated priorities: {weights} -> {normalized}") + verbose_proxy_logger.debug("Normalized over-allocated priorities: %s -> %s", weights, normalized) return normalized return weights @@ -282,7 +282,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): return max_saturation except Exception as e: - verbose_proxy_logger.error(f"Error checking saturation for {model}: {e}") + verbose_proxy_logger.error("Error checking saturation for %s: %s", model, e) # Fail open: assume not saturated on error return 0.0 @@ -454,7 +454,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): parent_otel_span=user_api_key_dict.parent_otel_span, ) - verbose_proxy_logger.debug(f"Atomic check+increment response: {json.dumps(atomic_response, indent=2)}") + verbose_proxy_logger.debug("Atomic check+increment response: %s", json.dumps(atomic_response, indent=2)) if atomic_response["overall_code"] == "OVER_LIMIT": resolved_model, llm_provider = resolve_llm_provider_for_rate_limit(model) @@ -518,8 +518,8 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): None, ) verbose_proxy_logger.error( - f"Dynamic rate limiter: OVER_LIMIT response with unknown " - f"descriptor_key(s) — refusing request. response={atomic_response}" + "Dynamic rate limiter: OVER_LIMIT response with unknown descriptor_key(s) — refusing request. response=%s", + atomic_response, ) raise ProxyRateLimitError( detail={ @@ -610,7 +610,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): # Get model configuration model_group_info: ModelGroupInfo | None = self.llm_router.get_model_group_info(model_group=model) if model_group_info is None: - verbose_proxy_logger.debug(f"No model group info for {model}, allowing request") + verbose_proxy_logger.debug("No model group info for %s, allowing request", model) return None try: @@ -640,7 +640,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error in dynamic rate limiter: {e}, allowing request") + verbose_proxy_logger.error("Error in dynamic rate limiter: %s, allowing request", e) # Fail open on unexpected errors return None @@ -676,7 +676,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): return response except Exception as e: - verbose_proxy_logger.exception(f"Error in dynamic rate limiter v3 post-call hook: {e}") + verbose_proxy_logger.exception("Error in dynamic rate limiter v3 post-call hook: %s", e) return response async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -786,9 +786,11 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): SAFE_PRIORITIES = {"low", "medium", "high", "default"} logged_priority = key_priority if key_priority in SAFE_PRIORITIES else "REDACTED" verbose_proxy_logger.debug( - f"[Dynamic Rate Limiter] Incremented tokens by {total_tokens} for " - f"model={model_group}, priority={logged_priority}" + "[Dynamic Rate Limiter] Incremented tokens by %s for model=%s, priority=%s", + total_tokens, + model_group, + logged_priority, ) except Exception as e: - verbose_proxy_logger.exception(f"Error in dynamic rate limiter success event: {e}") + verbose_proxy_logger.exception("Error in dynamic rate limiter success event: %s", e) diff --git a/litellm/proxy/hooks/key_management_event_hooks.py b/litellm/proxy/hooks/key_management_event_hooks.py index 7ef1341a168..800d4874a98 100644 --- a/litellm/proxy/hooks/key_management_event_hooks.py +++ b/litellm/proxy/hooks/key_management_event_hooks.py @@ -51,7 +51,7 @@ class KeyManagementEventHooks: try: await KeyManagementEventHooks._send_key_created_email(response.model_dump(exclude_none=True)) except Exception as e: - verbose_proxy_logger.warning(f"Failed to send key created email: {e}") + verbose_proxy_logger.warning("Failed to send key created email: %s", e) # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True if litellm.store_audit_logs is True: @@ -84,7 +84,7 @@ class KeyManagementEventHooks: team_id=data.team_id, ) except Exception as e: - verbose_proxy_logger.warning(f"Failed to store virtual key in secret manager: {e}") + verbose_proxy_logger.warning("Failed to store virtual key in secret manager: %s", e) @staticmethod async def async_key_updated_hook( @@ -168,7 +168,7 @@ class KeyManagementEventHooks: new_secret_name, ) except Exception as e: - verbose_proxy_logger.warning(f"Failed to rotate virtual key in secret manager: {e}") + verbose_proxy_logger.warning("Failed to rotate virtual key in secret manager: %s", e) # Send key rotated email if configured - non-blocking, independent operation try: @@ -177,7 +177,7 @@ class KeyManagementEventHooks: existing_key_alias=existing_key_row.key_alias, ) except Exception as e: - verbose_proxy_logger.warning(f"Failed to send key rotated email: {e}") + verbose_proxy_logger.warning("Failed to send key rotated email: %s", e) # store the audit log if litellm.store_audit_logs is True and existing_key_row.token is not None: @@ -273,7 +273,7 @@ class KeyManagementEventHooks: description = getattr(litellm._key_management_settings, "description", None) optional_params = await KeyManagementEventHooks._get_secret_manager_optional_params(team_id) verbose_proxy_logger.debug( - f"Creating secret with {secret_name} and tags={tags} and description={description}" + "Creating secret with %s and tags=%s and description=%s", secret_name, tags, description ) await litellm.secret_manager_client.async_write_secret( @@ -355,7 +355,8 @@ class KeyManagementEventHooks: ) else: verbose_proxy_logger.warning( - f"KeyManagementEventHooks._delete_virtual_key_from_secret_manager: Key alias not found for key {key.token}. Skipping deletion from secret manager." + "KeyManagementEventHooks._delete_virtual_key_from_secret_manager: Key alias not found for key %s. Skipping deletion from secret manager.", + key.token, ) @staticmethod @@ -385,7 +386,7 @@ class KeyManagementEventHooks: user_api_key_cache=user_api_key_cache, ) except Exception as exc: # pragma: no cover - defensive logging - verbose_proxy_logger.debug(f"Unable to load team metadata for team_id={team_id}: {exc}") + verbose_proxy_logger.debug("Unable to load team metadata for team_id=%s: %s", team_id, exc) return None metadata = getattr(team_obj, "metadata", None) diff --git a/litellm/proxy/hooks/litellm_skills/main.py b/litellm/proxy/hooks/litellm_skills/main.py index 983a59657ce..770578d988b 100644 --- a/litellm/proxy/hooks/litellm_skills/main.py +++ b/litellm/proxy/hooks/litellm_skills/main.py @@ -96,7 +96,7 @@ class SkillsInjectionHook(CustomLogger): if not skills or not isinstance(skills, list): return data - verbose_proxy_logger.debug(f"SkillsInjectionHook: Processing {len(skills)} skills") + verbose_proxy_logger.debug("SkillsInjectionHook: Processing %s skills", len(skills)) litellm_skills: list[LiteLLM_SkillsTable] = [] anthropic_skills: list[dict[str, Any]] = [] @@ -116,7 +116,7 @@ class SkillsInjectionHook(CustomLogger): if db_skill: litellm_skills.append(db_skill) else: - verbose_proxy_logger.warning(f"SkillsInjectionHook: Skill '{skill_id}' not found in LiteLLM DB") + verbose_proxy_logger.warning("SkillsInjectionHook: Skill '%s' not found in LiteLLM DB", skill_id) else: # Native Anthropic skill - pass through anthropic_skills.append(skill) @@ -198,9 +198,10 @@ class SkillsInjectionHook(CustomLogger): data.pop("container", None) verbose_proxy_logger.debug( - f"SkillsInjectionHook: Messages API - converted {len(litellm_skills)} skills to Anthropic tools, " - f"injected {len(skill_contents)} skill contents, " - f"added litellm_code_execution tool with {len(all_module_paths)} modules" + "SkillsInjectionHook: Messages API - converted %s skills to Anthropic tools, injected %s skill contents, added litellm_code_execution tool with %s modules", + len(litellm_skills), + len(skill_contents), + len(all_module_paths), ) return data @@ -266,9 +267,10 @@ class SkillsInjectionHook(CustomLogger): data.pop("container", None) verbose_proxy_logger.debug( - f"SkillsInjectionHook: Non-Anthropic model - converted {len(litellm_skills)} skills to tools, " - f"injected {len(skill_contents)} skill contents, " - f"added execute_code tool with {len(all_module_paths)} modules" + "SkillsInjectionHook: Non-Anthropic model - converted %s skills to tools, injected %s skill contents, added execute_code tool with %s modules", + len(litellm_skills), + len(skill_contents), + len(all_module_paths), ) return data @@ -295,7 +297,7 @@ class SkillsInjectionHook(CustomLogger): user_api_key_dict=user_api_key_dict, ) except Exception as e: - verbose_proxy_logger.warning(f"SkillsInjectionHook: Error fetching skill {skill_id}: {e}") + verbose_proxy_logger.warning("SkillsInjectionHook: Error fetching skill %s: %s", skill_id, e) return None def _is_anthropic_model(self, model: str) -> bool: @@ -500,8 +502,9 @@ class SkillsInjectionHook(CustomLogger): # Check if we're done (no tool calls) if stop_reason != "tool_use" or not tool_calls: verbose_proxy_logger.debug( - f"SkillsInjectionHook: Loop completed after {iteration + 1} iterations, " - f"{len(generated_files)} files generated" + "SkillsInjectionHook: Loop completed after %s iterations, %s files generated", + iteration + 1, + len(generated_files), ) return self._attach_files_to_response(current_response, generated_files) @@ -536,7 +539,7 @@ class SkillsInjectionHook(CustomLogger): messages.append({"role": "user", "content": tool_results}) # Make next LLM call - verbose_proxy_logger.debug(f"SkillsInjectionHook: Making LLM call iteration {iteration + 2}") + verbose_proxy_logger.debug("SkillsInjectionHook: Making LLM call iteration %s", iteration + 2) try: current_response = await litellm.anthropic.acreate( model=model, @@ -548,10 +551,10 @@ class SkillsInjectionHook(CustomLogger): verbose_proxy_logger.error("SkillsInjectionHook: LLM call returned None") return self._attach_files_to_response(response, generated_files) except Exception as e: - verbose_proxy_logger.error(f"SkillsInjectionHook: LLM call failed: {e}") + verbose_proxy_logger.error("SkillsInjectionHook: LLM call failed: %s", e) return self._attach_files_to_response(response, generated_files) - verbose_proxy_logger.warning(f"SkillsInjectionHook: Max iterations ({self.max_iterations}) reached") + verbose_proxy_logger.warning("SkillsInjectionHook: Max iterations (%s) reached", self.max_iterations) return self._attach_files_to_response(current_response, generated_files) async def _execute_code( @@ -563,7 +566,7 @@ class SkillsInjectionHook(CustomLogger): ) -> str: """Execute code in sandbox and return result string.""" try: - verbose_proxy_logger.debug(f"SkillsInjectionHook: Executing code ({len(code)} chars)") + verbose_proxy_logger.debug("SkillsInjectionHook: Executing code (%s chars)", len(code)) exec_result = executor.execute(code=code, skill_files=skill_files) @@ -731,8 +734,9 @@ print('No executable skill module found') # Check if we're done (no tool calls) if stop_reason != "tool_calls" or not assistant_message.tool_calls: verbose_proxy_logger.debug( - f"SkillsInjectionHook: Code execution loop completed after " - f"{iteration + 1} iterations, {len(generated_files)} files generated" + "SkillsInjectionHook: Code execution loop completed after %s iterations, %s files generated", + iteration + 1, + len(generated_files), ) # Attach generated files to response return self._attach_files_to_response(current_response, generated_files) @@ -761,7 +765,7 @@ print('No executable skill module found') ) # Make next LLM call using the messages API - verbose_proxy_logger.debug(f"SkillsInjectionHook: Making LLM call iteration {iteration + 2}") + verbose_proxy_logger.debug("SkillsInjectionHook: Making LLM call iteration %s", iteration + 2) current_response = await litellm.anthropic.acreate( model=model, messages=messages, @@ -770,7 +774,7 @@ print('No executable skill module found') ) # Max iterations reached - verbose_proxy_logger.warning(f"SkillsInjectionHook: Max iterations ({self.max_iterations}) reached") + verbose_proxy_logger.warning("SkillsInjectionHook: Max iterations (%s) reached", self.max_iterations) return self._attach_files_to_response(current_response, generated_files) async def _execute_code_tool( @@ -785,7 +789,7 @@ print('No executable skill module found') args = json.loads(tool_call.function.arguments) code = args.get("code", "") - verbose_proxy_logger.debug(f"SkillsInjectionHook: Executing code ({len(code)} chars)") + verbose_proxy_logger.debug("SkillsInjectionHook: Executing code (%s chars)", len(code)) exec_result = executor.execute( code=code, @@ -811,7 +815,7 @@ print('No executable skill module found') tool_result += f"\n- {f['name']} ({len(file_content)} bytes)" verbose_proxy_logger.debug( - f"SkillsInjectionHook: Generated file {f['name']} ({len(file_content)} bytes)" + "SkillsInjectionHook: Generated file %s (%s bytes)", f["name"], len(file_content) ) if exec_result.get("error"): @@ -820,7 +824,7 @@ print('No executable skill module found') return tool_result except Exception as e: - verbose_proxy_logger.error(f"SkillsInjectionHook: Code execution failed: {e}") + verbose_proxy_logger.error("SkillsInjectionHook: Code execution failed: %s", e) return f"Code execution failed: {e}" def _attach_files_to_response( @@ -840,7 +844,7 @@ print('No executable skill module found') # Handle dict response (Anthropic/messages API format) if isinstance(response, dict): response["_litellm_generated_files"] = generated_files - verbose_proxy_logger.debug(f"SkillsInjectionHook: Attached {len(generated_files)} files to dict response") + verbose_proxy_logger.debug("SkillsInjectionHook: Attached %s files to dict response", len(generated_files)) return response # Handle object response (OpenAI format) @@ -855,7 +859,7 @@ print('No executable skill module found') response.model_extra = {} response.model_extra["_litellm_generated_files"] = generated_files - verbose_proxy_logger.debug(f"SkillsInjectionHook: Attached {len(generated_files)} files to response") + verbose_proxy_logger.debug("SkillsInjectionHook: Attached %s files to response", len(generated_files)) return response diff --git a/litellm/proxy/hooks/max_budget_limiter.py b/litellm/proxy/hooks/max_budget_limiter.py index 4a768b4e7de..7790dd8e175 100644 --- a/litellm/proxy/hooks/max_budget_limiter.py +++ b/litellm/proxy/hooks/max_budget_limiter.py @@ -75,5 +75,5 @@ class _PROXY_MaxBudgetLimiter(CustomLogger): raise e except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.max_budget_limiter.py::async_pre_call_hook(): Exception occured - {e}" + "litellm.proxy.hooks.max_budget_limiter.py::async_pre_call_hook(): Exception occured - %s", e ) diff --git a/litellm/proxy/hooks/mcp_semantic_filter/hook.py b/litellm/proxy/hooks/mcp_semantic_filter/hook.py index 10293bc5e5f..0356cbfa702 100644 --- a/litellm/proxy/hooks/mcp_semantic_filter/hook.py +++ b/litellm/proxy/hooks/mcp_semantic_filter/hook.py @@ -62,8 +62,9 @@ class SemanticToolFilterHook(CustomLogger): self.filter = semantic_filter verbose_proxy_logger.debug( - f"Initialized SemanticToolFilterHook with filter: " - f"enabled={semantic_filter.enabled}, top_k={semantic_filter.top_k}" + "Initialized SemanticToolFilterHook with filter: enabled=%s, top_k=%s", + semantic_filter.enabled, + semantic_filter.top_k, ) def _should_expand_mcp_tools(self, tools: list[Any]) -> bool: @@ -114,22 +115,24 @@ class SemanticToolFilterHook(CustomLogger): if hasattr(tool, "model_dump"): tool_dict = tool.model_dump(exclude_none=True) verbose_proxy_logger.debug( - f"Converted Pydantic tool to dict: {type(tool).__name__} -> dict with keys: {list(tool_dict.keys())}" + "Converted Pydantic tool to dict: %s -> dict with keys: %s", + type(tool).__name__, + list(tool_dict.keys()), ) openai_tools_as_dicts.append(tool_dict) elif hasattr(tool, "dict"): tool_dict = tool.dict(exclude_none=True) - verbose_proxy_logger.debug(f"Converted Pydantic tool (v1) to dict: {type(tool).__name__} -> dict") + verbose_proxy_logger.debug("Converted Pydantic tool (v1) to dict: %s -> dict", type(tool).__name__) openai_tools_as_dicts.append(tool_dict) elif isinstance(tool, dict): - verbose_proxy_logger.debug(f"Tool is already a dict with keys: {list(tool.keys())}") + verbose_proxy_logger.debug("Tool is already a dict with keys: %s", list(tool.keys())) openai_tools_as_dicts.append(tool) else: - verbose_proxy_logger.warning(f"Tool is unknown type: {type(tool)}, passing as-is") + verbose_proxy_logger.warning("Tool is unknown type: %s, passing as-is", type(tool)) openai_tools_as_dicts.append(tool) verbose_proxy_logger.debug( - f"Expanded {len(mcp_tools)} MCP reference(s) to {len(openai_tools_as_dicts)} tools (all as dicts)" + "Expanded %s MCP reference(s) to %s tools (all as dicts)", len(mcp_tools), len(openai_tools_as_dicts) ) return openai_tools_as_dicts @@ -235,13 +238,14 @@ class SemanticToolFilterHook(CustomLogger): metadata["litellm_semantic_filter_tools"] = tool_names_csv verbose_proxy_logger.info( - f"Semantic tool filter: {filter_stats} MCP tools " - f"({len(native_tools)} native preserved, " - f"{len(filtered_tools)} total)" + "Semantic tool filter: %s MCP tools (%s native preserved, %s total)", + filter_stats, + len(native_tools), + len(filtered_tools), ) else: verbose_proxy_logger.info( - f"Semantic tool filter: all {len(native_tools)} tools are native, no MCP filtering applied" + "Semantic tool filter: all %s tools are native, no MCP filtering applied", len(native_tools) ) def _emit_filter_metadata_safe( @@ -266,7 +270,8 @@ class SemanticToolFilterHook(CustomLogger): ) except Exception as e: verbose_proxy_logger.warning( - f"Failed to emit semantic filter metadata: {e}", + "Failed to emit semantic filter metadata: %s", + e, exc_info=True, ) @@ -284,7 +289,7 @@ class SemanticToolFilterHook(CustomLogger): tools list to only include semantically relevant tools. """ if call_type not in ("completion", "acompletion", "aresponses"): - verbose_proxy_logger.debug(f"Skipping semantic filter for call_type={call_type}") + verbose_proxy_logger.debug("Skipping semantic filter for call_type=%s", call_type) return None tools = data.get("tools") @@ -321,16 +326,17 @@ class SemanticToolFilterHook(CustomLogger): filtered_tools=narrowed_tools, ) verbose_proxy_logger.info( - f"Expanded MCP references to {len(expanded_tools)} tools " - f"({len(native_tools_before_expand)} native preserved), " - f"semantic filter selected {len(filtered_expanded_tools)}" + "Expanded MCP references to %s tools (%s native preserved), semantic filter selected %s", + len(expanded_tools), + len(native_tools_before_expand), + len(filtered_expanded_tools), ) return data except SemanticToolFilterContextWindowError as e: raise HTTPException(status_code=400, detail={"error": str(e)}) from e except Exception as e: - verbose_proxy_logger.error(f"Failed to expand MCP references: {e}", exc_info=True) + verbose_proxy_logger.error("Failed to expand MCP references: %s", e, exc_info=True) return None messages = data.get("messages", []) @@ -361,9 +367,10 @@ class SemanticToolFilterHook(CustomLogger): native_tools.append(t) verbose_proxy_logger.debug( - f"Applying semantic filter: {len(mcp_tools)} MCP tools, " - f"{len(native_tools)} native tools, " - f"query: '{user_query[:50]}...'" + "Applying semantic filter: %s MCP tools, %s native tools, query: '%s...'", + len(mcp_tools), + len(native_tools), + user_query[:50], ) if mcp_tools: @@ -404,7 +411,7 @@ class SemanticToolFilterHook(CustomLogger): except SemanticToolFilterContextWindowError as e: raise HTTPException(status_code=400, detail={"error": str(e)}) from e except Exception as e: - verbose_proxy_logger.warning(f"Semantic tool filter hook failed: {e}. Proceeding with all tools.") + verbose_proxy_logger.warning("Semantic tool filter hook failed: %s. Proceeding with all tools.", e) return None async def async_post_call_response_headers_hook( @@ -497,18 +504,19 @@ class SemanticToolFilterHook(CustomLogger): hook = SemanticToolFilterHook(semantic_filter) verbose_proxy_logger.info( - f"✅ MCP Semantic Tool Filter enabled: " - f"embedding_model={embedding_model}, top_k={top_k}, " - f"similarity_threshold={similarity_threshold}" + "✅ MCP Semantic Tool Filter enabled: embedding_model=%s, top_k=%s, similarity_threshold=%s", + embedding_model, + top_k, + similarity_threshold, ) return hook except ImportError as e: verbose_proxy_logger.warning( - f"semantic-router not installed. Install with: pip install 'litellm[semantic-router]'. Error: {e}" + "semantic-router not installed. Install with: pip install 'litellm[semantic-router]'. Error: %s", e ) return None except Exception as e: - verbose_proxy_logger.exception(f"Failed to initialize MCP semantic tool filter: {e}") + verbose_proxy_logger.exception("Failed to initialize MCP semantic tool filter: %s", e) return None diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py index 2aeb505bd97..630fcdb8fe7 100644 --- a/litellm/proxy/hooks/model_max_budget_limiter.py +++ b/litellm/proxy/hooks/model_max_budget_limiter.py @@ -56,7 +56,7 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): model=model, internal_model_max_budget=internal_model_max_budget ) if _current_model_budget_info is None: - verbose_proxy_logger.debug(f"Model {model} not found in internal_model_max_budget") + verbose_proxy_logger.debug("Model %s not found in internal_model_max_budget", model) return True # check if current model is within budget @@ -122,7 +122,7 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): model=model, internal_model_max_budget=internal_model_max_budget ) if _current_model_budget_info is None: - verbose_proxy_logger.debug(f"Model {model} not found in end_user_model_max_budget") + verbose_proxy_logger.debug("Model %s not found in end_user_model_max_budget", model) return True # check if current model is within budget diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index 04f34d0e9cf..081200366d6 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -69,7 +69,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): rate_limit_type: Literal["key", "model_per_key", "user", "customer", "team"], values_to_update_in_cache: list[tuple[Any, Any]], ) -> dict: - verbose_proxy_logger.info(f"Current Usage of {rate_limit_type} in this minute: {current}") + verbose_proxy_logger.info("Current Usage of %s in this minute: %s", rate_limit_type, current) if current is None: if max_parallel_requests == 0 or tpm_limit == 0 or rpm_limit == 0: # base case — at least one dimension is set to 0 (effectively @@ -776,7 +776,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): litellm_parent_otel_span=litellm_parent_otel_span, ) # save in cache for up to 1 min. except Exception as e: - verbose_proxy_logger.exception(f"Inside Parallel Request Limiter: An exception occurred - {e}") + verbose_proxy_logger.exception("Inside Parallel Request Limiter: An exception occurred - %s", e) async def get_internal_user_object( self, diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 98f1e650845..af4818dec02 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -490,7 +490,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): parallel_request_limiter=self, ) except Exception as e: - verbose_proxy_logger.debug(f"Could not load batch rate limiter: {e}") + verbose_proxy_logger.debug("Could not load batch rate limiter: %s", e) return self._batch_rate_limiter def _get_current_time(self) -> datetime: @@ -579,9 +579,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): total_estimated = estimated_input_tokens + max_tokens_estimate verbose_proxy_logger.debug( - f"TPM reservation estimate: input={estimated_input_tokens}, " - f"max_tokens={max_tokens_estimate} (explicit={explicit_max_tokens is not None}), " - f"total={total_estimated}" + "TPM reservation estimate: input=%s, max_tokens=%s (explicit=%s), total=%s", + estimated_input_tokens, + max_tokens_estimate, + explicit_max_tokens is not None, + total_estimated, ) return total_estimated @@ -808,7 +810,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) all_cache_values.extend(group_cache_values) except Exception as e: - verbose_proxy_logger.warning(f"Redis Lua script failed for hash tag {hash_tag}: {e}") + verbose_proxy_logger.warning("Redis Lua script failed for hash tag %s: %s", hash_tag, e) # Fallback to in-memory cache for this group group_cache_values = await self.in_memory_cache_sliding_window( keys=group_keys, @@ -1055,7 +1057,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) counts = [max(0, int(value)) for value in raw_counts] except Exception as e: # noqa: BLE001 - any Redis/Lua failure degrades to the local mirror, never a 500 - verbose_proxy_logger.warning(f"parallel_count_script failed, using local mirror: {e}") + verbose_proxy_logger.warning("parallel_count_script failed, using local mirror: %s", e) counts = await self._read_local_gauge_counts(gauge_keys, parent_otel_span) else: counts = await self._read_local_gauge_counts(gauge_keys, parent_otel_span) @@ -1085,7 +1087,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ], ) except Exception as e: # noqa: BLE001 - any Redis/Lua failure degrades to in-memory enforcement, never a 500 - verbose_proxy_logger.warning(f"parallel_acquire_script failed, falling back to in-memory gauge: {e}") + verbose_proxy_logger.warning("parallel_acquire_script failed, falling back to in-memory gauge: %s", e) async with self._check_and_increment_lock: return await self._acquire_parallel_slots_in_memory(gauges, slot_id, parent_otel_span) if int(raw[0]) == 1: @@ -1212,7 +1214,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) return except Exception as e: # noqa: BLE001 - any Redis/Lua failure degrades to the in-memory release, never a 500 - verbose_proxy_logger.warning(f"parallel_release_script failed, falling back to in-memory release: {e}") + verbose_proxy_logger.warning("parallel_release_script failed, falling back to in-memory release: %s", e) async with self._check_and_increment_lock: for counter_key in counter_keys: @@ -1387,12 +1389,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # to its pre-call state, then fall back to in-memory for the # whole call (counters there are independent of Redis). verbose_proxy_logger.error( - f"atomic_check_and_increment_by_n: Redis Lua execution " - f"failed ({type(e).__name__}: {e}). Refunding " - f"{len(applied)} prior descriptors and falling back to " - f"in-memory enforcement — counters will diverge from " - f"Redis until window expires (window_size=" - f"{self.window_size}s)." + "atomic_check_and_increment_by_n: Redis Lua execution failed (%s: %s). Refunding %s prior descriptors and falling back to in-memory enforcement — counters will diverge from Redis until window expires (window_size=%ss).", + type(e).__name__, + e, + len(applied), + self.window_size, ) await self._refund_applied_descriptor_groups(applied) flat_meta: list[dict[str, Any]] = [m for _k, _a, group_meta in descriptor_groups for m in group_meta] @@ -1434,7 +1435,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) except Exception as e: verbose_proxy_logger.warning( - f"Failed to refund {entry['counter_key']} on cross-descriptor rollback: {e}" + "Failed to refund %s on cross-descriptor rollback: %s", entry["counter_key"], e ) def _build_atomic_response( @@ -2227,18 +2228,20 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if failure_count > DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE: verbose_proxy_logger.debug( - f"[Dynamic Rate Limit] Deployment {deployment_id} has {failure_count} failures " - f"in current minute - enforcing rate limits for model {model}" + "[Dynamic Rate Limit] Deployment %s has %s failures in current minute - enforcing rate limits for model %s", + deployment_id, + failure_count, + model, ) return True verbose_proxy_logger.debug( - f"[Dynamic Rate Limit] No failures detected for model {model} - allowing dynamic exceeding" + "[Dynamic Rate Limit] No failures detected for model %s - allowing dynamic exceeding", model ) return False except Exception as e: - verbose_proxy_logger.debug(f"Error checking model failure status: {e}, defaulting to enforce limits") + verbose_proxy_logger.debug("Error checking model failure status: %s, defaulting to enforce limits", e) # Fail safe: enforce limits if we can't check return True @@ -2573,7 +2576,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if stored_response is not None: stored_response["statuses"].extend(tpm_response["statuses"]) - verbose_proxy_logger.debug(f"TPM tokens reserved: {estimated_tokens} for model {requested_model}") + verbose_proxy_logger.debug( + "TPM tokens reserved: %s for model %s", estimated_tokens, requested_model + ) def _create_pipeline_operations( self, @@ -2704,8 +2709,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ttl_value = op["ttl"] if op["ttl"] is not None else 0 verbose_proxy_logger.debug( - f"Executing TTL-preserving increment for key={op['key']}, " - f"increment={op['increment_value']}, ttl={ttl_value}" + "Executing TTL-preserving increment for key=%s, increment=%s, ttl=%s", + op["key"], + op["increment_value"], + ttl_value, ) keys.append(op["key"]) args.extend([op["increment_value"], ttl_value]) @@ -2740,11 +2747,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): await self._execute_token_increment_script(pipeline_operations) verbose_proxy_logger.debug( - f"Successfully executed TTL-preserving increment for {len(pipeline_operations)} keys" + "Successfully executed TTL-preserving increment for %s keys", len(pipeline_operations) ) except Exception as e: - verbose_proxy_logger.warning(f"TTL preservation failed, falling back to regular pipeline: {e}") + verbose_proxy_logger.warning("TTL preservation failed, falling back to regular pipeline: %s", e) # Fallback to regular pipeline on error await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline( increment_list=pipeline_operations, @@ -2950,9 +2957,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) if reserved_tokens > 0 and total_tokens < reserved_tokens: verbose_proxy_logger.debug( - f"Releasing unused TPM budget on success: " - f"reserved={reserved_tokens}, actual={total_tokens}, " - f"release={reserved_tokens - total_tokens}" + "Releasing unused TPM budget on success: reserved=%s, actual=%s, release=%s", + reserved_tokens, + total_tokens, + reserved_tokens - total_tokens, ) pipeline_operations.extend( self._build_reservation_aware_tpm_ops( @@ -3001,7 +3009,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) except Exception as e: - verbose_proxy_logger.exception(f"Error in rate limit success event: {e}") + verbose_proxy_logger.exception("Error in rate limit success event: %s", e) async def async_logging_hook( self, @@ -3095,7 +3103,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if stash is not None and not stash.reservation_released: reserved_tokens = stash.reserved_tokens if stash is not None and reserved_tokens > 0: - verbose_proxy_logger.debug(f"Releasing reserved TPM tokens on failure: {reserved_tokens}") + verbose_proxy_logger.debug("Releasing reserved TPM tokens on failure: %s", reserved_tokens) # Refund only against the scopes the reservation actually # charged. _build_reservation_aware_tpm_ops with # actual_tokens=0 emits -reserved on reserved scopes and 0 @@ -3118,7 +3126,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if stash is not None and reserved_tokens > 0: stash.reservation_released = True except Exception as e: - verbose_proxy_logger.exception(f"Error in rate limit failure event: {e}") + verbose_proxy_logger.exception("Error in rate limit failure event: %s", e) async def async_release_max_parallel_requests_on_disconnect( self, @@ -3183,7 +3191,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) except Exception as e: - verbose_proxy_logger.exception(f"Error in rate limit post-call hook: {e}") + verbose_proxy_logger.exception("Error in rate limit post-call hook: %s", e) async def async_post_call_failure_hook( self, @@ -3231,12 +3239,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): reserved_tokens=reserved_tokens, ) if ops: - verbose_proxy_logger.debug(f"Releasing reserved TPM tokens on proxy-level rejection: {reserved_tokens}") + verbose_proxy_logger.debug( + "Releasing reserved TPM tokens on proxy-level rejection: %s", reserved_tokens + ) await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline( increment_list=ops, litellm_parent_otel_span=user_api_key_dict.parent_otel_span, ) stash.reservation_released = True except Exception as e: - verbose_proxy_logger.exception(f"Error releasing TPM reservation on post-call failure: {e}") + verbose_proxy_logger.exception("Error releasing TPM reservation on post-call failure: %s", e) return diff --git a/litellm/proxy/hooks/prompt_injection_detection.py b/litellm/proxy/hooks/prompt_injection_detection.py index e7192b9b063..1cb80bc37f6 100644 --- a/litellm/proxy/hooks/prompt_injection_detection.py +++ b/litellm/proxy/hooks/prompt_injection_detection.py @@ -197,7 +197,7 @@ class _OPTIONAL_PromptInjectionDetection(CustomLogger): raise e except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {e}" + "litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - %s", e ) async def async_moderation_hook( # type: ignore diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 0319a680714..e58923fa416 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -195,7 +195,9 @@ class _ProxyDBLogger(CustomLogger): verbose_proxy_logger.debug("INSIDE _PROXY_track_cost_callback") try: verbose_proxy_logger.debug( - f"kwargs stream: {kwargs.get('stream', None)} + complete streaming response: {kwargs.get('complete_streaming_response', None)}" + "kwargs stream: %s + complete streaming response: %s", + kwargs.get("stream", None), + kwargs.get("complete_streaming_response", None), ) parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs=kwargs) litellm_params = kwargs.get("litellm_params", {}) or {} @@ -225,10 +227,14 @@ class _ProxyDBLogger(CustomLogger): user_api_key = metadata.get("user_api_key", None) if kwargs.get("cache_hit", False) is True: response_cost = 0.0 - verbose_proxy_logger.debug(f"Cache Hit: response_cost {response_cost}, for user_id {user_id}") + verbose_proxy_logger.debug("Cache Hit: response_cost %s, for user_id %s", response_cost, user_id) verbose_proxy_logger.debug( - f"user_api_key {user_api_key}, user_id {user_id}, team_id {team_id}, end_user_id {end_user_id}" + "user_api_key %s, user_id %s, team_id %s, end_user_id %s", + user_api_key, + user_id, + team_id, + end_user_id, ) call_type: str | None = kwargs.get("call_type") if _should_track_cost_callback( diff --git a/litellm/proxy/hooks/responses_id_security.py b/litellm/proxy/hooks/responses_id_security.py index 8b6bfd95892..5a8abcc9324 100644 --- a/litellm/proxy/hooks/responses_id_security.py +++ b/litellm/proxy/hooks/responses_id_security.py @@ -83,7 +83,9 @@ class ResponsesIDSecurity(CustomLogger): if response_id_user_id and response_id_user_id != user_api_key_dict.user_id: if general_settings.get("disable_responses_id_security", False): verbose_proxy_logger.debug( - f"Responses ID Security is disabled. User {user_api_key_dict.user_id} is accessing response id {response_id_user_id} which is not associated with them." + "Responses ID Security is disabled. User %s is accessing response id %s which is not associated with them.", + user_api_key_dict.user_id, + response_id_user_id, ) return True raise HTTPException( @@ -94,7 +96,10 @@ class ResponsesIDSecurity(CustomLogger): if response_id_team_id and response_id_team_id != user_api_key_dict.team_id: if general_settings.get("disable_responses_id_security", False): verbose_proxy_logger.debug( - f"Responses ID Security is disabled. Response belongs to team {response_id_team_id} but user {user_api_key_dict.user_id} is accessing it with team id {user_api_key_dict.team_id}." + "Responses ID Security is disabled. Response belongs to team %s but user %s is accessing it with team id %s.", + response_id_team_id, + user_api_key_dict.user_id, + user_api_key_dict.team_id, ) return True raise HTTPException( diff --git a/litellm/proxy/hooks/user_management_event_hooks.py b/litellm/proxy/hooks/user_management_event_hooks.py index b242f763fcb..b434bc001b9 100644 --- a/litellm/proxy/hooks/user_management_event_hooks.py +++ b/litellm/proxy/hooks/user_management_event_hooks.py @@ -71,7 +71,7 @@ class UserManagementEventHooks: ) ) except Exception as e: - verbose_proxy_logger.warning(f"Unable to create audit log for user on `/user/new` - {e}") + verbose_proxy_logger.warning("Unable to create audit log for user on `/user/new` - %s", e) @staticmethod async def async_send_user_invitation_email( diff --git a/litellm/proxy/image_endpoints/endpoints.py b/litellm/proxy/image_endpoints/endpoints.py index 36f702e1b4b..04e3982b412 100644 --- a/litellm/proxy/image_endpoints/endpoints.py +++ b/litellm/proxy/image_endpoints/endpoints.py @@ -185,7 +185,7 @@ async def image_generation( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.image_generation(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.image_generation(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 6864caccea2..5b9a9c9a202 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -905,7 +905,7 @@ class LiteLLMProxyRequestSetup: user = LiteLLMProxyRequestSetup._get_case_insensitive_header(headers, header_name) if user is not None: - verbose_logger.info(f'found user "{user}" in header "{header_name}"') + verbose_logger.info('found user "%s" in header "%s"', user, header_name) return user @@ -918,7 +918,7 @@ class LiteLLMProxyRequestSetup: return None for header, value in headers.items(): if header.lower() == "openai-organization": - verbose_logger.info(f"found openai org id: {value}, sending to llm") + verbose_logger.info("found openai org id: %s, sending to llm", value) return value return None @@ -1059,14 +1059,14 @@ class LiteLLMProxyRequestSetup: if agent_id_from_header: metadata_from_headers["agent_id"] = agent_id_from_header - verbose_proxy_logger.debug(f"Extracted agent_id from header: {agent_id_from_header}") + verbose_proxy_logger.debug("Extracted agent_id from header: %s", agent_id_from_header) if chain_id: metadata_from_headers["trace_id"] = chain_id metadata_from_headers["session_id"] = chain_id data["litellm_session_id"] = chain_id data["litellm_trace_id"] = chain_id - verbose_proxy_logger.debug(f"Extracted chain_id from header (trace-id/session-id): {chain_id}") + verbose_proxy_logger.debug("Extracted chain_id from header (trace-id/session-id): %s", chain_id) else: body_metadata = data.get("metadata") session_id = _get_anthropic_session_id_from_metadata(body_metadata) @@ -1475,8 +1475,8 @@ async def add_litellm_data_to_request( allow_client_message_redaction_opt_out=_allow_client_message_redaction_opt_out, ) _logging_safe_headers = redact_credential_headers(_headers) - verbose_proxy_logger.debug(f"Request Headers: {_logging_safe_headers}") - verbose_proxy_logger.debug(f"Raw Headers: {_raw_headers}") + verbose_proxy_logger.debug("Request Headers: %s", _logging_safe_headers) + verbose_proxy_logger.debug("Raw Headers: %s", _raw_headers) if forward_llm_auth and "x-api-key" in _headers: data["api_key"] = _headers["x-api-key"] @@ -1579,7 +1579,7 @@ async def add_litellm_data_to_request( data["metadata"] = safe_json_loads(data["metadata"]) if not isinstance(data["metadata"], dict): verbose_proxy_logger.warning( - f"Failed to parse 'metadata' as JSON dict. Received value: {data['metadata']}" + "Failed to parse 'metadata' as JSON dict. Received value: %s", data["metadata"] ) # requester_metadata is snapshotted AFTER the strip below so # downstream consumers (e.g. PANW guardrail reading user_ip / @@ -1592,7 +1592,7 @@ async def add_litellm_data_to_request( parsed_litellm_metadata = safe_json_loads(data["litellm_metadata"]) if not isinstance(parsed_litellm_metadata, dict): verbose_proxy_logger.warning( - f"Failed to parse 'litellm_metadata' as JSON dict. Received value: {data['litellm_metadata']}" + "Failed to parse 'litellm_metadata' as JSON dict. Received value: %s", data["litellm_metadata"] ) else: data["litellm_metadata"] = parsed_litellm_metadata @@ -2409,7 +2409,7 @@ def _add_guardrails_from_policies_in_metadata( if not policy_names: return - verbose_proxy_logger.debug(f"Policy engine: resolving guardrails from key/team policies: {policy_names}") + verbose_proxy_logger.debug("Policy engine: resolving guardrails from key/team policies: %s", policy_names) # Check if policy registry is initialized registry = get_policy_registry() @@ -2434,10 +2434,10 @@ def _add_guardrails_from_policies_in_metadata( ) resolved_guardrails.update(resolved_policy.guardrails) verbose_proxy_logger.debug( - f"Policy engine: resolved guardrails from policy '{policy_name}': {resolved_policy.guardrails}" + "Policy engine: resolved guardrails from policy '%s': %s", policy_name, resolved_policy.guardrails ) else: - verbose_proxy_logger.warning(f"Policy engine: policy '{policy_name}' not found in registry") + verbose_proxy_logger.warning("Policy engine: policy '%s' not found in registry", policy_name) if not resolved_guardrails: return @@ -2461,7 +2461,7 @@ def _add_guardrails_from_policies_in_metadata( data[metadata_variable_name]["applied_policies"].extend(list(policy_names)) verbose_proxy_logger.debug( - f"Policy engine: added guardrails from key/team policies to request metadata: {list(resolved_guardrails)}" + "Policy engine: added guardrails from key/team policies to request metadata: %s", list(resolved_guardrails) ) @@ -2592,13 +2592,13 @@ def _match_and_track_policies( matching_policy_names = [m["policy_name"] for m in matches_with_reasons] policy_reasons = {m["policy_name"]: m["matched_via"] for m in matches_with_reasons} - verbose_proxy_logger.debug(f"Policy engine: matched policies via attachments: {matching_policy_names}") + verbose_proxy_logger.debug("Policy engine: matched policies via attachments: %s", matching_policy_names) # Combine attachment-based policies with dynamic request body policies all_policy_names = set(matching_policy_names) if request_body_policies and isinstance(request_body_policies, list): all_policy_names.update(request_body_policies) - verbose_proxy_logger.debug(f"Policy engine: added dynamic policies from request body: {request_body_policies}") + verbose_proxy_logger.debug("Policy engine: added dynamic policies from request body: %s", request_body_policies) if not all_policy_names: return [], {} @@ -2610,7 +2610,7 @@ def _match_and_track_policies( policies=policies_override, ) - verbose_proxy_logger.debug(f"Policy engine: applied policies (conditions matched): {applied_policy_names}") + verbose_proxy_logger.debug("Policy engine: applied policies (conditions matched): %s", applied_policy_names) # Track applied policies in metadata for response headers for policy_name in applied_policy_names: @@ -2641,7 +2641,7 @@ def _apply_resolved_guardrails_to_metadata( policy_names=policy_names, ) - verbose_proxy_logger.debug(f"Policy engine: resolved guardrails: {resolved_guardrails}") + verbose_proxy_logger.debug("Policy engine: resolved guardrails: %s", resolved_guardrails) # Resolve pipelines from matching policies pipelines = PolicyResolver.resolve_pipelines_for_context( @@ -2661,7 +2661,9 @@ def _apply_resolved_guardrails_to_metadata( data[metadata_variable_name]["_guardrail_pipelines"] = pipelines data[metadata_variable_name]["_pipeline_managed_guardrails"] = pipeline_managed_guardrails verbose_proxy_logger.debug( - f"Policy engine: resolved {len(pipelines)} pipeline(s), managed guardrails: {pipeline_managed_guardrails}" + "Policy engine: resolved %s pipeline(s), managed guardrails: %s", + len(pipelines), + pipeline_managed_guardrails, ) if not resolved_guardrails and not pipelines: @@ -2678,7 +2680,7 @@ def _apply_resolved_guardrails_to_metadata( combined -= pipeline_managed_guardrails data[metadata_variable_name]["guardrails"] = list(combined) - verbose_proxy_logger.debug(f"Policy engine: added guardrails to request metadata: {list(combined)}") + verbose_proxy_logger.debug("Policy engine: added guardrails to request metadata: %s", list(combined)) async def add_guardrails_from_policy_engine( @@ -2714,8 +2716,9 @@ async def add_guardrails_from_policy_engine( registry = get_policy_registry() verbose_proxy_logger.debug( - f"Policy engine: registry initialized={registry.is_initialized()}, " - f"policy_count={len(registry.get_all_policies())}" + "Policy engine: registry initialized=%s, policy_count=%s", + registry.is_initialized(), + len(registry.get_all_policies()), ) if not registry.is_initialized(): verbose_proxy_logger.debug("Policy engine not initialized, skipping policy matching") @@ -2733,8 +2736,11 @@ async def add_guardrails_from_policy_engine( ) verbose_proxy_logger.debug( - f"Policy engine: matching policies for context team_alias={context.team_alias}, " - f"key_alias={context.key_alias}, model={context.model}, tags={context.tags}" + "Policy engine: matching policies for context team_alias=%s, key_alias=%s, model=%s, tags=%s", + context.team_alias, + context.key_alias, + context.model, + context.tags, ) # Separate policy names from policy version IDs (policy_) @@ -2760,9 +2766,9 @@ async def add_guardrails_from_policy_engine( pname, policy = result merged_policies[pname] = policy fetched_policy_names.append(pname) - verbose_proxy_logger.debug(f"Policy engine: loaded version by ID policy_{policy_id} -> {pname}") + verbose_proxy_logger.debug("Policy engine: loaded version by ID policy_%s -> %s", policy_id, pname) else: - verbose_proxy_logger.debug(f"Policy engine: policy version {policy_id} not found in cache, skipping") + verbose_proxy_logger.debug("Policy engine: policy version %s not found in cache, skipping", policy_id) # Build request body list: names + policy names from fetched versions request_body_policies = request_body_names + fetched_policy_names diff --git a/litellm/proxy/management_endpoints/cache_settings_endpoints.py b/litellm/proxy/management_endpoints/cache_settings_endpoints.py index 22438e6f336..9b803d94d3c 100644 --- a/litellm/proxy/management_endpoints/cache_settings_endpoints.py +++ b/litellm/proxy/management_endpoints/cache_settings_endpoints.py @@ -386,7 +386,8 @@ class CacheSettingsManager: verbose_proxy_logger.info("Cache settings initialized from database") except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.management_endpoints.cache_settings_endpoints.py::CacheSettingsManager::init_cache_settings_in_db - {e}" + "litellm.proxy.management_endpoints.cache_settings_endpoints.py::CacheSettingsManager::init_cache_settings_in_db - %s", + e, ) @staticmethod @@ -480,7 +481,7 @@ async def get_cache_settings( redis_type_descriptions=REDIS_TYPE_DESCRIPTIONS, ) except Exception as e: - verbose_proxy_logger.error(f"Error fetching cache settings: {e}") + verbose_proxy_logger.error("Error fetching cache settings: %s", e) raise HTTPException(status_code=500, detail=f"Error fetching cache settings: {e}") @@ -539,7 +540,7 @@ async def test_cache_connection( return CacheTestResponse(**result) except Exception as e: - verbose_proxy_logger.error(f"Error testing cache connection: {e}") + verbose_proxy_logger.error("Error testing cache connection: %s", e) return CacheTestResponse( status="failed", message=f"Cache connection test failed: {e}", @@ -652,5 +653,5 @@ async def update_cache_settings( "settings": _redact_credentials(cache_settings), } except Exception as e: - verbose_proxy_logger.error(f"Error updating cache settings: {e}") + verbose_proxy_logger.error("Error updating cache settings: %s", e) raise HTTPException(status_code=500, detail=f"Error updating cache settings: {e}") diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 132a8409e06..1f83fa6a65c 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -1001,7 +1001,7 @@ async def get_daily_activity( ) except Exception as e: - verbose_proxy_logger.exception(f"Error fetching daily activity: {e}") + verbose_proxy_logger.exception("Error fetching daily activity: %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to fetch analytics: {e}"}, @@ -1091,7 +1091,7 @@ async def get_daily_activity_aggregated( ) except Exception as e: - verbose_proxy_logger.exception(f"Error fetching aggregated daily activity: {e}") + verbose_proxy_logger.exception("Error fetching aggregated daily activity: %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to fetch analytics: {e}"}, diff --git a/litellm/proxy/management_endpoints/common_utils.py b/litellm/proxy/management_endpoints/common_utils.py index abb1b686d1d..387d4421188 100644 --- a/litellm/proxy/management_endpoints/common_utils.py +++ b/litellm/proxy/management_endpoints/common_utils.py @@ -222,7 +222,7 @@ async def _user_has_admin_privileges( except Exception as e: # If there's an error checking, default to False for security - verbose_proxy_logger.debug(f"Error checking admin privileges for user {user_api_key_dict.user_id}: {e}") + verbose_proxy_logger.debug("Error checking admin privileges for user %s: %s", user_api_key_dict.user_id, e) return False return False @@ -366,7 +366,7 @@ async def admin_can_invite_user( return False except Exception as e: - verbose_proxy_logger.debug(f"Error checking invite permission for user {user_api_key_dict.user_id}: {e}") + verbose_proxy_logger.debug("Error checking invite permission for user %s: %s", user_api_key_dict.user_id, e) return False diff --git a/litellm/proxy/management_endpoints/cost_tracking_settings.py b/litellm/proxy/management_endpoints/cost_tracking_settings.py index f2295452f4d..d199cea5fd3 100644 --- a/litellm/proxy/management_endpoints/cost_tracking_settings.py +++ b/litellm/proxy/management_endpoints/cost_tracking_settings.py @@ -59,7 +59,7 @@ def _resolve_model_for_cost_lookup(model: str) -> tuple[str, str | None]: # Check base_model first (needed for Azure custom deployment names) base_model = model_info.get("base_model") or litellm_params.get("base_model") if base_model: - verbose_proxy_logger.debug(f"Resolved model '{model}' to base_model '{base_model}' from router") + verbose_proxy_logger.debug("Resolved model '%s' to base_model '%s' from router", model, base_model) custom_llm_provider = litellm_params.get("custom_llm_provider") return ( str(base_model), @@ -69,14 +69,14 @@ def _resolve_model_for_cost_lookup(model: str) -> tuple[str, str | None]: resolved_model = litellm_params.get("model") if resolved_model: - verbose_proxy_logger.debug(f"Resolved model '{model}' to '{resolved_model}' from router") + verbose_proxy_logger.debug("Resolved model '%s' to '%s' from router", model, resolved_model) custom_llm_provider = litellm_params.get("custom_llm_provider") return ( str(resolved_model), (str(custom_llm_provider) if custom_llm_provider is not None else None), ) except Exception as e: - verbose_proxy_logger.debug(f"Could not resolve model '{model}' from router: {e}") + verbose_proxy_logger.debug("Could not resolve model '%s' from router: %s", model, e) # Return original model if not resolved return model, custom_llm_provider @@ -129,7 +129,7 @@ async def get_cost_discount_config( return {"values": cost_discount_config} except Exception as e: - verbose_proxy_logger.error(f"Error fetching cost discount config: {e}") + verbose_proxy_logger.error("Error fetching cost discount config: %s", e) return {"values": {}} @@ -216,7 +216,7 @@ async def update_cost_discount_config( # Update in-memory litellm.cost_discount_config litellm.cost_discount_config = cost_discount_config - verbose_proxy_logger.info(f"Updated cost_discount_config: {cost_discount_config}") + verbose_proxy_logger.info("Updated cost_discount_config: %s", cost_discount_config) return { "message": "Cost discount configuration updated successfully", @@ -224,7 +224,7 @@ async def update_cost_discount_config( "values": cost_discount_config, } except Exception as e: - verbose_proxy_logger.error(f"Error updating cost discount config: {e}") + verbose_proxy_logger.error("Error updating cost discount config: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to update cost discount config: {e}"}, @@ -262,7 +262,7 @@ async def get_cost_margin_config( return {"values": cost_margin_config} except Exception as e: - verbose_proxy_logger.error(f"Error fetching cost margin config: {e}") + verbose_proxy_logger.error("Error fetching cost margin config: %s", e) return {"values": {}} @@ -390,7 +390,7 @@ async def update_cost_margin_config( # Update in-memory litellm.cost_margin_config litellm.cost_margin_config = cost_margin_config - verbose_proxy_logger.info(f"Updated cost_margin_config: {cost_margin_config}") + verbose_proxy_logger.info("Updated cost_margin_config: %s", cost_margin_config) return { "message": "Cost margin configuration updated successfully", @@ -398,7 +398,7 @@ async def update_cost_margin_config( "values": cost_margin_config, } except Exception as e: - verbose_proxy_logger.error(f"Error updating cost margin config: {e}") + verbose_proxy_logger.error("Error updating cost margin config: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to update cost margin config: {e}"}, @@ -450,7 +450,7 @@ async def estimate_cost( # Resolve model name (handles router aliases like 'e-model-router' -> 'azure_ai/gpt-4') resolved_model, resolved_provider = _resolve_model_for_cost_lookup(request.model) - verbose_proxy_logger.debug(f"Cost estimate: request.model='{request.model}' resolved to '{resolved_model}'") + verbose_proxy_logger.debug("Cost estimate: request.model='%s' resolved to '%s'", request.model, resolved_model) # Create a mock response with usage for completion_cost mock_response = ModelResponse( diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index ff384190d31..ab59ee65a0a 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -103,7 +103,7 @@ async def block_user(data: BlockUsers): return {"blocked_users": records} except Exception as e: - verbose_proxy_logger.error(f"An error occurred - {e}") + verbose_proxy_logger.error("An error occurred - %s", e) raise HTTPException(status_code=500, detail={"error": str(e)}) @@ -377,7 +377,8 @@ async def new_end_user( # It should have been converted to object_permission_id by _set_object_permission if "object_permission" in new_end_user_obj: verbose_proxy_logger.warning( - f"object_permission still in new_end_user_obj after _set_object_permission: {new_end_user_obj.get('object_permission')}" + "object_permission still in new_end_user_obj after _set_object_permission: %s", + new_end_user_obj.get("object_permission"), ) new_end_user_obj.pop("object_permission", None) @@ -390,7 +391,7 @@ async def new_end_user( return _to_customer_response(end_user_record) except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.management_endpoints.customer_endpoints.new_end_user(): Exception occured - {e}" + "litellm.proxy.management_endpoints.customer_endpoints.new_end_user(): Exception occured - %s", e ) if "Unique constraint failed on the fields: (`user_id`)" in str(e): raise ProxyException( @@ -455,7 +456,7 @@ async def end_user_info( except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.management_endpoints.customer_endpoints.end_user_info(): Exception occured - {e}" + "litellm.proxy.management_endpoints.customer_endpoints.end_user_info(): Exception occured - %s", e ) raise handle_exception_on_proxy(e) @@ -613,7 +614,8 @@ async def update_end_user( # It should have been converted to object_permission_id by handle_update_object_permission_common if "object_permission" in update_end_user_table_data: verbose_proxy_logger.warning( - f"object_permission still in update_end_user_table_data: {update_end_user_table_data.get('object_permission')}" + "object_permission still in update_end_user_table_data: %s", + update_end_user_table_data.get("object_permission"), ) update_end_user_table_data.pop("object_permission", None) @@ -627,7 +629,7 @@ async def update_end_user( ) if response is None: raise ValueError(f"Failed updating customer data. User ID does not exist passed user_id={data.user_id}") - verbose_proxy_logger.debug(f"received response from updating prisma client. response={response}") + verbose_proxy_logger.debug("received response from updating prisma client. response=%s", response) return _to_customer_response(response) else: @@ -636,7 +638,7 @@ async def update_end_user( # update based on remaining passed in values except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.update_end_user(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.update_end_user(): Exception occured - %s", e) raise handle_exception_on_proxy(e) @@ -701,7 +703,7 @@ async def delete_end_user( response = await EndUserRepository(prisma_client).table.delete_many( where={"user_id": {"in": data.user_ids}} ) - verbose_proxy_logger.debug(f"received response from updating prisma client. response={response}") + verbose_proxy_logger.debug("received response from updating prisma client. response=%s", response) return DeleteCustomersResponse( deleted_customers=response, message="Successfully deleted customers with ids: " + str(data.user_ids), @@ -711,7 +713,7 @@ async def delete_end_user( # update based on remaining passed in values except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.delete_end_user(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.delete_end_user(): Exception occured - %s", e) raise handle_exception_on_proxy(e) @@ -767,7 +769,7 @@ async def list_end_user( except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.management_endpoints.customer_endpoints.list_end_user(): Exception occured - {e}" + "litellm.proxy.management_endpoints.customer_endpoints.list_end_user(): Exception occured - %s", e ) raise handle_exception_on_proxy(e) diff --git a/litellm/proxy/management_endpoints/fallback_management_endpoints.py b/litellm/proxy/management_endpoints/fallback_management_endpoints.py index 3df5384b551..00dc4f0f23b 100644 --- a/litellm/proxy/management_endpoints/fallback_management_endpoints.py +++ b/litellm/proxy/management_endpoints/fallback_management_endpoints.py @@ -169,7 +169,7 @@ async def create_fallback( setattr(llm_router, fallback_key, existing_fallbacks) verbose_proxy_logger.info( - f"Fallback configured: {data.model} -> {data.fallback_models} (type: {data.fallback_type})" + "Fallback configured: %s -> %s (type: %s)", data.model, data.fallback_models, data.fallback_type ) return FallbackResponse( @@ -182,7 +182,7 @@ async def create_fallback( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error creating fallback: {e}", exc_info=True) + verbose_proxy_logger.error("Error creating fallback: %s", e, exc_info=True) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to create fallback: {e}"}, @@ -239,7 +239,7 @@ async def get_fallback( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error getting fallback: {e}", exc_info=True) + verbose_proxy_logger.error("Error getting fallback: %s", e, exc_info=True) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to get fallback: {e}"}, @@ -339,7 +339,7 @@ async def delete_fallback( # Update the in-memory router configuration setattr(llm_router, fallback_key, updated_fallbacks) - verbose_proxy_logger.info(f"Fallback deleted: {model} (type: {fallback_type})") + verbose_proxy_logger.info("Fallback deleted: %s (type: %s)", model, fallback_type) return FallbackDeleteResponse( model=model, @@ -350,7 +350,7 @@ async def delete_fallback( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error deleting fallback: {e}", exc_info=True) + verbose_proxy_logger.error("Error deleting fallback: %s", e, exc_info=True) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to delete fallback: {e}"}, diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 8e31b1f6e62..0f159000fb2 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -330,7 +330,8 @@ async def _add_user_to_team( except HTTPException as e: if e.status_code == 400 and ("already exists" in str(e) or "doesn't exist" in str(e)): verbose_proxy_logger.debug( - f"litellm.proxy.management_endpoints.internal_user_endpoints.new_user(): User already exists in team - {e}" + "litellm.proxy.management_endpoints.internal_user_endpoints.new_user(): User already exists in team - %s", + e, ) else: verbose_proxy_logger.error( @@ -348,7 +349,8 @@ async def _add_user_to_team( and ProxyErrorTypes.team_member_already_in_team in e.type ): verbose_proxy_logger.debug( - f"litellm.proxy.management_endpoints.internal_user_endpoints.new_user(): User already exists in team - {e}" + "litellm.proxy.management_endpoints.internal_user_endpoints.new_user(): User already exists in team - %s", + e, ) else: verbose_proxy_logger.error( @@ -605,7 +607,7 @@ async def new_user( return new_user_response except Exception as e: - verbose_proxy_logger.exception(f"/user/new: Exception occured - {e}") + verbose_proxy_logger.exception("/user/new: Exception occured - %s", e) raise handle_exception_on_proxy(e) @@ -900,7 +902,7 @@ async def user_info( return response_data except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.user_info(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.user_info(): Exception occured - %s", e) raise handle_exception_on_proxy(e) @@ -1050,7 +1052,7 @@ async def user_info_v2( object_permission=user_data.get("object_permission"), ) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.user_info_v2(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.user_info_v2(): Exception occured - %s", e) raise handle_exception_on_proxy(e) @@ -1233,7 +1235,7 @@ async def _schedule_user_update_audit_log( ) ) except Exception as audit_error: - verbose_proxy_logger.warning(f"Failed to create audit log for user {response.get('user_id')}: {audit_error}") + verbose_proxy_logger.warning("Failed to create audit log for user %s: %s", response.get("user_id"), audit_error) def _check_user_update_authz( @@ -1320,7 +1322,7 @@ async def _invalidate_cached_user_entitlement(user_id: str | None, object_permis try: await user_api_key_cache.async_delete_cache(key=key) except Exception as e: # noqa: BLE001 # a cache we cannot clear still expires; never fail the write - verbose_proxy_logger.warning(f"Failed to invalidate cached entitlement key {key!r}: {e}") + verbose_proxy_logger.warning("Failed to invalidate cached entitlement key %r: %s", key, e) async def _update_single_user_helper( @@ -1569,7 +1571,7 @@ async def user_update( ) return response except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.user_update(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.user_update(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -1618,12 +1620,12 @@ async def bulk_update_processed_users( successful_updates += 1 except Exception as e: verbose_proxy_logger.exception( - f"Failed to update user {user_request.user_id or user_request.user_email}: {e}" + "Failed to update user %s: %s", user_request.user_id or user_request.user_email, e ) # Record failure error_message = str(e) verbose_proxy_logger.error( - f"Failed to update user {user_request.user_id or user_request.user_email}: {error_message}" + "Failed to update user %s: %s", user_request.user_id or user_request.user_email, error_message ) results.append( @@ -1643,7 +1645,7 @@ async def bulk_update_processed_users( failed_updates=failed_updates, ) except Exception as e: - verbose_proxy_logger.exception(f"Failed to update users: {e}") + verbose_proxy_logger.exception("Failed to update users: %s", e) raise HTTPException(status_code=500, detail={"error": str(e)}) @@ -1806,10 +1808,10 @@ async def bulk_user_update( ) ) except Exception as audit_error: - verbose_proxy_logger.warning(f"Failed to create bulk audit log: {audit_error}") + verbose_proxy_logger.warning("Failed to create bulk audit log: %s", audit_error) except Exception as e: - verbose_proxy_logger.exception(f"Failed to perform bulk update: {e}") + verbose_proxy_logger.exception("Failed to perform bulk update: %s", e) # Fall back to individual updates if bulk update fails for user in all_users_in_db: user_update_request = data.user_updates.model_copy() @@ -2133,7 +2135,7 @@ async def get_users( else: user_key_counts = {} - verbose_proxy_logger.debug(f"Total count of users: {total_count}") + verbose_proxy_logger.debug("Total count of users: %s", total_count) # Calculate total pages total_pages = -(-total_count // page_size) # Ceiling division @@ -2593,7 +2595,7 @@ async def ui_view_users( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error searching users: {e}") + verbose_proxy_logger.exception("Error searching users: %s", e) raise HTTPException(status_code=500, detail=f"Error searching users: {e}") @@ -2716,7 +2718,7 @@ async def get_user_daily_activity( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"/spend/daily/analytics: Exception occured - {e}") + verbose_proxy_logger.exception("/spend/daily/analytics: Exception occured - %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to fetch analytics: {e}"}, @@ -2808,7 +2810,7 @@ async def get_user_daily_activity_aggregated( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"/user/daily/activity/aggregated: Exception occured - {e}") + verbose_proxy_logger.exception("/user/daily/activity/aggregated: Exception occured - %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Failed to fetch analytics: {e}"}, diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index d4403b3f5db..f37bc6da23e 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -940,7 +940,7 @@ async def _common_key_generation_helper( data = apply_enterprise_key_management_params(data, team_table) except Exception as e: verbose_proxy_logger.debug( - f"litellm.proxy.proxy_server.generate_key_fn(): Enterprise key management params not applied - {e}" + "litellm.proxy.proxy_server.generate_key_fn(): Enterprise key management params not applied - %s", e ) # TODO: @ishaan-jaff: Migrate all budget tracking to use LiteLLM_BudgetTable @@ -1693,7 +1693,7 @@ async def generate_key_fn( check_db_only=True, ) except Exception as e: - verbose_proxy_logger.debug(f"Error getting team object in `/key/generate`: {e}") + verbose_proxy_logger.debug("Error getting team object in `/key/generate`: %s", e) # For non-admin callers, team must exist (LIT-1884) if not _is_proxy_admin: raise HTTPException( @@ -1732,7 +1732,7 @@ async def generate_key_fn( ) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.generate_key_fn(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.generate_key_fn(): Exception occured - %s", e) raise handle_exception_on_proxy(e) @@ -1866,7 +1866,7 @@ async def generate_service_account_key_fn( check_db_only=True, ) except Exception as e: - verbose_proxy_logger.debug(f"Error getting team object in `/key/generate`: {e}") + verbose_proxy_logger.debug("Error getting team object in `/key/generate`: %s", e) team_table = None if team_table is not None: @@ -1934,7 +1934,9 @@ def prepare_metadata_fields(data: BaseModel, non_default_values: dict, existing_ casted_metadata[k] = v except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.prepare_metadata_fields(): Exception occured - {e}") + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.prepare_metadata_fields(): Exception occured - %s", e + ) non_default_values["metadata"] = encrypt_callback_vars(casted_metadata) return non_default_values @@ -2052,7 +2054,7 @@ async def _handle_update_object_permission( # Add the object_permission_id to data_json if one was created/updated if object_permission_id is not None: data_json["object_permission_id"] = object_permission_id - verbose_proxy_logger.debug(f"updated object_permission_id: {object_permission_id}") + verbose_proxy_logger.debug("updated object_permission_id: %s", object_permission_id) return data_json @@ -2797,7 +2799,7 @@ async def update_key_fn( return {"key": key, **response["data"]} # update based on remaining passed in values except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.update_key_fn(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.update_key_fn(): Exception occured - %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"Authentication Error({e})"), @@ -2935,7 +2937,7 @@ async def bulk_update_keys( ) except Exception as e: - verbose_proxy_logger.exception(f"Failed to update key {key_update_item.key}: {e}") + verbose_proxy_logger.exception("Failed to update key %s: %s", key_update_item.key, e) if isinstance(e, HTTPException): error_detail = e.detail @@ -3175,7 +3177,7 @@ async def bulk_update_team_keys( except Exception as e: # Log the hashed prefix — `token` may be a raw sk-... and ERROR logs persist. - verbose_proxy_logger.exception(f"Failed to update key {db_token[:12]}... in team {data.team_id}: {e}") + verbose_proxy_logger.exception("Failed to update key %s... in team %s: %s", db_token[:12], data.team_id, e) failed_updates.append( _build_failed_team_key_update( token=token, @@ -3305,7 +3307,7 @@ async def delete_key_fn( litellm_changed_by = None ## only allow user to delete keys they own - verbose_proxy_logger.debug(f"user_api_key_dict.user_role: {user_api_key_dict.user_role}") + verbose_proxy_logger.debug("user_api_key_dict.user_role: %s", user_api_key_dict.user_role) num_keys_to_be_deleted = 0 deleted_keys = [] @@ -3338,7 +3340,7 @@ async def delete_key_fn( param="keys", code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) - verbose_proxy_logger.debug(f"/key/delete - deleted_keys={number_deleted_keys}") + verbose_proxy_logger.debug("/key/delete - deleted_keys=%s", number_deleted_keys) try: assert num_keys_to_be_deleted == len(deleted_keys) @@ -3351,7 +3353,7 @@ async def delete_key_fn( ) verbose_proxy_logger.debug( - f"/keys/delete - cache after delete: {user_api_key_cache.in_memory_cache.cache_dict}" + "/keys/delete - cache after delete: %s", user_api_key_cache.in_memory_cache.cache_dict ) asyncio.create_task( @@ -3366,7 +3368,7 @@ async def delete_key_fn( return {"deleted_keys": deleted_keys} except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.delete_key_fn(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.delete_key_fn(): Exception occured - %s", e) raise handle_exception_on_proxy(e) @@ -3906,7 +3908,7 @@ async def generate_key_helper_fn( # If it's not valid JSON/YAML, keep as is or set to empty dict key_data["router_settings"] = {} except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.generate_key_helper_fn(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.generate_key_helper_fn(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise e @@ -4113,7 +4115,7 @@ async def delete_verification_tokens( raise Exception("DB not connected. prisma_client is None") except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.delete_verification_tokens(): Exception occured - {e}" + "litellm.proxy.proxy_server.delete_verification_tokens(): Exception occured - %s", e ) verbose_proxy_logger.debug(traceback.format_exc()) raise e @@ -4385,10 +4387,10 @@ async def _rotate_master_key( }, ) except Exception as e: - verbose_proxy_logger.error(f"Failed to re-encrypt credential {cred.credential_name}: {e}") + verbose_proxy_logger.error("Failed to re-encrypt credential %s: %s", cred.credential_name, e) # Continue with next credential instead of failing entire rotation continue - verbose_proxy_logger.debug(f"Successfully re-encrypted {len(credentials)} credentials with new master key") + verbose_proxy_logger.debug("Successfully re-encrypted %s credentials with new master key", len(credentials)) def _require_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None: @@ -5446,7 +5448,7 @@ async def list_keys( return response except Exception as e: - verbose_proxy_logger.exception(f"Error in list_keys: {e}") + verbose_proxy_logger.exception("Error in list_keys: %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"error({e})"), @@ -5585,8 +5587,12 @@ async def key_aliases( total_pages = -(-total_count // size) if total_count > 0 else 0 verbose_proxy_logger.debug( - f"key_aliases: page={page}, size={size}, search={search!r}, " - f"total_count={total_count}, total_pages={total_pages}" + "key_aliases: page=%s, size=%s, search=%r, total_count=%s, total_pages=%s", + page, + size, + search, + total_count, + total_pages, ) return { @@ -5598,7 +5604,7 @@ async def key_aliases( } except Exception as e: - verbose_proxy_logger.exception(f"Error in key_aliases: {e}") + verbose_proxy_logger.exception("Error in key_aliases: %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"error({e})"), @@ -5779,7 +5785,7 @@ def _build_key_filter_conditions( if expires_filter is not None and expires_filter in VALID_EXPIRES_FILTER_VALUES: where = {"AND": [where, _build_expires_where_clause(expires_filter, datetime.now(timezone.utc))]} - verbose_proxy_logger.debug(f"Filter conditions: {where}") + verbose_proxy_logger.debug("Filter conditions: %s", where) return where @@ -5850,7 +5856,7 @@ async def _list_key_helper( # Calculate skip for pagination skip = (page - 1) * size - verbose_proxy_logger.debug(f"Pagination: skip={skip}, take={size}") + verbose_proxy_logger.debug("Pagination: skip=%s, take=%s", skip, size) order_by: dict[str, str] | None = ( _validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None @@ -5890,7 +5896,7 @@ async def _list_key_helper( include={"object_permission": True}, ) - verbose_proxy_logger.debug(f"Fetched {len(keys)} keys") + verbose_proxy_logger.debug("Fetched %s keys", len(keys)) # Get total count of keys if use_deleted_table: @@ -5902,7 +5908,7 @@ async def _list_key_helper( where=where # type: ignore ) - verbose_proxy_logger.debug(f"Total count of keys: {total_count}") + verbose_proxy_logger.debug("Total count of keys: %s", total_count) # Calculate total pages total_pages = -(-total_count // size) # Ceiling division diff --git a/litellm/proxy/management_endpoints/management_v1/budgets.py b/litellm/proxy/management_endpoints/management_v1/budgets.py index 1bd47a940be..a0afcdc56cc 100644 --- a/litellm/proxy/management_endpoints/management_v1/budgets.py +++ b/litellm/proxy/management_endpoints/management_v1/budgets.py @@ -191,7 +191,7 @@ async def list_budgets( raise except Exception as e: # noqa: BLE001 # a driver error answers as a problem document, not the OpenAI error shape verbose_proxy_logger.exception( - f"litellm.proxy.management_endpoints.management_v1.budgets.list_budgets(): Exception occured - {e}" + "litellm.proxy.management_endpoints.management_v1.budgets.list_budgets(): Exception occured - %s", e ) raise ManagementProblem( ProblemDetail( diff --git a/litellm/proxy/management_endpoints/management_v1/spend_logs.py b/litellm/proxy/management_endpoints/management_v1/spend_logs.py index 403e2760fb9..3614a598393 100644 --- a/litellm/proxy/management_endpoints/management_v1/spend_logs.py +++ b/litellm/proxy/management_endpoints/management_v1/spend_logs.py @@ -187,8 +187,8 @@ async def list_spend_log_end_users( raise except Exception as e: verbose_proxy_logger.exception( - "litellm.proxy.management_endpoints.management_v1.spend_logs.list_spend_log_end_users(): " - f"Exception occured - {e}" + "litellm.proxy.management_endpoints.management_v1.spend_logs.list_spend_log_end_users(): Exception occured - %s", + e, ) raise ManagementProblem( ProblemDetail( diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index da86a7f06f2..6a212dd3bf9 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -94,7 +94,7 @@ DEFAULT_MCP_REGISTRY_VERSION = "1.0.0" try: importlib.import_module("mcp") except ImportError as e: - verbose_logger.debug(f"MCP module not found: {e}") + verbose_logger.debug("MCP module not found: %s", e) MCP_AVAILABLE = False if MCP_AVAILABLE: @@ -399,7 +399,7 @@ if MCP_AVAILABLE: try: encrypted_payload = encrypt_value_helper(payload_json) except Exception as e: - verbose_proxy_logger.debug(f"Failed to encrypt temporary MCP server payload for Redis cache: {e}") + verbose_proxy_logger.debug("Failed to encrypt temporary MCP server payload for Redis cache: %s", e) return if not isinstance(encrypted_payload, str): @@ -413,7 +413,7 @@ if MCP_AVAILABLE: ttl=max(1, ttl_seconds), ) except Exception as e: - verbose_proxy_logger.debug(f"Failed to write temporary MCP server to Redis cache: {e}") + verbose_proxy_logger.debug("Failed to write temporary MCP server to Redis cache: %s", e) async def _get_temporary_mcp_server_from_redis( server_id: str, @@ -435,7 +435,7 @@ if MCP_AVAILABLE: key=f"{TEMPORARY_MCP_SERVER_REDIS_KEY_PREFIX}:{server_id}" ) except Exception as e: - verbose_proxy_logger.debug(f"Failed reading temporary MCP server from Redis cache: {e}") + verbose_proxy_logger.debug("Failed reading temporary MCP server from Redis cache: %s", e) return None if not isinstance(cached_server, str): @@ -454,7 +454,7 @@ if MCP_AVAILABLE: try: loaded = json.loads(decrypted_json) except Exception as e: - verbose_proxy_logger.debug(f"Invalid decrypted temporary MCP payload in Redis cache: {e}") + verbose_proxy_logger.debug("Invalid decrypted temporary MCP payload in Redis cache: %s", e) return None if not isinstance(loaded, dict): return None @@ -463,7 +463,7 @@ if MCP_AVAILABLE: try: return MCPServer.model_validate(payload_dict) except Exception as e: - verbose_proxy_logger.debug(f"Invalid temporary MCP server payload in Redis cache: {e}") + verbose_proxy_logger.debug("Invalid temporary MCP server payload in Redis cache: %s", e) return None async def get_cached_temporary_mcp_server( @@ -814,7 +814,7 @@ if MCP_AVAILABLE: if hasattr(server, "mcp_access_groups") and server.mcp_access_groups: access_groups.update(server.mcp_access_groups) except Exception as e: - verbose_proxy_logger.debug(f"Error getting MCP access groups: {e}") + verbose_proxy_logger.debug("Error getting MCP access groups: %s", e) # Convert to sorted list access_groups_list = sorted(list(access_groups)) @@ -864,7 +864,7 @@ if MCP_AVAILABLE: entry = _build_mcp_registry_entry_for_server(server, base_url) except Exception as e: verbose_proxy_logger.debug( - f"Skipping MCP server {getattr(server, 'server_id', 'unknown')} in registry: {e}" + "Skipping MCP server %s in registry: %s", getattr(server, "server_id", "unknown"), e ) continue registry_servers.append({"server": entry}) @@ -1183,7 +1183,7 @@ if MCP_AVAILABLE: touched_by=user_api_key_dict.user_id or user_api_key_dict.team_id, ) except Exception as e: - verbose_proxy_logger.exception(f"Error registering mcp server: {e}") + verbose_proxy_logger.exception("Error registering mcp server: %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Error registering mcp server: {e}"}, @@ -1398,7 +1398,7 @@ if MCP_AVAILABLE: mcp_server.last_health_check = health_result.last_health_check mcp_server.health_check_error = health_result.health_check_error except Exception as e: - verbose_proxy_logger.debug(f"Error performing health check on server {server_id}: {e}") + verbose_proxy_logger.debug("Error performing health check on server %s: %s", server_id, e) mcp_server.status = "unknown" mcp_server.last_health_check = datetime.now() mcp_server.health_check_error = str(e) @@ -1483,7 +1483,7 @@ if MCP_AVAILABLE: touched_by=user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME, ) except Exception as e: - verbose_proxy_logger.exception(f"Error creating mcp server: {e}") + verbose_proxy_logger.exception("Error creating mcp server: %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Error creating mcp server: {e}"}, @@ -1498,7 +1498,7 @@ if MCP_AVAILABLE: await global_mcp_server_manager.reload_servers_from_database() except Exception as e: verbose_proxy_logger.exception( - f"MCP server {new_mcp_server.server_id} created but in-memory registry refresh failed: {e}" + "MCP server %s created but in-memory registry refresh failed: %s", new_mcp_server.server_id, e ) return _redact_mcp_credentials(new_mcp_server) @@ -1559,7 +1559,7 @@ if MCP_AVAILABLE: ttl_seconds=TEMPORARY_MCP_SERVER_TTL_SECONDS, ) except Exception as e: - verbose_proxy_logger.exception(f"Error caching temporary mcp server: {e}") + verbose_proxy_logger.exception("Error caching temporary mcp server: %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": f"Error caching temporary mcp server: {e}"}, @@ -2566,7 +2566,7 @@ if MCP_AVAILABLE: await proxy_config.save_config(new_config=config) verbose_proxy_logger.debug( - f"Updated public mcp servers to: {litellm.public_mcp_servers} by user: {user_api_key_dict.user_id}" + "Updated public mcp servers to: %s by user: %s", litellm.public_mcp_servers, user_api_key_dict.user_id ) return { @@ -2577,7 +2577,7 @@ if MCP_AVAILABLE: except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error making agent public: {e}") + verbose_proxy_logger.exception("Error making agent public: %s", e) raise HTTPException(status_code=500, detail=str(e)) # --- MCP Discovery --- @@ -2598,7 +2598,7 @@ if MCP_AVAILABLE: with open(_MCP_REGISTRY_PATH, "r") as f: data: dict[str, Any] = json.load(f) except Exception as e: - verbose_proxy_logger.warning(f"Failed to load MCP registry from {_MCP_REGISTRY_PATH}: {e}") + verbose_proxy_logger.warning("Failed to load MCP registry from %s: %s", _MCP_REGISTRY_PATH, e) data = {"servers": []} _mcp_registry_cache = data return data @@ -2685,7 +2685,7 @@ if MCP_AVAILABLE: try: return _load_openapi_registry() except Exception as e: - verbose_proxy_logger.warning(f"Failed to load OpenAPI registry from {_OPENAPI_REGISTRY_PATH}: {e}") + verbose_proxy_logger.warning("Failed to load OpenAPI registry from %s: %s", _OPENAPI_REGISTRY_PATH, e) return {"apis": []} # --------------------------------------------------------------------------- diff --git a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py index b294b2674e4..65a6e3639ec 100644 --- a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py @@ -121,7 +121,7 @@ async def _tag_deployment_with_access_group( where={"model_id": model_id}, data={"model_info": json.dumps(updated_model_info)}, ) - verbose_proxy_logger.debug(f"Updated deployment {model_id} with access group: {access_group}") + verbose_proxy_logger.debug("Updated deployment %s with access group: %s", model_id, access_group) return (model_id, updated_model_info) @@ -175,7 +175,7 @@ async def update_deployments_with_access_group( so callers can verify each one survived the post-write reload """ deployments = await ModelRepository(prisma_client).table.find_many(where={"model_name": {"in": model_names}}) - verbose_proxy_logger.debug(f"Found {len(deployments)} deployments for model_names: {model_names}") + verbose_proxy_logger.debug("Found %s deployments for model_names: %s", len(deployments), model_names) found_names = {deployment.model_name for deployment in deployments} for model_name in model_names: @@ -212,7 +212,7 @@ async def update_specific_deployments_with_access_group( their unique model_id. Returns the (model_id, updated model_info) pair of every deployment actually written. """ - verbose_proxy_logger.debug(f"Updating specific deployment model_ids: {model_ids}") + verbose_proxy_logger.debug("Updating specific deployment model_ids: %s", model_ids) tagged = [ await _tag_deployment_with_access_group( model_id=model_id, @@ -344,7 +344,7 @@ async def create_model_group( prisma_client, ) - verbose_proxy_logger.debug(f"Creating access group: {data.access_group} with models: {data.model_names}") + verbose_proxy_logger.debug("Creating access group: %s with models: %s", data.access_group, data.model_names) # Validation: Check if access_group is provided if not data.access_group or not data.access_group.strip(): @@ -426,7 +426,7 @@ async def create_model_group( ) verbose_proxy_logger.info( - f"Successfully created access group '{data.access_group}' with {models_updated} models updated" + "Successfully created access group '%s' with %s models updated", data.access_group, models_updated ) return NewModelGroupResponse( @@ -439,7 +439,7 @@ async def create_model_group( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error creating access group '{data.access_group}': {e}") + verbose_proxy_logger.exception("Error creating access group '%s': %s", data.access_group, e) raise HTTPException( status_code=500, detail={"error": f"Failed to create access group: {e}"}, @@ -489,7 +489,7 @@ async def list_access_groups( return ListAccessGroupsResponse(access_groups=access_groups_list) except Exception as e: - verbose_proxy_logger.exception(f"Error listing access groups: {e}") + verbose_proxy_logger.exception("Error listing access groups: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to list access groups: {e}"}, @@ -546,7 +546,7 @@ async def get_access_group_info( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error getting access group info for '{access_group}': {e}") + verbose_proxy_logger.exception("Error getting access group info for '%s': %s", access_group, e) raise HTTPException( status_code=500, detail={"error": f"Failed to get access group info: {e}"}, @@ -600,7 +600,7 @@ async def update_access_group( detail={"error": "Database not connected."}, ) - verbose_proxy_logger.debug(f"Updating access group: {access_group} with models: {data.model_names}") + verbose_proxy_logger.debug("Updating access group: %s with models: %s", access_group, data.model_names) # Validation: Check that at least one of model_names or model_ids is provided has_model_names = data.model_names and len(data.model_names) > 0 @@ -686,7 +686,7 @@ async def update_access_group( ) verbose_proxy_logger.info( - f"Successfully updated access group '{access_group}' with {models_updated} models updated" + "Successfully updated access group '%s' with %s models updated", access_group, models_updated ) return NewModelGroupResponse( @@ -699,7 +699,7 @@ async def update_access_group( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error updating access group '{access_group}': {e}") + verbose_proxy_logger.exception("Error updating access group '%s': %s", access_group, e) raise HTTPException( status_code=500, detail={"error": f"Failed to update access group: {e}"}, @@ -744,7 +744,7 @@ async def delete_access_group( detail={"error": "Database not connected."}, ) - verbose_proxy_logger.debug(f"Deleting access group: {access_group}") + verbose_proxy_logger.debug("Deleting access group: %s", access_group) # Validation: Check if access group exists try: @@ -788,7 +788,7 @@ async def delete_access_group( ) verbose_proxy_logger.info( - f"Successfully deleted access group '{access_group}' from {models_updated} deployments" + "Successfully deleted access group '%s' from %s deployments", access_group, models_updated ) return DeleteModelGroupResponse( @@ -800,7 +800,7 @@ async def delete_access_group( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error deleting access group '{access_group}': {e}") + verbose_proxy_logger.exception("Error deleting access group '%s': %s", access_group, e) raise HTTPException( status_code=500, detail={"error": f"Failed to delete access group: {e}"}, diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 475c5813cdd..aa5312d58cb 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -365,7 +365,7 @@ async def patch_model( return updated_model except Exception as e: - verbose_proxy_logger.exception(f"Error in patch_model: {e}") + verbose_proxy_logger.exception("Error in patch_model: %s", e) if isinstance(e, (HTTPException, ProxyException)): raise e @@ -472,7 +472,7 @@ async def _set_model_blocked_status( return updated_model except Exception as e: - verbose_proxy_logger.exception(f"Error in model {action}: {e}") + verbose_proxy_logger.exception("Error in model %s: %s", action, e) if isinstance(e, (HTTPException, ProxyException)): raise e @@ -1233,7 +1233,7 @@ async def delete_model( ) except Exception as e: - verbose_proxy_logger.exception(f"Failed to delete model. Due to error - {e}") + verbose_proxy_logger.exception("Failed to delete model. Due to error - %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"Authentication Error({e})"), @@ -1399,7 +1399,7 @@ async def add_new_model( passed_model_info=model_params.model_info, ) except Exception as e: - verbose_proxy_logger.exception(f"Exception in add_new_model: {e}") + verbose_proxy_logger.exception("Exception in add_new_model: %s", e) else: raise HTTPException( @@ -1439,7 +1439,7 @@ async def add_new_model( return model_response except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.add_new_model(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.add_new_model(): Exception occured - %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"Authentication Error({e})"), @@ -1592,7 +1592,7 @@ async def update_model( return model_response except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.update_model(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.update_model(): Exception occured - %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"Authentication Error({e})"), @@ -1675,7 +1675,7 @@ async def update_public_model_groups( litellm.public_model_groups = request.model_groups verbose_proxy_logger.debug( - f"Updated public model groups to: {request.model_groups} by user: {user_api_key_dict.user_id}" + "Updated public model groups to: %s by user: %s", request.model_groups, user_api_key_dict.user_id ) return { @@ -1685,7 +1685,7 @@ async def update_public_model_groups( } except Exception as e: - verbose_proxy_logger.exception(f"Error updating public model groups: {e}") + verbose_proxy_logger.exception("Error updating public model groups: %s", e) if isinstance(e, HTTPException): raise e @@ -1743,7 +1743,7 @@ async def update_useful_links( litellm.public_model_groups_links = request.useful_links verbose_proxy_logger.debug( - f"Updated useful links to: {request.useful_links} by user: {user_api_key_dict.user_id}" + "Updated useful links to: %s by user: %s", request.useful_links, user_api_key_dict.user_id ) return { @@ -1753,7 +1753,7 @@ async def update_useful_links( } except Exception as e: - verbose_proxy_logger.exception(f"Error updating public model groups: {e}") + verbose_proxy_logger.exception("Error updating public model groups: %s", e) if isinstance(e, HTTPException): raise e @@ -1976,9 +1976,9 @@ async def clear_cache() -> frozenset[str] | None: ) verbose_proxy_logger.debug( - f"Cleared {len(db_model_ids)} DB models, preserved {len(config_models)} config models" + "Cleared %s DB models, preserved %s config models", len(db_model_ids), len(config_models) ) return still_desired_ids except Exception as e: - verbose_proxy_logger.exception(f"Failed to clear cache and reload models. Due to error - {e}") + verbose_proxy_logger.exception("Failed to clear cache and reload models. Due to error - %s", e) return None diff --git a/litellm/proxy/management_endpoints/organization_endpoints.py b/litellm/proxy/management_endpoints/organization_endpoints.py index 949c35e4182..66cf111324a 100644 --- a/litellm/proxy/management_endpoints/organization_endpoints.py +++ b/litellm/proxy/management_endpoints/organization_endpoints.py @@ -498,7 +498,7 @@ async def new_organization( new_organization_row = _STR_OBJECT_DICT_ADAPTER.validate_python( prisma_client.jsonify_object(organization_row.json(exclude_none=True)) ) - verbose_proxy_logger.info(f"new_organization_row: {json.dumps(new_organization_row, indent=2)}") + verbose_proxy_logger.info("new_organization_row: %s", json.dumps(new_organization_row, indent=2)) response = await _table(OrganizationRepository(prisma_client)).create( data={ **new_organization_row, @@ -1258,7 +1258,7 @@ async def organization_member_add( updated_organization_memberships=updated_organization_memberships, ) except Exception as e: - verbose_proxy_logger.exception(f"Error adding member to organization: {e}") + verbose_proxy_logger.exception("Error adding member to organization: %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"Authentication Error({e})"), @@ -1444,7 +1444,7 @@ async def organization_member_update( ) return final_organization_membership_pydantic except Exception as e: - verbose_proxy_logger.exception(f"Error updating member in organization: {e}") + verbose_proxy_logger.exception("Error updating member in organization: %s", e) raise e @@ -1493,7 +1493,7 @@ async def organization_member_delete( return member_to_delete except Exception as e: - verbose_proxy_logger.exception(f"Error deleting member from organization: {e}") + verbose_proxy_logger.exception("Error deleting member from organization: %s", e) raise e diff --git a/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py b/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py index fb139902978..a55e32446f5 100644 --- a/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py +++ b/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py @@ -432,7 +432,7 @@ async def validate_policy( from litellm.proxy.policy_engine.policy_validator import PolicyValidator from litellm.proxy.proxy_server import prisma_client - verbose_proxy_logger.debug(f"Validating policy configuration with {len(data.policies)} policies") + verbose_proxy_logger.debug("Validating policy configuration with %s policies", len(data.policies)) validator = PolicyValidator(prisma_client=prisma_client) diff --git a/litellm/proxy/management_endpoints/router_settings_endpoints.py b/litellm/proxy/management_endpoints/router_settings_endpoints.py index 0adc0610c60..a4d4e2fe5cb 100644 --- a/litellm/proxy/management_endpoints/router_settings_endpoints.py +++ b/litellm/proxy/management_endpoints/router_settings_endpoints.py @@ -120,7 +120,7 @@ async def get_router_settings( routing_strategy_descriptions=ROUTING_STRATEGY_DESCRIPTIONS, ) except Exception as e: - verbose_proxy_logger.error(f"Error fetching router settings: {e}") + verbose_proxy_logger.error("Error fetching router settings: %s", e) raise @@ -168,5 +168,5 @@ async def get_router_fields( routing_strategy_descriptions=ROUTING_STRATEGY_DESCRIPTIONS, ) except Exception as e: - verbose_proxy_logger.error(f"Error fetching router fields: {e}") + verbose_proxy_logger.error("Error fetching router fields: %s", e) raise diff --git a/litellm/proxy/management_endpoints/scim/scim_v2.py b/litellm/proxy/management_endpoints/scim/scim_v2.py index b3140ec911e..4437bfcbb1c 100644 --- a/litellm/proxy/management_endpoints/scim/scim_v2.py +++ b/litellm/proxy/management_endpoints/scim/scim_v2.py @@ -365,7 +365,7 @@ async def _get_scim_upsert_user_setting() -> bool: # Default to True if not set (backward compatibility) return bool(scim_upsert_user) except Exception as e: - verbose_proxy_logger.warning(f"Error reading scim_upsert_user setting, defaulting to True: {e}") + verbose_proxy_logger.warning("Error reading scim_upsert_user setting, defaulting to True: %s", e) # Default to True for backward compatibility return True @@ -401,7 +401,7 @@ async def _get_scim_admin_group() -> str | None: litellm_settings = config.get("litellm_settings", {}) or {} return litellm_settings.get("scim_admin_group") or None except Exception as e: - verbose_proxy_logger.warning(f"Error reading scim_admin_group setting, defaulting to None: {e}") + verbose_proxy_logger.warning("Error reading scim_admin_group setting, defaulting to None: %s", e) return None @@ -882,11 +882,11 @@ async def _create_user_if_not_exists(user_id: str, created_via: str = "scim_grou ) created_user = await new_user(data=new_user_request) - verbose_proxy_logger.info(f"Created user {user_id} via {created_via}") + verbose_proxy_logger.info("Created user %s via %s", user_id, created_via) return created_user except Exception as e: - verbose_proxy_logger.exception(f"Failed to create user {user_id}: {e}") + verbose_proxy_logger.exception("Failed to create user %s: %s", user_id, e) return None @@ -1886,15 +1886,15 @@ async def patch_team_membership( except ProxyException as e: # Handle duplicate membership gracefully - this is idempotent if e.type == ProxyErrorTypes.team_member_already_in_team: - verbose_proxy_logger.debug(f"User {user_id} is already in team {_team_id}, skipping add") + verbose_proxy_logger.debug("User %s is already in team %s, skipping add", user_id, _team_id) elif raise_on_error: raise else: - verbose_proxy_logger.exception(f"Error adding user to team {_team_id}: {e}") + verbose_proxy_logger.exception("Error adding user to team %s: %s", _team_id, e) except Exception as e: if raise_on_error: raise - verbose_proxy_logger.exception(f"Error adding user to team {_team_id}: {e}") + verbose_proxy_logger.exception("Error adding user to team %s: %s", _team_id, e) for _team_id in teams_ids_to_remove_user_from: try: @@ -1904,15 +1904,15 @@ async def patch_team_membership( ) except HTTPException as e: if _is_user_not_in_team_error(e): - verbose_proxy_logger.debug(f"User {user_id} is not in team {_team_id}, skipping remove") + verbose_proxy_logger.debug("User %s is not in team %s, skipping remove", user_id, _team_id) elif raise_on_error: raise else: - verbose_proxy_logger.exception(f"Error removing user from team {_team_id}: {e}") + verbose_proxy_logger.exception("Error removing user from team %s: %s", _team_id, e) except Exception as e: if raise_on_error: raise - verbose_proxy_logger.exception(f"Error removing user from team {_team_id}: {e}") + verbose_proxy_logger.exception("Error removing user from team %s: %s", _team_id, e) return True @@ -2045,7 +2045,7 @@ async def get_groups( # team creation, so reading it here would report an empty member # list to the IdP and trigger repeated re-provisioning. members = await _get_team_members_display(await _get_team_member_user_ids_from_team(team)) - verbose_proxy_logger.debug(f"SCIM GET GROUPS members: {members}") + verbose_proxy_logger.debug("SCIM GET GROUPS members: %s", members) team_alias = getattr(team, "team_alias", team.team_id) team_created_at = team.created_at.isoformat() if team.created_at else None team_updated_at = team.updated_at.isoformat() if team.updated_at else None @@ -2063,7 +2063,7 @@ async def get_groups( ) scim_groups.append(scim_group) - verbose_proxy_logger.debug(f"SCIM GET GROUPS response: {scim_groups}") + verbose_proxy_logger.debug("SCIM GET GROUPS response: %s", scim_groups) return SCIMListResponse( totalResults=total_count, startIndex=startIndex, @@ -2092,7 +2092,7 @@ async def get_group( team = await _check_team_exists(group_id) scim_group = await ScimTransformations.transform_litellm_team_to_scim_group(team) - verbose_proxy_logger.debug(f"SCIM GET GROUP response: {scim_group}") + verbose_proxy_logger.debug("SCIM GET GROUP response: %s", scim_group) return scim_group except Exception as e: @@ -2178,8 +2178,8 @@ async def update_group( # Extract and validate group members (all users must exist) member_result = await _extract_group_member_ids(group) - verbose_proxy_logger.debug(f"SCIM PUT GROUP all_member_ids: {member_result.all_member_ids}") - verbose_proxy_logger.debug(f"SCIM PUT GROUP created_users: {len(member_result.created_users)}") + verbose_proxy_logger.debug("SCIM PUT GROUP all_member_ids: %s", member_result.all_member_ids) + verbose_proxy_logger.debug("SCIM PUT GROUP created_users: %s", len(member_result.created_users)) # Prepare update data existing_metadata = existing_team.metadata if existing_team.metadata else {} @@ -2202,9 +2202,9 @@ async def update_group( # Handle user-team relationship changes current_members = set(await _get_team_member_user_ids_from_team(existing_team)) - verbose_proxy_logger.debug(f"SCIM PUT GROUP current_members: {current_members}") + verbose_proxy_logger.debug("SCIM PUT GROUP current_members: %s", current_members) final_members = set(member_result.all_member_ids) - verbose_proxy_logger.debug(f"SCIM PUT GROUP final_members: {final_members}") + verbose_proxy_logger.debug("SCIM PUT GROUP final_members: %s", final_members) await _handle_group_membership_changes( group_id=group_id, @@ -2380,8 +2380,8 @@ async def _handle_group_membership_changes(group_id: str, current_members: set[s members_to_add = final_members - current_members members_to_remove = current_members - final_members - verbose_proxy_logger.debug(f"members_to_add: {members_to_add}") - verbose_proxy_logger.debug(f"members_to_remove: {members_to_remove}") + verbose_proxy_logger.debug("members_to_add: %s", members_to_add) + verbose_proxy_logger.debug("members_to_remove: %s", members_to_remove) # Use existing helper functions for team membership changes for member_id in members_to_add: diff --git a/litellm/proxy/management_endpoints/sso/custom_microsoft_sso.py b/litellm/proxy/management_endpoints/sso/custom_microsoft_sso.py index 2f900f9b6b6..07108712dbb 100644 --- a/litellm/proxy/management_endpoints/sso/custom_microsoft_sso.py +++ b/litellm/proxy/management_endpoints/sso/custom_microsoft_sso.py @@ -68,10 +68,10 @@ class CustomMicrosoftSSO(MicrosoftSSO): if custom_authorization_endpoint or custom_token_endpoint or custom_userinfo_endpoint: verbose_proxy_logger.debug( - f"Using custom Microsoft SSO endpoints - " - f"authorization: {authorization_endpoint}, " - f"token: {token_endpoint}, " - f"userinfo: {userinfo_endpoint}" + "Using custom Microsoft SSO endpoints - authorization: %s, token: %s, userinfo: %s", + authorization_endpoint, + token_endpoint, + userinfo_endpoint, ) return DiscoveryDocument( diff --git a/litellm/proxy/management_endpoints/sso/saml_sso.py b/litellm/proxy/management_endpoints/sso/saml_sso.py index 37b641ca123..c73c57702e6 100644 --- a/litellm/proxy/management_endpoints/sso/saml_sso.py +++ b/litellm/proxy/management_endpoints/sso/saml_sso.py @@ -449,7 +449,9 @@ class SAMLAuthHandler: display_name = " ".join(part for part in (first_name, last_name) if part) or email - verbose_proxy_logger.info(f"SAML login: subject={user_id}, email={email}, attributes={list(attributes.keys())}") + verbose_proxy_logger.info( + "SAML login: subject=%s, email=%s, attributes=%s", user_id, email, list(attributes.keys()) + ) try: return CustomOpenID( diff --git a/litellm/proxy/management_endpoints/tag_management_endpoints.py b/litellm/proxy/management_endpoints/tag_management_endpoints.py index 8e701fa9e20..d6db8dc05cd 100644 --- a/litellm/proxy/management_endpoints/tag_management_endpoints.py +++ b/litellm/proxy/management_endpoints/tag_management_endpoints.py @@ -201,7 +201,7 @@ async def _get_model_names(prisma_client: "PrismaClient", model_ids: Sequence[st models = await _table(ModelRepository(prisma_client)).find_many(where={"model_id": {"in": model_ids}}) return {model.model_id: model.model_name for model in models} except Exception as e: - verbose_proxy_logger.error(f"Error getting model names: {e}") + verbose_proxy_logger.error("Error getting model names: %s", e) return {} @@ -331,7 +331,7 @@ async def new_tag( "tag": tag_config, } except Exception as e: - verbose_proxy_logger.exception(f"Error creating tag: {e}") + verbose_proxy_logger.exception("Error creating tag: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -372,7 +372,7 @@ async def _add_tag_to_deployment(deployment: "Deployment", tag: str): data={"litellm_params": json.dumps(existing_params)}, ) except Exception as e: - verbose_proxy_logger.exception(f"Error adding tag to deployment: {e}") + verbose_proxy_logger.exception("Error adding tag to deployment: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -461,7 +461,7 @@ async def update_tag( "tag": tag_config, } except Exception as e: - verbose_proxy_logger.exception(f"Error updating tag: {e}") + verbose_proxy_logger.exception("Error updating tag: %s", e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 32b22dd6ade..dd20dd97e00 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -354,7 +354,7 @@ async def add_team_callbacks( except ProxyException as e: raise e except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.add_team_callbacks(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.add_team_callbacks(): Exception occured - %s", e) raise ProxyException( message="Internal Server Error, " + str(e), type=ProxyErrorTypes.internal_server_error.value, @@ -492,7 +492,7 @@ async def disable_team_logging( except ProxyException: raise except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.disable_team_logging(): Exception occurred - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.disable_team_logging(): Exception occurred - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) raise ProxyException( message="Internal Server Error, " + str(e), @@ -585,7 +585,7 @@ async def get_team_callbacks( except ProxyException: raise except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.get_team_callbacks(): Exception occurred - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.get_team_callbacks(): Exception occurred - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index bdd7e18a850..61c3fe4505f 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -465,7 +465,11 @@ class TeamMemberBudgetHandler: user_api_key_dict=user_api_key_dict, ) verbose_proxy_logger.info( - f"Updated team member budget table: {budget_row.budget_id}, with team_member_budget={team_member_budget}, team_member_rpm_limit={team_member_rpm_limit}, team_member_tpm_limit={team_member_tpm_limit}" + "Updated team member budget table: %s, with team_member_budget=%s, team_member_rpm_limit=%s, team_member_tpm_limit=%s", + budget_row.budget_id, + team_member_budget, + team_member_rpm_limit, + team_member_tpm_limit, ) if updated_kv.get("metadata") is None: updated_kv["metadata"] = {} @@ -2241,7 +2245,7 @@ async def handle_update_object_permission(data_json: dict, existing_team_row: Li # Add the object_permission_id to data_json if one was created/updated if object_permission_id is not None: data_json["object_permission_id"] = object_permission_id - verbose_proxy_logger.debug(f"updated object_permission_id: {object_permission_id}") + verbose_proxy_logger.debug("updated object_permission_id: %s", object_permission_id) return data_json @@ -2333,7 +2337,9 @@ def team_member_add_duplication_check( ) elif len(invalid_team_members) > 0: verbose_proxy_logger.info( - f"Some users are already in team. Existing members={existing_team_row.members_with_roles}. Duplicate members={invalid_team_members}", + "Some users are already in team. Existing members=%s. Duplicate members=%s", + existing_team_row.members_with_roles, + invalid_team_members, ) @@ -3837,7 +3843,7 @@ async def _add_team_member_budget_table( team_info_response_object.team_member_budget_table = team_budget except Exception: verbose_proxy_logger.info( - f"Team member budget table not found, passed team_member_budget_id={team_member_budget_id}" + "Team member budget table not found, passed team_member_budget_id=%s", team_member_budget_id ) return team_info_response_object @@ -3975,7 +3981,9 @@ async def team_info( except Exception as e: verbose_proxy_logger.error( - f"litellm.proxy.management_endpoints.team_endpoints.py::team_info - Exception occurred - {e}\n{traceback.format_exc()}" + "litellm.proxy.management_endpoints.team_endpoints.py::team_info - Exception occurred - %s\n%s", + e, + traceback.format_exc(), ) if isinstance(e, HTTPException): raise ProxyException( @@ -4915,7 +4923,7 @@ async def get_paginated_teams( ) return teams, total_count except Exception as e: - verbose_proxy_logger.exception(f"[Non-Blocking] Error getting paginated teams: {e}") + verbose_proxy_logger.exception("[Non-Blocking] Error getting paginated teams: %s", e) return [], 0 diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index bc05f72ae14..a2e880eab5f 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -443,7 +443,7 @@ def build_cli_sso_attribution_metadata( metadata: dict[str, Any] = {} for source_claim, dest_key in claim_map: if not _is_safe_cli_sso_metadata_dest_key(dest_key): - verbose_proxy_logger.debug(f"Skipping unsafe CLI SSO metadata destination key: {dest_key}") + verbose_proxy_logger.debug("Skipping unsafe CLI SSO metadata destination key: %s", dest_key) continue raw_value = _extract_sso_claim_value(result=result, claim_path=source_claim) @@ -504,11 +504,12 @@ async def _persist_cli_sso_user_metadata( data={"metadata": merged_metadata}, ) verbose_proxy_logger.info( - f"Persisted CLI SSO attribution metadata for user {user_id}: " - f"{list(_flatten_cli_sso_metadata_for_poll(attribution_metadata).keys())}" + "Persisted CLI SSO attribution metadata for user %s: %s", + user_id, + list(_flatten_cli_sso_metadata_for_poll(attribution_metadata).keys()), ) except Exception as e: - verbose_proxy_logger.error(f"Failed to persist CLI SSO attribution metadata for user {user_id}: {e}") + verbose_proxy_logger.error("Failed to persist CLI SSO attribution metadata for user %s: %s", user_id, e) def _cli_poll_attribution_metadata_from_session( @@ -745,13 +746,15 @@ def determine_role_from_groups( role_groups = role_mappings.roles[role] if isinstance(role_groups, list) and user_groups_set.intersection(set(role_groups)): verbose_proxy_logger.debug( - f"User groups {user_groups} matched role '{role.value}' via groups: {role_groups}" + "User groups %s matched role '%s' via groups: %s", user_groups, role.value, role_groups ) return role # No matching groups found, return default_role verbose_proxy_logger.debug( - f"User groups {user_groups} did not match any role mappings, using default_role: {role_mappings.default_role}" + "User groups %s did not match any role mappings, using default_role: %s", + user_groups, + role_mappings.default_role, ) return role_mappings.default_role @@ -827,7 +830,7 @@ def process_sso_jwt_access_token( if user_groups: user_role = determine_role_from_groups(user_groups, role_mappings) verbose_proxy_logger.debug( - f"Determined role '{user_role}' from access token groups '{user_groups}' using role_mappings" + "Determined role '%s' from access token groups '%s' using role_mappings", user_role, user_groups ) elif role_mappings.default_role: user_role = role_mappings.default_role @@ -839,7 +842,7 @@ def process_sso_jwt_access_token( if user_role_from_token is not None: user_role = get_litellm_user_role(user_role_from_token) verbose_proxy_logger.debug( - f"Extracted role '{user_role}' from access token field '{generic_user_role_attribute_name}'" + "Extracted role '%s' from access token field '%s'", user_role, generic_user_role_attribute_name ) if user_role is not None: @@ -847,7 +850,7 @@ def process_sso_jwt_access_token( result["user_role"] = user_role else: setattr(result, "user_role", user_role) - verbose_proxy_logger.debug(f"Set user_role='{user_role}' from JWT access token") + verbose_proxy_logger.debug("Set user_role='%s' from JWT access token", user_role) return access_token_payload @@ -976,7 +979,7 @@ async def google_login( ) is True ): - verbose_proxy_logger.info(f"Redirecting to SSO login for {redirect_url}") + verbose_proxy_logger.info("Redirecting to SSO login for %s", redirect_url) sso_redirect = await SSOAuthenticationHandler.get_sso_login_redirect( redirect_url=redirect_url, microsoft_client_id=microsoft_client_id, @@ -1031,7 +1034,9 @@ def generic_response_convertor( generic_user_extra_attributes = os.getenv("GENERIC_USER_EXTRA_ATTRIBUTES", None) verbose_proxy_logger.debug( - f" generic_user_id_attribute_name: {generic_user_id_attribute_name}\n generic_user_email_attribute_name: {generic_user_email_attribute_name}" + " generic_user_id_attribute_name: %s\n generic_user_email_attribute_name: %s", + generic_user_id_attribute_name, + generic_user_email_attribute_name, ) all_teams = [] @@ -1048,7 +1053,9 @@ def generic_response_convertor( if team_ids_from_db_mapping: all_teams.extend(team_ids_from_db_mapping) verbose_proxy_logger.debug( - f"Loaded team_ids from DB team_mappings.team_ids_jwt_field='{team_mappings.team_ids_jwt_field}': {team_ids_from_db_mapping}" + "Loaded team_ids from DB team_mappings.team_ids_jwt_field='%s': %s", + team_mappings.team_ids_jwt_field, + team_ids_from_db_mapping, ) else: team_ids = jwt_handler.get_all_jwt_team_ids(cast(dict, response)) @@ -1080,13 +1087,15 @@ def generic_response_convertor( if user_groups: user_role = determine_role_from_groups(user_groups, role_mappings) verbose_proxy_logger.debug( - f"Determined role '{user_role.value if user_role else None}' from groups '{user_groups}' using role_mappings" + "Determined role '%s' from groups '%s' using role_mappings", + user_role.value if user_role else None, + user_groups, ) else: # No groups found, use default_role user_role = role_mappings.default_role verbose_proxy_logger.debug( - f"No groups found in '{group_claim}', using default_role: {role_mappings.default_role}" + "No groups found in '%s', using default_role: %s", group_claim, role_mappings.default_role ) # Fallback to existing logic if role_mappings not used @@ -1097,7 +1106,9 @@ def generic_response_convertor( if role is not None: user_role = role verbose_proxy_logger.debug( - f"Found valid LitellmUserRoles '{role.value}' from SSO attribute '{generic_user_role_attribute_name}'" + "Found valid LitellmUserRoles '%s' from SSO attribute '%s'", + role.value, + generic_user_role_attribute_name, ) # Build extra_fields dict from GENERIC_USER_EXTRA_ATTRIBUTES if specified @@ -1163,9 +1174,12 @@ def _setup_generic_sso_env_vars( ) verbose_proxy_logger.debug( - f"authorization_endpoint: {generic_authorization_endpoint}\ntoken_endpoint: {generic_token_endpoint}\nuserinfo_endpoint: {generic_userinfo_endpoint}" + "authorization_endpoint: %s\ntoken_endpoint: %s\nuserinfo_endpoint: %s", + generic_authorization_endpoint, + generic_token_endpoint, + generic_userinfo_endpoint, ) - verbose_proxy_logger.debug(f"GENERIC_REDIRECT_URI: {redirect_url}\nGENERIC_CLIENT_ID: {generic_client_id}\n") + verbose_proxy_logger.debug("GENERIC_REDIRECT_URI: %s\nGENERIC_CLIENT_ID: %s\n", redirect_url, generic_client_id) return ( generic_client_secret, @@ -1201,11 +1215,11 @@ async def _setup_team_mappings() -> Optional["TeamMappings"]: if team_mappings and team_mappings.team_ids_jwt_field: verbose_proxy_logger.debug( - f"Loaded team_mappings with team_ids_jwt_field: '{team_mappings.team_ids_jwt_field}'" + "Loaded team_mappings with team_ids_jwt_field: '%s'", team_mappings.team_ids_jwt_field ) except Exception as e: verbose_proxy_logger.debug( - f"Could not load team_mappings from database: {e}. Continuing with config-based team mapping." + "Could not load team_mappings from database: %s. Continuing with config-based team mapping.", e ) return team_mappings @@ -1232,10 +1246,10 @@ async def _setup_role_mappings() -> Optional["RoleMappings"]: role_mappings = role_mappings_data if role_mappings: - verbose_proxy_logger.debug(f"Loaded role_mappings for provider '{role_mappings.provider}'") + verbose_proxy_logger.debug("Loaded role_mappings for provider '%s'", role_mappings.provider) except Exception as e: verbose_proxy_logger.debug( - f"Could not load role_mappings from database: {e}. Continuing with existing role logic." + "Could not load role_mappings from database: %s. Continuing with existing role logic.", e ) generic_role_mappings = os.getenv("GENERIC_ROLE_MAPPINGS_ROLES", None) @@ -1257,12 +1271,12 @@ async def _setup_role_mappings() -> Optional["RoleMappings"]: role_mappings = RoleMappings(**role_mappings_data) verbose_proxy_logger.debug( - f"Loaded role_mappings from environments for provider '{role_mappings.provider}'." + "Loaded role_mappings from environments for provider '%s'.", role_mappings.provider ) return role_mappings except TypeError as e: verbose_proxy_logger.warning( - f"Error decoding role mappings from environment variables: {e}. Continuing with existing role logic." + "Error decoding role mappings from environment variables: %s. Continuing with existing role logic.", e ) return role_mappings @@ -1534,7 +1548,7 @@ async def create_team_member_add_task(team_id, user_info): user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), ) except Exception as e: - verbose_proxy_logger.debug(f"[Non-Blocking] Error trying to add sso user to db: {e}") + verbose_proxy_logger.debug("[Non-Blocking] Error trying to add sso user to db: %s", e) async def add_missing_team_member(user_info: NewUserResponse | LiteLLM_UserTable, sso_teams: list[str]): @@ -1552,7 +1566,7 @@ async def add_missing_team_member(user_info: NewUserResponse | LiteLLM_UserTable try: await asyncio.gather(*tasks) except Exception as e: - verbose_proxy_logger.debug(f"[Non-Blocking] Error trying to add sso user to db: {e}") + verbose_proxy_logger.debug("[Non-Blocking] Error trying to add sso user to db: %s", e) def get_disabled_non_admin_personal_key_creation(): @@ -1583,7 +1597,7 @@ async def get_existing_user_info_from_db( sso_user_id=user_id, ) except Exception as e: - verbose_proxy_logger.debug(f"Error getting user object: {e}") + verbose_proxy_logger.debug("Error getting user object: %s", e) user_info = None return user_info @@ -1629,7 +1643,7 @@ async def get_user_info_from_db( break verbose_proxy_logger.debug( - f"user_info: {user_info}; litellm.default_internal_user_params: {litellm.default_internal_user_params}" + "user_info: %s; litellm.default_internal_user_params: %s", user_info, litellm.default_internal_user_params ) # Upsert SSO User to LiteLLM DB @@ -1648,7 +1662,7 @@ async def get_user_info_from_db( return user_info except Exception as e: - verbose_proxy_logger.exception(f"[Non-Blocking] Error trying to add sso user to db: {e}") + verbose_proxy_logger.exception("[Non-Blocking] Error trying to add sso user to db: %s", e) return None @@ -1660,8 +1674,8 @@ def _should_use_role_from_sso_response(sso_role: str | None) -> bool: if not is_valid_litellm_user_role(sso_role): verbose_proxy_logger.debug( - f"SSO role '{sso_role}' is not a valid LiteLLM user role. " - "Ignoring role from SSO response. See LitellmUserRoles enum for valid roles." + "SSO role '%s' is not a valid LiteLLM user role. Ignoring role from SSO response. See LitellmUserRoles enum for valid roles.", + sso_role, ) return False return True @@ -1694,7 +1708,7 @@ def _build_sso_user_update_data( # Only include if it's a valid LiteLLM role if _should_use_role_from_sso_response(sso_role_str): update_data["user_role"] = sso_role_str - verbose_proxy_logger.info(f"Updating user {user_id} role from SSO: {sso_role_str}") + verbose_proxy_logger.info("Updating user %s role from SSO: %s", user_id, sso_role_str) return update_data @@ -1726,7 +1740,7 @@ async def _sync_user_role_from_jwt_role_map( if mapped_role is None: return - verbose_proxy_logger.info(f"SSO jwt_litellm_role_map matched role: {mapped_role.value}") + verbose_proxy_logger.info("SSO jwt_litellm_role_map matched role: %s", mapped_role.value) # Update user_defined_values so downstream code uses the mapped role if user_defined_values is not None: @@ -1762,7 +1776,7 @@ def apply_user_info_values_to_sso_user_defined_values( if _should_use_role_from_sso_response(sso_role): # SSO provided a valid role, keep it and log that we're using it - verbose_proxy_logger.info(f"Using SSO role: {sso_role} (DB role was: {db_role})") + verbose_proxy_logger.info("Using SSO role: %s (DB role was: %s)", sso_role, db_role) else: # SSO didn't provide a valid role, fall back to DB role or default if user_info is None or user_info.user_role is None: @@ -1770,7 +1784,7 @@ def apply_user_info_values_to_sso_user_defined_values( verbose_proxy_logger.debug("No SSO or DB role found, using default: INTERNAL_USER_VIEW_ONLY") else: user_defined_values["user_role"] = user_info.user_role - verbose_proxy_logger.debug(f"Using DB role: {user_info.user_role}") + verbose_proxy_logger.debug("Using DB role: %s", user_info.user_role) # Preserve the user's existing models from the database if user_info is not None and hasattr(user_info, "models") and user_info.models: @@ -1803,13 +1817,13 @@ async def check_and_update_if_proxy_admin_id(user_role: str, user_id: str, prism @router.get("/sso/callback", tags=["experimental"], include_in_schema=False) async def auth_callback(request: Request, state: str | None = None): """Verify login""" - verbose_proxy_logger.info(f"Starting SSO callback with state: {state}") + verbose_proxy_logger.info("Starting SSO callback with state: %s", state) oauth_error = request.query_params.get("error") if oauth_error: oauth_error_description = request.query_params.get("error_description") verbose_proxy_logger.warning( - f"SSO callback received OAuth error: {oauth_error}, description: {oauth_error_description}" + "SSO callback received OAuth error: %s, description: %s", oauth_error, oauth_error_description ) raise HTTPException( status_code=401, @@ -1861,7 +1875,7 @@ async def auth_callback(request: Request, state: str | None = None): ) redirect_url = SSOAuthenticationHandler.get_redirect_url_for_sso(request=request, sso_callback_route="sso/callback") - verbose_proxy_logger.info(f"Redirecting to {redirect_url}") + verbose_proxy_logger.info("Redirecting to %s", redirect_url) result = None if google_client_id is not None: result = await GoogleSSOHandler.get_google_callback_response( @@ -2044,7 +2058,7 @@ async def _fetch_cli_sso_team_details( } ) except Exception as e: - verbose_proxy_logger.error(f"Error fetching team details for CLI SSO session: {e}") + verbose_proxy_logger.error("Error fetching team details for CLI SSO session: %s", e) return team_details @@ -2111,7 +2125,7 @@ async def _complete_cli_sso_callback_session( _set_cli_sso_flow(login_id=key, cache=cli_sso_session_cache, flow=flow) verbose_proxy_logger.info( - f"Stored CLI SSO session for user: {user_info.user_id}, teams: {teams}, num_teams: {len(teams)}" + "Stored CLI SSO session for user: %s, teams: %s, num_teams: %s", user_info.user_id, teams, len(teams) ) verify_url = get_custom_url( request_base_url=str(request.base_url), @@ -2165,7 +2179,7 @@ async def cli_sso_callback( result=result_non_none, generic_client_id=os.getenv("GENERIC_CLIENT_ID", None), ) - verbose_proxy_logger.debug(f"parsed_openid_result: {parsed_openid_result}") + verbose_proxy_logger.debug("parsed_openid_result: %s", parsed_openid_result) user_defined_values = await _build_cli_sso_user_defined_values( result=result_non_none, parsed_openid_result=parsed_openid_result, @@ -2196,7 +2210,7 @@ async def cli_sso_callback( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error with CLI SSO callback: {e}") + verbose_proxy_logger.error("Error with CLI SSO callback: %s", e) raise HTTPException(status_code=500, detail=f"Failed to process CLI SSO: {e}") @@ -2236,7 +2250,11 @@ async def cli_poll_key( user_id = session_data["user_id"] verbose_proxy_logger.info( - f"CLI poll: user={user_id}, team_id={team_id}, user_teams={user_teams}, num_teams={len(user_teams)}" + "CLI poll: user=%s, team_id=%s, user_teams=%s, num_teams=%s", + user_id, + team_id, + user_teams, + len(user_teams), ) # If no team_id provided and user has teams, return teams list for selection @@ -2244,7 +2262,7 @@ async def cli_poll_key( # clients we return rich team details (id + alias); older clients # can continue to rely on the simple "teams" list. if team_id is None and len(user_teams) > 1: - verbose_proxy_logger.info(f"Returning teams list for user {user_id} to select from: {user_teams}") + verbose_proxy_logger.info("Returning teams list for user %s to select from: %s", user_id, user_teams) # Best-effort construction of team_details if it wasn't # already cached for some reason. team_details_response: list[dict[str, Any]] | None = None @@ -2298,7 +2316,7 @@ async def cli_poll_key( # Delete cache entry (single-use) cli_sso_session_cache.delete_cache(key=_get_cli_sso_flow_cache_key(key_id)) - verbose_proxy_logger.info(f"CLI JWT generated for user: {user_id}, team: {team_id}") + verbose_proxy_logger.info("CLI JWT generated for user: %s, team: %s", user_id, team_id) poll_response = { "status": "ready", "key": jwt_token, @@ -2319,7 +2337,7 @@ async def cli_poll_key( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error polling for CLI JWT: {e}") + verbose_proxy_logger.error("Error polling for CLI JWT: %s", e) raise HTTPException(status_code=500, detail=f"Error checking session status: {e}") @@ -2337,7 +2355,7 @@ async def insert_sso_user( Returns: Tuple[str, str]: User ID and User Role """ - verbose_proxy_logger.debug(f"Inserting SSO user into DB. User values: {user_defined_values}") + verbose_proxy_logger.debug("Inserting SSO user into DB. User values: %s", user_defined_values) if result_openid is None: raise ValueError("result_openid is None") if isinstance(result_openid, dict): @@ -2357,7 +2375,7 @@ async def insert_sso_user( preserved_role = sso_role user_defined_values.update(litellm.default_internal_user_params) # type: ignore user_defined_values["user_role"] = preserved_role # Restore preserved role - verbose_proxy_logger.debug(f"Preserved SSO-extracted role '{preserved_role}'") + verbose_proxy_logger.debug("Preserved SSO-extracted role '%s'", preserved_role) else: # SSO didn't provide a valid role, apply all defaults including role user_defined_values.update(litellm.default_internal_user_params) # type: ignore @@ -2671,7 +2689,9 @@ class SSOAuthenticationHandler: redirect_uri=redirect_url, ) verbose_proxy_logger.info( - f"In /google-login/key/generate, \nGOOGLE_REDIRECT_URI: {redirect_url}\nGOOGLE_CLIENT_ID: {google_client_id}" + "In /google-login/key/generate, \nGOOGLE_REDIRECT_URI: %s\nGOOGLE_CLIENT_ID: %s", + redirect_url, + google_client_id, ) with google_sso: return await google_sso.get_login_redirect(state=state) @@ -2733,10 +2753,13 @@ class SSOAuthenticationHandler: code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) verbose_proxy_logger.debug( - f"authorization_endpoint: {generic_authorization_endpoint}\ntoken_endpoint: {generic_token_endpoint}\nuserinfo_endpoint: {generic_userinfo_endpoint}" + "authorization_endpoint: %s\ntoken_endpoint: %s\nuserinfo_endpoint: %s", + generic_authorization_endpoint, + generic_token_endpoint, + generic_userinfo_endpoint, ) verbose_proxy_logger.debug( - f"GENERIC_REDIRECT_URI: {redirect_url}\nGENERIC_CLIENT_ID: {generic_client_id}\n" + "GENERIC_REDIRECT_URI: %s\nGENERIC_CLIENT_ID: %s\n", redirect_url, generic_client_id ) discovery = DiscoveryDocument( authorization_endpoint=generic_authorization_endpoint, @@ -2992,7 +3015,7 @@ class SSOAuthenticationHandler: ) return user_info except Exception as e: - verbose_proxy_logger.exception(f"Error upserting SSO user into LiteLLM DB: {e}") + verbose_proxy_logger.exception("Error upserting SSO user into LiteLLM DB: %s", e) return user_info @staticmethod @@ -3075,11 +3098,11 @@ class SSOAuthenticationHandler: ) try: team_obj = await TeamRepository(prisma_client).table.find_first(where={"team_id": litellm_team_id}) - verbose_proxy_logger.debug(f"Team object: {team_obj}") + verbose_proxy_logger.debug("Team object: %s", team_obj) # only create a new team if it doesn't exist if team_obj: - verbose_proxy_logger.debug(f"Team already exists: {litellm_team_id} - {litellm_team_name}") + verbose_proxy_logger.debug("Team already exists: %s - %s", litellm_team_id, litellm_team_name) return team_request: NewTeamRequest = NewTeamRequest( @@ -3104,7 +3127,7 @@ class SSOAuthenticationHandler: ), ) except Exception as e: - verbose_proxy_logger.exception(f"Error creating Litellm Team: {e}") + verbose_proxy_logger.exception("Error creating Litellm Team: %s", e) @staticmethod def _cast_and_deepcopy_litellm_default_team_params( @@ -3188,7 +3211,7 @@ class SSOAuthenticationHandler: if _user_role is not None: # Convert enum to string if needed user_role = _user_role.value if isinstance(_user_role, LitellmUserRoles) else _user_role - verbose_proxy_logger.debug(f"Extracted user_role from SSO result: {user_role}") + verbose_proxy_logger.debug("Extracted user_role from SSO result: %s", user_role) # generic client id - override with custom attribute name if specified if generic_client_id is not None and result is not None: @@ -3252,7 +3275,7 @@ class SSOAuthenticationHandler: user_email = parsed_openid_result.get("user_email") user_id = parsed_openid_result.get("user_id") user_role = parsed_openid_result.get("user_role") - verbose_proxy_logger.info(f"SSO callback result: {result}") + verbose_proxy_logger.info("SSO callback result: %s", result) user_info = None user_id_models: list = [] @@ -3318,7 +3341,7 @@ class SSOAuthenticationHandler: "Unable to map user identity to known values. 'user_defined_values' is None. File an issue - https://github.com/BerriAI/litellm/issues" ) - verbose_proxy_logger.info(f"user_defined_values for creating ui key: {user_defined_values}") + verbose_proxy_logger.info("user_defined_values for creating ui key: %s", user_defined_values) response = await generate_key_helper_fn( request_type="key", @@ -3346,7 +3369,7 @@ class SSOAuthenticationHandler: user_role=user_role, user_id=user_id, prisma_client=prisma_client ) - verbose_proxy_logger.debug(f"user_role: {user_role}; ui_access_mode: {ui_access_mode}") + verbose_proxy_logger.debug("user_role: %s; ui_access_mode: %s", user_role, ui_access_mode) ## CHECK IF ROLE ALLOWED TO USE PROXY ## is_admin_only_access = check_is_admin_only_access(ui_access_mode or {}) if is_admin_only_access: @@ -3412,7 +3435,7 @@ class SSOAuthenticationHandler: if user_id is not None and isinstance(user_id, str): litellm_dashboard_ui += "?login=success" - verbose_proxy_logger.info(f"Redirecting to {litellm_dashboard_ui}") + verbose_proxy_logger.info("Redirecting to %s", litellm_dashboard_ui) redirect_response = RedirectResponse(url=litellm_dashboard_ui, status_code=303) redirect_response.set_cookie(key="token", value=jwt_token) return redirect_response @@ -4012,7 +4035,7 @@ class MicrosoftSSOHandler: # Extract app roles from the id_token JWT app_roles = MicrosoftSSOHandler.get_app_roles_from_id_token(id_token=microsoft_sso.id_token) - verbose_proxy_logger.debug(f"Extracted app roles from id_token: {app_roles}") + verbose_proxy_logger.debug("Extracted app roles from id_token: %s", app_roles) # Combine groups and app roles user_role: LitellmUserRoles | None = None @@ -4022,10 +4045,10 @@ class MicrosoftSSOHandler: role = get_litellm_user_role(role_str) if role is not None: user_role = role - verbose_proxy_logger.debug(f"Found valid LitellmUserRoles '{role.value}' in app_roles") + verbose_proxy_logger.debug("Found valid LitellmUserRoles '%s' in app_roles", role.value) break - verbose_proxy_logger.debug(f"Combined team_ids (groups + app roles): {user_team_ids}") + verbose_proxy_logger.debug("Combined team_ids (groups + app roles): %s", user_team_ids) # if user is trying to get the raw sso response for debugging, return the raw sso response if return_raw_sso_response: @@ -4047,7 +4070,7 @@ class MicrosoftSSOHandler: user_role: LitellmUserRoles | None, ) -> CustomOpenID: response = response or {} - verbose_proxy_logger.debug(f"Microsoft SSO Callback Response: {response}") + verbose_proxy_logger.debug("Microsoft SSO Callback Response: %s", response) openid_response = CustomOpenID( email=normalize_email(response.get(MICROSOFT_USER_EMAIL_ATTRIBUTE) or response.get("mail")), display_name=response.get(MICROSOFT_USER_DISPLAY_NAME_ATTRIBUTE), @@ -4058,7 +4081,7 @@ class MicrosoftSSOHandler: team_ids=team_ids, user_role=user_role, ) - verbose_proxy_logger.debug(f"Microsoft SSO OpenID Response: {openid_response}") + verbose_proxy_logger.debug("Microsoft SSO OpenID Response: %s", openid_response) return openid_response @staticmethod @@ -4091,14 +4114,14 @@ class MicrosoftSSOHandler: roles = decoded_token.get("app_roles", []) or decoded_token.get("roles", []) if roles and isinstance(roles, list): - verbose_proxy_logger.debug(f"Found {len(roles)} app role(s) in id_token: {roles}") + verbose_proxy_logger.debug("Found %s app role(s) in id_token: %s", len(roles), roles) return roles else: verbose_proxy_logger.debug("No app roles found in id_token or roles claim is not a list") return [] except Exception as e: - verbose_proxy_logger.error(f"Error extracting app roles from id_token: {e}") + verbose_proxy_logger.error("Error extracting app roles from id_token: %s", e) return [] @staticmethod @@ -4130,7 +4153,7 @@ class MicrosoftSSOHandler: async_client=async_client, access_token=access_token, ) - verbose_proxy_logger.debug(f"Service principal group IDs: {service_principal_group_ids}") + verbose_proxy_logger.debug("Service principal group IDs: %s", service_principal_group_ids) if len(service_principal_group_ids) > 0: await MicrosoftSSOHandler.create_litellm_teams_from_service_principal_team_ids( service_principal_teams=service_principal_teams, @@ -4151,7 +4174,8 @@ class MicrosoftSSOHandler: if next_link is not None and page_count >= MicrosoftSSOHandler.MAX_GRAPH_API_PAGES: verbose_proxy_logger.warning( - f"Reached maximum page limit of {MicrosoftSSOHandler.MAX_GRAPH_API_PAGES}. Some groups may not be included." + "Reached maximum page limit of %s. Some groups may not be included.", + MicrosoftSSOHandler.MAX_GRAPH_API_PAGES, ) # If service_principal_group_ids is not empty, only return group_ids that are in both all_group_ids and service_principal_group_ids @@ -4161,7 +4185,7 @@ class MicrosoftSSOHandler: return all_group_ids except Exception as e: - verbose_proxy_logger.error(f"Error getting user groups from Microsoft Graph API: {e}") + verbose_proxy_logger.error("Error getting user groups from Microsoft Graph API: %s", e) return [] @staticmethod @@ -4238,7 +4262,7 @@ class MicrosoftSSOHandler: while next_link is not None and page_count < MicrosoftSSOHandler.MAX_GRAPH_API_PAGES: response = await async_client.get(next_link, headers=headers) response_json = response.json() - verbose_proxy_logger.debug(f"Response from service principal app role assigned to: {response_json}") + verbose_proxy_logger.debug("Response from service principal app role assigned to: %s", response_json) for _object in response_json.get("value", []): if _object.get("principalType") == "Group": @@ -4257,7 +4281,8 @@ class MicrosoftSSOHandler: if next_link is not None and page_count >= MicrosoftSSOHandler.MAX_GRAPH_API_PAGES: verbose_proxy_logger.warning( - f"Reached maximum page limit of {MicrosoftSSOHandler.MAX_GRAPH_API_PAGES}. Some service principal group assignments may not be included." + "Reached maximum page limit of %s. Some service principal group assignments may not be included.", + MicrosoftSSOHandler.MAX_GRAPH_API_PAGES, ) return group_ids, service_principal_teams @@ -4271,13 +4296,13 @@ class MicrosoftSSOHandler: When a user sets a `SERVICE_PRINCIPAL_ID` in the env, litellm will fetch groups under that service principal and create Litellm Teams from them """ - verbose_proxy_logger.debug(f"Creating Litellm Teams from Service Principal Teams: {service_principal_teams}") + verbose_proxy_logger.debug("Creating Litellm Teams from Service Principal Teams: %s", service_principal_teams) for service_principal_team in service_principal_teams: litellm_team_id: str | None = service_principal_team.get("principalId") litellm_team_name: str | None = service_principal_team.get("principalDisplayName") if not litellm_team_id: verbose_proxy_logger.debug( - f"Skipping team creation for {litellm_team_name} because it has no principalId" + "Skipping team creation for %s because it has no principalId", litellm_team_name ) continue diff --git a/litellm/proxy/management_helpers/audit_logs.py b/litellm/proxy/management_helpers/audit_logs.py index 8677627c607..081bd6af866 100644 --- a/litellm/proxy/management_helpers/audit_logs.py +++ b/litellm/proxy/management_helpers/audit_logs.py @@ -231,4 +231,4 @@ async def create_audit_log_for_update(request_data: LiteLLM_AuditLogs): ) except Exception as e: # [Non-Blocking Exception. Do not allow blocking LLM API call] - verbose_proxy_logger.error(f"Failed Creating audit log {e}") + verbose_proxy_logger.error("Failed Creating audit log %s", e) diff --git a/litellm/proxy/management_helpers/object_permission_utils.py b/litellm/proxy/management_helpers/object_permission_utils.py index 24bfbe2f9a9..69665151f07 100644 --- a/litellm/proxy/management_helpers/object_permission_utils.py +++ b/litellm/proxy/management_helpers/object_permission_utils.py @@ -166,7 +166,7 @@ async def handle_update_object_permission_common( }, ) - verbose_proxy_logger.debug(f"created_object_permission_row: {created_object_permission_row}") + verbose_proxy_logger.debug("created_object_permission_row: %s", created_object_permission_row) return created_object_permission_row.object_permission_id @@ -572,8 +572,8 @@ async def validate_key_mcp_servers_against_team( } if stale_identifiers: verbose_proxy_logger.warning( - "validate_key_mcp_servers_against_team: ignoring stale MCP server " - f"identifiers (no longer in registry or DB): {sorted(stale_identifiers)}" + "validate_key_mcp_servers_against_team: ignoring stale MCP server identifiers (no longer in registry or DB): %s", + sorted(stale_identifiers), ) _rewrite_object_permission_mcp_identifiers( object_permission=object_permission, diff --git a/litellm/proxy/ocr_endpoints/endpoints.py b/litellm/proxy/ocr_endpoints/endpoints.py index ca25be9d92c..acc3f85076f 100644 --- a/litellm/proxy/ocr_endpoints/endpoints.py +++ b/litellm/proxy/ocr_endpoints/endpoints.py @@ -96,9 +96,10 @@ async def _parse_multipart_form(request: Request) -> dict[str, Any]: data[field_name] = field_value verbose_proxy_logger.debug( - f"OCR multipart form request parsed - model: {data.get('model')}, " - f"document_type: {document['type']}, " - f"filename: {uploaded_file.filename}" + "OCR multipart form request parsed - model: %s, document_type: %s, filename: %s", + data.get("model"), + document["type"], + uploaded_file.filename, ) return data diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 87514b46dbd..76b956f31b2 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -983,10 +983,10 @@ async def ensure_batch_response_managed_file_ids( user_api_key_dict=user_api_key_dict, ) setattr(response, file_attr, new_unified_file_id) - verbose_proxy_logger.debug(f"Converted batch {file_attr} {raw_file_id!r} to managed ID before DB write") + verbose_proxy_logger.debug("Converted batch %s %r to managed ID before DB write", file_attr, raw_file_id) except Exception as e: verbose_proxy_logger.warning( - f"Failed to convert batch {file_attr}={raw_file_id!r} to managed ID before DB write: {e}" + "Failed to convert batch %s=%r to managed ID before DB write: %s", file_attr, raw_file_id, e ) @@ -1042,12 +1042,16 @@ async def get_batch_from_database( # The stored batch object has the raw provider input_file_id. Resolve to unified ID. await resolve_input_file_id_to_unified(response, prisma_client) - verbose_proxy_logger.debug(f"Retrieved batch {batch_id} from ManagedObjectTable with status={response.status}") + verbose_proxy_logger.debug( + "Retrieved batch %s from ManagedObjectTable with status=%s", batch_id, response.status + ) return db_batch_object, response except Exception as e: - verbose_proxy_logger.warning(f"Failed to retrieve batch from ManagedObjectTable: {e}, falling back to provider") + verbose_proxy_logger.warning( + "Failed to retrieve batch from ManagedObjectTable: %s, falling back to provider", e + ) return None, None @@ -1103,10 +1107,10 @@ async def update_batch_in_database( if db_batch_object: verbose_proxy_logger.info( - f"Updating batch {batch_id} status from {db_batch_object.status} to {response.status}" + "Updating batch %s status from %s to %s", batch_id, db_batch_object.status, response.status ) else: - verbose_proxy_logger.info(f"Updating batch {batch_id} status to {response.status} after {operation}") + verbose_proxy_logger.info("Updating batch %s status to %s after %s", batch_id, response.status, operation) # Normalize status for database storage db_status = response.status if response.status != "completed" else "complete" @@ -1138,7 +1142,9 @@ async def update_batch_in_database( # retry without it so the status update still succeeds. err_str = str(col_err).lower() if "batch_processed" in err_str and update_data.get("batch_processed") is not None: - verbose_proxy_logger.warning(f"batch_processed column not found, retrying update without it: {col_err}") + verbose_proxy_logger.warning( + "batch_processed column not found, retrying update without it: %s", col_err + ) update_data.pop("batch_processed", None) await ManagedObjectRepository(prisma_client).table.update( where={"unified_object_id": batch_id}, @@ -1147,4 +1153,4 @@ async def update_batch_in_database( else: raise except Exception as e: - verbose_proxy_logger.error(f"Failed to update batch status in ManagedObjectTable: {e}") + verbose_proxy_logger.error("Failed to update batch status in ManagedObjectTable: %s", e) diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 5d4c3c04818..51b9139cff1 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -208,7 +208,7 @@ async def route_create_file( original_id = response.id encoded_id = encode_file_id_with_model(file_id=original_id, model=model) response.id = encoded_id - verbose_proxy_logger.debug(f"Encoded file ID: {original_id} -> {encoded_id} (model: {model})") + verbose_proxy_logger.debug("Encoded file ID: %s -> %s (model: %s)", original_id, encoded_id, model) return response @@ -549,7 +549,7 @@ async def create_file( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.create_file(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.create_file(): Exception occured - %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "message", str(e.detail)), @@ -845,7 +845,7 @@ async def get_file_content( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.retrieve_file_content(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.retrieve_file_content(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -1032,7 +1032,7 @@ async def get_file( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.retrieve_file(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.retrieve_file(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -1238,7 +1238,7 @@ async def delete_file( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.delete_file(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.delete_file(): Exception occured - %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "message", str(e.detail)), @@ -1337,7 +1337,7 @@ async def list_files( **data, # type: ignore ) - verbose_proxy_logger.debug(f"Listed files using model: {model_used}") + verbose_proxy_logger.debug("Listed files using model: %s", model_used) elif target_model_names and isinstance(target_model_names, str): target_model_names_list = target_model_names.split(",") @@ -1427,7 +1427,7 @@ async def list_files( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.list_files(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.list_files(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( diff --git a/litellm/proxy/openai_files_endpoints/storage_backend_service.py b/litellm/proxy/openai_files_endpoints/storage_backend_service.py index b9658d8efee..200bad13a9f 100644 --- a/litellm/proxy/openai_files_endpoints/storage_backend_service.py +++ b/litellm/proxy/openai_files_endpoints/storage_backend_service.py @@ -82,7 +82,7 @@ class StorageBackendFileService: file_naming_strategy="uuid", ) - verbose_proxy_logger.debug(f"Storage backend upload complete: backend={target_storage}, url={storage_url}") + verbose_proxy_logger.debug("Storage backend upload complete: backend=%s, url=%s", target_storage, storage_url) # Create file object with storage metadata file_object = StorageBackendFileService._create_file_object_with_storage_metadata( @@ -223,8 +223,10 @@ class StorageBackendFileService: file_object.id = base64_unified_file_id verbose_proxy_logger.debug( - f"Storing file in managed files: unified_id={base64_unified_file_id}, " - f"storage_backend={target_storage}, storage_url={storage_url}" + "Storing file in managed files: unified_id=%s, storage_backend=%s, storage_url=%s", + base64_unified_file_id, + target_storage, + storage_url, ) # Store in managed files diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 0d9b0ab9c49..026a76a767c 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -752,7 +752,7 @@ async def handle_bedrock_passthrough_router_model( is_streaming = any(action in endpoint for action in BEDROCK_STREAMING_ACTIONS) verbose_proxy_logger.debug( - f"Bedrock router passthrough: model='{model}', endpoint='{endpoint}', streaming={is_streaming}" + "Bedrock router passthrough: model='%s', endpoint='%s', streaming=%s", model, endpoint, is_streaming ) # Use the common processing path (same as non-router models) @@ -843,8 +843,8 @@ async def handle_bedrock_count_tokens( if key != "user_api_key_dict": # Don't overwrite user_api_key_dict litellm_params[key] = value # type: ignore - verbose_proxy_logger.debug(f"Count tokens litellm_params: {litellm_params}") - verbose_proxy_logger.debug(f"Resolved model: {resolved_model}") + verbose_proxy_logger.debug("Count tokens litellm_params: %s", litellm_params) + verbose_proxy_logger.debug("Resolved model: %s", resolved_model) # Handle the count tokens request result = await handler.handle_count_tokens_request( @@ -857,13 +857,13 @@ async def handle_bedrock_count_tokens( except BedrockError as e: # Convert BedrockError to HTTPException for FastAPI - verbose_proxy_logger.error(f"BedrockError in handle_bedrock_count_tokens: {e}") + verbose_proxy_logger.error("BedrockError in handle_bedrock_count_tokens: %s", e) raise HTTPException(status_code=e.status_code, detail={"error": e.message}) except HTTPException: # Re-raise HTTP exceptions as-is raise except Exception as e: - verbose_proxy_logger.error(f"Error in handle_bedrock_count_tokens: {e}") + verbose_proxy_logger.error("Error in handle_bedrock_count_tokens: %s", e) raise HTTPException(status_code=500, detail={"error": f"CountTokens processing error: {e}"}) @@ -947,7 +947,9 @@ async def bedrock_llm_proxy_route( ) # Fall back to existing implementation for direct Bedrock models - verbose_proxy_logger.debug(f"Bedrock passthrough: Using direct Bedrock model '{model}' for endpoint '{endpoint}'") + verbose_proxy_logger.debug( + "Bedrock passthrough: Using direct Bedrock model '%s' for endpoint '%s'", model, endpoint + ) data: dict[str, Any] = {} base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) @@ -1148,7 +1150,7 @@ def _resolve_vertex_model_from_router( endpoint = endpoint.replace(model_id, actual_model) except Exception as e: - verbose_proxy_logger.debug(f"Error resolving vertex model from router for model {model_id}: {e}") + verbose_proxy_logger.debug("Error resolving vertex model from router for model %s: %s", model_id, e) return encoded_endpoint, endpoint, vertex_project, vertex_location diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index 5d045ff2852..275283e3e56 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -920,7 +920,7 @@ class AnthropicPassthroughLoggingHandler: } except Exception as e: - verbose_proxy_logger.error(f"Error in batch_creation_handler: {e}") + verbose_proxy_logger.error("Error in batch_creation_handler: %s", e) # Return basic response on error litellm_model_response = ModelResponse() litellm_model_response.id = str(uuid.uuid4()) @@ -1017,7 +1017,9 @@ class AnthropicPassthroughLoggingHandler: ) verbose_proxy_logger.info( - f"Stored Anthropic batch managed object with unified_object_id={unified_object_id}, batch_id={model_object_id}" + "Stored Anthropic batch managed object with unified_object_id=%s, batch_id=%s", + unified_object_id, + model_object_id, ) else: verbose_proxy_logger.warning( @@ -1025,7 +1027,7 @@ class AnthropicPassthroughLoggingHandler: ) except Exception as e: - verbose_proxy_logger.error(f"Error storing Anthropic batch managed object: {e}") + verbose_proxy_logger.error("Error storing Anthropic batch managed object: %s", e) @staticmethod def get_actual_model_id_from_router(model_name: str) -> str: @@ -1038,14 +1040,14 @@ class AnthropicPassthroughLoggingHandler: if model_ids and len(model_ids) > 0: # Use the first model ID found actual_model_id = model_ids[0] - verbose_proxy_logger.info(f"Found model ID in router: {actual_model_id}") + verbose_proxy_logger.info("Found model ID in router: %s", actual_model_id) return actual_model_id else: # Fallback to model name actual_model_id = model_name - verbose_proxy_logger.warning(f"Model not found in router, using model name: {actual_model_id}") + verbose_proxy_logger.warning("Model not found in router, using model name: %s", actual_model_id) return actual_model_id else: # Fallback if router is not available - verbose_proxy_logger.warning(f"Router not available, using model name: {model_name}") + verbose_proxy_logger.warning("Router not available, using model name: %s", model_name) return model_name diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py index 397f1d94a34..97cbe9a0615 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py @@ -203,7 +203,7 @@ class AssemblyAIPassthroughLoggingHandler: return response.json() except Exception as e: - verbose_proxy_logger.exception(f"[Non blocking logging error] Error getting AssemblyAI transcript: {e}") + verbose_proxy_logger.exception("[Non blocking logging error] Error getting AssemblyAI transcript: %s", e) return None def _poll_assembly_for_transcript_response( @@ -275,7 +275,7 @@ class AssemblyAIPassthroughLoggingHandler: return None except Exception as e: - verbose_proxy_logger.exception(f"[Non blocking logging error] Error getting AssemblyAI model info: {e}") + verbose_proxy_logger.exception("[Non blocking logging error] Error getting AssemblyAI model info: %s", e) return None @staticmethod diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index 63414a1c19e..18e56d914c6 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -183,7 +183,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): return cost except Exception as e: - verbose_proxy_logger.warning(f"Error calculating image generation cost: {e}") + verbose_proxy_logger.warning("Error calculating image generation cost: %s", e) return 0.0 @staticmethod @@ -217,7 +217,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): return cost except Exception as e: - verbose_proxy_logger.warning(f"Error calculating image editing cost: {e}") + verbose_proxy_logger.warning("Error calculating image editing cost: %s", e) return 0.0 @staticmethod @@ -445,7 +445,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): } except Exception as e: - verbose_proxy_logger.error(f"Error in OpenAI passthrough cost tracking: {e}") + verbose_proxy_logger.error("Error in OpenAI passthrough cost tracking: %s", e) # Fall back to base handler without cost tracking base_handler = OpenAIPassthroughLoggingHandler() return base_handler.passthrough_chat_handler( @@ -501,7 +501,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): all_openai_chunks.append(transformed_chunk) except (StopIteration, StopAsyncIteration, Exception) as e: - verbose_proxy_logger.debug(f"Error parsing streaming chunk: {e}") + verbose_proxy_logger.debug("Error parsing streaming chunk: %s", e) continue if not all_openai_chunks: @@ -514,7 +514,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): return complete_streaming_response except Exception as e: - verbose_proxy_logger.error(f"Error building complete streaming response: {e}") + verbose_proxy_logger.error("Error building complete streaming response: %s", e) return None @staticmethod @@ -608,7 +608,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): } except Exception as e: - verbose_proxy_logger.error(f"Error in OpenAI streaming passthrough cost tracking: {e}") + verbose_proxy_logger.error("Error in OpenAI streaming passthrough cost tracking: %s", e) return { "result": None, "kwargs": {}, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_ai_live_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_ai_live_passthrough_logging_handler.py index 11672571f3f..fd703b22549 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_ai_live_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_ai_live_passthrough_logging_handler.py @@ -148,11 +148,11 @@ class VertexAILivePassthroughLoggingHandler(BasePassthroughLoggingHandler): # Get model pricing information model_info = get_model_info(model=model, custom_llm_provider=custom_llm_provider) - verbose_proxy_logger.debug(f"Vertex AI Live API model info for '{model}': {model_info}") + verbose_proxy_logger.debug("Vertex AI Live API model info for '%s': %s", model, model_info) # Check if pricing info is available if not model_info or not model_info.get("input_cost_per_token"): - verbose_proxy_logger.error(f"No pricing info found for {model} in local model pricing database") + verbose_proxy_logger.error("No pricing info found for %s in local model pricing database", model) return 0.0 total_cost = 0.0 @@ -221,7 +221,7 @@ class VertexAILivePassthroughLoggingHandler(BasePassthroughLoggingHandler): return total_cost except Exception as e: - verbose_proxy_logger.error(f"Error calculating Vertex AI Live API cost: {e}") + verbose_proxy_logger.error("Error calculating Vertex AI Live API cost: %s", e) return 0.0 @staticmethod @@ -302,7 +302,9 @@ class VertexAILivePassthroughLoggingHandler(BasePassthroughLoggingHandler): # Extract model from request body or kwargs model = kwargs.get("model", "gemini-2.0-flash-live-preview-04-09") custom_llm_provider = kwargs.get("custom_llm_provider", "vertex_ai") - verbose_proxy_logger.debug(f"Vertex AI Live API model: {model}, custom_llm_provider: {custom_llm_provider}") + verbose_proxy_logger.debug( + "Vertex AI Live API model: %s, custom_llm_provider: %s", model, custom_llm_provider + ) # Extract usage metadata from WebSocket messages usage_metadata = self._extract_usage_metadata_from_websocket_messages(websocket_messages) @@ -360,7 +362,7 @@ class VertexAILivePassthroughLoggingHandler(BasePassthroughLoggingHandler): } except Exception as e: - verbose_proxy_logger.error(f"Error in Vertex AI Live API passthrough handler: {e}") + verbose_proxy_logger.error("Error in Vertex AI Live API passthrough handler: %s", e) return { "result": None, "kwargs": kwargs, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index 233127c3fef..bab16d0ba54 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -744,7 +744,7 @@ class VertexPassthroughLoggingHandler: } except Exception as e: - verbose_proxy_logger.error(f"Error in batch_prediction_jobs_handler: {e}") + verbose_proxy_logger.error("Error in batch_prediction_jobs_handler: %s", e) # Return basic response on error litellm_model_response = ModelResponse() litellm_model_response.id = str(uuid.uuid4()) @@ -841,7 +841,9 @@ class VertexPassthroughLoggingHandler: ) verbose_proxy_logger.info( - f"Stored batch managed object with unified_object_id={unified_object_id}, batch_id={model_object_id}" + "Stored batch managed object with unified_object_id=%s, batch_id=%s", + unified_object_id, + model_object_id, ) else: verbose_proxy_logger.warning( @@ -849,7 +851,7 @@ class VertexPassthroughLoggingHandler: ) except Exception as e: - verbose_proxy_logger.error(f"Error storing batch managed object: {e}") + verbose_proxy_logger.error("Error storing batch managed object: %s", e) @staticmethod def get_actual_model_id_from_router(model_name: str) -> str: @@ -864,15 +866,15 @@ class VertexPassthroughLoggingHandler: if model_ids and len(model_ids) > 0: # Use the first model ID found actual_model_id = model_ids[0] - verbose_proxy_logger.info(f"Found model ID in router: {actual_model_id}") + verbose_proxy_logger.info("Found model ID in router: %s", actual_model_id) return actual_model_id else: # Fallback to constructed model name actual_model_id = extracted_model_name - verbose_proxy_logger.warning(f"Model not found in router, using constructed name: {actual_model_id}") + verbose_proxy_logger.warning("Model not found in router, using constructed name: %s", actual_model_id) return actual_model_id else: # Fallback if router is not available extracted_model_name = VertexPassthroughLoggingHandler.extract_model_name_from_vertex_path(model_name) - verbose_proxy_logger.warning(f"Router not available, using constructed model name: {extracted_model_name}") + verbose_proxy_logger.warning("Router not available, using constructed model name: %s", extracted_model_name) return extracted_model_name diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index b8aba215d10..171df5b4d2c 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -294,7 +294,7 @@ async def chat_completion_pass_through_endpoint( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.completion(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.completion(): Exception occured - %s", e) error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), @@ -890,7 +890,7 @@ async def pass_through_request( if "metadata" not in _parsed_body: _parsed_body["metadata"] = {} _parsed_body["metadata"]["guardrails"] = guardrails_to_run - verbose_proxy_logger.debug(f"Added guardrails to passthrough request metadata: {guardrails_to_run}") + verbose_proxy_logger.debug("Added guardrails to passthrough request metadata: %s", guardrails_to_run) ## LOGGING OBJECT ## - initialize before pre_call_hook so guardrails can access it # Surface the requested model (when the body carries one) so logging/spans @@ -1502,7 +1502,7 @@ async def pass_through_request( ) else: verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.pass_through_endpoint(): Exception occured - {e}" + "litellm.proxy.proxy_server.pass_through_endpoint(): Exception occured - %s", e ) ######################################################### @@ -1887,12 +1887,12 @@ async def websocket_passthrough_request( websocket_messages: list[dict[str, Any]] = [] litellm_call_id = str(uuid.uuid4()) - verbose_proxy_logger.info(f"WebSocket passthrough ({endpoint}): Starting WebSocket connection to {target}") + verbose_proxy_logger.info("WebSocket passthrough (%s): Starting WebSocket connection to %s", endpoint, target) # Only accept the WebSocket if requested (for generic usage) if accept_websocket: await websocket.accept() - verbose_proxy_logger.debug(f"WebSocket passthrough ({endpoint}): WebSocket connection accepted") + verbose_proxy_logger.debug("WebSocket passthrough (%s): WebSocket connection accepted", endpoint) # Prepare headers for the upstream connection upstream_headers = custom_headers.copy() @@ -1985,13 +1985,15 @@ async def websocket_passthrough_request( ) try: - verbose_proxy_logger.debug(f"WebSocket passthrough ({endpoint}): Establishing upstream connection to {target}") + verbose_proxy_logger.debug( + "WebSocket passthrough (%s): Establishing upstream connection to %s", endpoint, target + ) async with connect( target, additional_headers=upstream_headers, ) as upstream_ws: verbose_proxy_logger.info( - f"WebSocket passthrough ({endpoint}): Upstream connection established successfully" + "WebSocket passthrough (%s): Upstream connection established successfully", endpoint ) async def forward_client_to_upstream() -> None: @@ -2011,14 +2013,17 @@ async def websocket_passthrough_request( # Try to extract model from client setup message for Vertex AI Live if endpoint and "/vertex_ai/live" in endpoint: verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Processing client message for model extraction" + "WebSocket passthrough (%s): Processing client message for model extraction", + endpoint, ) try: client_message = json.loads(text_data) if isinstance(client_message, dict) and "setup" in client_message: setup_data = client_message["setup"] verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Found setup data in client message: {setup_data}" + "WebSocket passthrough (%s): Found setup data in client message: %s", + endpoint, + setup_data, ) if isinstance(setup_data, dict) and "model" in setup_data: extracted_model = _extract_model_from_vertex_ai_setup(setup_data) @@ -2030,23 +2035,32 @@ async def websocket_passthrough_request( logging_obj.model_call_details["model"] = extracted_model logging_obj.model_call_details["custom_llm_provider"] = "vertex_ai" verbose_proxy_logger.info( - f"WebSocket passthrough ({endpoint}): Successfully extracted model '{extracted_model}' and set provider to 'vertex_ai' from client setup message" + "WebSocket passthrough (%s): Successfully extracted model '%s' and set provider to 'vertex_ai' from client setup message", + endpoint, + extracted_model, ) else: verbose_proxy_logger.warning( - f"WebSocket passthrough ({endpoint}): Failed to extract model from client setup data: {setup_data}" + "WebSocket passthrough (%s): Failed to extract model from client setup data: %s", + endpoint, + setup_data, ) else: verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Setup data does not contain model field: {setup_data}" + "WebSocket passthrough (%s): Setup data does not contain model field: %s", + endpoint, + setup_data, ) else: verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Client message does not contain setup data" + "WebSocket passthrough (%s): Client message does not contain setup data", + endpoint, ) except (json.JSONDecodeError, KeyError, TypeError) as e: verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Client message is not a valid setup message: {e}" + "WebSocket passthrough (%s): Client message is not a valid setup message: %s", + endpoint, + e, ) # Not a JSON message or doesn't contain setup data @@ -2057,7 +2071,7 @@ async def websocket_passthrough_request( raise except Exception: verbose_proxy_logger.exception( - f"WebSocket passthrough ({endpoint}): error forwarding client message" + "WebSocket passthrough (%s): error forwarding client message", endpoint ) await upstream_ws.close() @@ -2070,12 +2084,13 @@ async def websocket_passthrough_request( if isinstance(raw_response, str): raw_response = raw_response.encode("ascii") setup_response = json.loads(raw_response.decode("ascii")) - verbose_proxy_logger.debug(f"Setup response: {setup_response}") + verbose_proxy_logger.debug("Setup response: %s", setup_response) # Extract model and provider from setup response for Vertex AI Live if endpoint and "/vertex_ai/live" in endpoint: verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Processing server setup response for model extraction" + "WebSocket passthrough (%s): Processing server setup response for model extraction", + endpoint, ) extracted_model = _extract_model_from_vertex_ai_setup(setup_response) if extracted_model: @@ -2086,15 +2101,20 @@ async def websocket_passthrough_request( logging_obj.model_call_details["model"] = extracted_model logging_obj.model_call_details["custom_llm_provider"] = "vertex_ai_language_models" verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Successfully extracted model '{extracted_model}' and set provider to 'vertex_ai' from server setup response" + "WebSocket passthrough (%s): Successfully extracted model '%s' and set provider to 'vertex_ai' from server setup response", + endpoint, + extracted_model, ) else: verbose_proxy_logger.warning( - f"WebSocket passthrough ({endpoint}): Failed to extract model from server setup response: {setup_response}" + "WebSocket passthrough (%s): Failed to extract model from server setup response: %s", + endpoint, + setup_response, ) else: verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Not a Vertex AI Live endpoint, skipping model extraction" + "WebSocket passthrough (%s): Not a Vertex AI Live endpoint, skipping model extraction", + endpoint, ) # Send the setup response to the client @@ -2120,14 +2140,14 @@ async def websocket_passthrough_request( pass except (ConnectionClosedOK, ConnectionClosedError) as e: - verbose_proxy_logger.debug(f"Upstream WebSocket connection closed: {e}") + verbose_proxy_logger.debug("Upstream WebSocket connection closed: %s", e) except asyncio.CancelledError: verbose_proxy_logger.debug("asyncio.CancelledError in forward_upstream_to_client") raise except Exception as e: - verbose_proxy_logger.debug(f"Exception in forward_upstream_to_client: {e}") + verbose_proxy_logger.debug("Exception in forward_upstream_to_client: %s", e) verbose_proxy_logger.exception( - f"WebSocket passthrough ({endpoint}): error forwarding upstream message" + "WebSocket passthrough (%s): error forwarding upstream message", endpoint ) raise @@ -2218,7 +2238,7 @@ async def websocket_passthrough_request( ) except InvalidStatus as exc: - verbose_proxy_logger.exception(f"WebSocket passthrough ({endpoint}): upstream rejected WebSocket connection") + verbose_proxy_logger.exception("WebSocket passthrough (%s): upstream rejected WebSocket connection", endpoint) # Prepare request payload for logging request_payload = {} @@ -2244,7 +2264,9 @@ async def websocket_passthrough_request( reason="Upstream connection rejected", ) except Exception as e: - verbose_proxy_logger.exception(f"WebSocket passthrough ({endpoint}): unexpected error while proxying WebSocket") + verbose_proxy_logger.exception( + "WebSocket passthrough (%s): unexpected error while proxying WebSocket", endpoint + ) # Prepare request payload for logging request_payload = {} @@ -2321,8 +2343,9 @@ async def _relay_passthrough_response_bytes( finally: if not upstream_fully_relayed: verbose_proxy_logger.warning( - f"Passthrough stream for {url_route} ended before upstream body was fully relayed; " - f"{bytes_relayed} bytes were sent to the client" + "Passthrough stream for %s ended before upstream body was fully relayed; %s bytes were sent to the client", + url_route, + bytes_relayed, ) await response.aclose() GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( @@ -2370,7 +2393,7 @@ def _extract_model_from_vertex_ai_setup(setup_response: dict) -> str | None: model_name = model_path.split("/models/")[-1] return model_name except Exception as e: - verbose_proxy_logger.debug(f"Error extracting model from setup response: {e}") + verbose_proxy_logger.debug("Error extracting model from setup response: %s", e) return None diff --git a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py index ed35e3d1b46..bd7f7b94f7a 100644 --- a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py +++ b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py @@ -49,12 +49,14 @@ class PassthroughEndpointRouter: custom_llm_provider=custom_llm_provider, region_name=region_name, ) - verbose_router_logger.debug(f"Pass-through llm endpoints router, looking for credentials for {credential_name}") + verbose_router_logger.debug( + "Pass-through llm endpoints router, looking for credentials for %s", credential_name + ) if credential_name in self.credentials: - verbose_router_logger.debug(f"Found credentials for {credential_name}") + verbose_router_logger.debug("Found credentials for %s", credential_name) return self.credentials[credential_name] else: - verbose_router_logger.debug(f"No credentials found for {credential_name}, looking for env variable") + verbose_router_logger.debug("No credentials found for %s, looking for env variable", credential_name) _env_variable_name = self._get_default_env_variable_name_passthrough_endpoint( custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index 9a4a28c7678..b8fd6d757b5 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -89,7 +89,7 @@ class PassThroughStreamingHandler: yield chunk except Exception as e: - verbose_proxy_logger.error(f"Error in chunk_processor: {e}") + verbose_proxy_logger.error("Error in chunk_processor: %s", e) raise finally: # GeneratorExit (raised on client disconnect) is not caught by @@ -115,7 +115,7 @@ class PassThroughStreamingHandler: ) ) except Exception as e: - verbose_proxy_logger.error(f"Error scheduling chunk_processor logging: {e}") + verbose_proxy_logger.error("Error scheduling chunk_processor logging: %s", e) @staticmethod async def _route_streaming_logging_to_handler( @@ -165,7 +165,7 @@ class PassThroughStreamingHandler: **kwargs, ) except Exception as e: - verbose_proxy_logger.error(f"Error in _route_streaming_logging_to_handler: {e}") + verbose_proxy_logger.error("Error in _route_streaming_logging_to_handler: %s", e) @staticmethod def _build_passthrough_logging_result( diff --git a/litellm/proxy/policy_engine/attachment_registry.py b/litellm/proxy/policy_engine/attachment_registry.py index 797f72f7667..9bbe5405965 100644 --- a/litellm/proxy/policy_engine/attachment_registry.py +++ b/litellm/proxy/policy_engine/attachment_registry.py @@ -58,14 +58,14 @@ class AttachmentRegistry: try: attachment = self._parse_attachment(attachment_data) self._attachments.append(attachment) - verbose_proxy_logger.debug(f"Loaded attachment for policy: {attachment.policy}") + verbose_proxy_logger.debug("Loaded attachment for policy: %s", attachment.policy) except Exception as e: - verbose_proxy_logger.error(f"Error loading attachment: {e}") + verbose_proxy_logger.error("Error loading attachment: %s", e) raise ValueError(f"Invalid attachment: {e}") from e self._config_attachments = tuple(self._attachments) self._initialized = True - verbose_proxy_logger.info(f"Loaded {len(self._attachments)} policy attachments") + verbose_proxy_logger.info("Loaded %s policy attachments", len(self._attachments)) def _parse_attachment(self, attachment_data: dict[str, Any]) -> PolicyAttachment: """ @@ -123,9 +123,12 @@ class AttachmentRegistry: } ) verbose_proxy_logger.debug( - f"Attachment matched: policy={attachment.policy}, " - f"matched_via={matched_via}, " - f"context=(team={context.team_alias}, key={context.key_alias}, model={context.model})" + "Attachment matched: policy=%s, matched_via=%s, context=(team=%s, key=%s, model=%s)", + attachment.policy, + matched_via, + context.team_alias, + context.key_alias, + context.model, ) return results @@ -222,7 +225,7 @@ class AttachmentRegistry: """ self._attachments.append(attachment) self._initialized = True - verbose_proxy_logger.debug(f"Added attachment for policy: {attachment.policy}") + verbose_proxy_logger.debug("Added attachment for policy: %s", attachment.policy) def remove_attachments_for_policy(self, policy_name: str) -> int: """ @@ -238,7 +241,7 @@ class AttachmentRegistry: self._attachments = [a for a in self._attachments if a.policy != policy_name] removed_count = original_count - len(self._attachments) if removed_count > 0: - verbose_proxy_logger.debug(f"Removed {removed_count} attachment(s) for policy: {policy_name}") + verbose_proxy_logger.debug("Removed %s attachment(s) for policy: %s", removed_count, policy_name) return removed_count def remove_attachment_by_id(self, attachment_id: str) -> bool: @@ -317,7 +320,7 @@ class AttachmentRegistry: updated_by=created_attachment.updated_by, ) except Exception as e: - verbose_proxy_logger.exception(f"Error adding attachment to DB: {e}") + verbose_proxy_logger.exception("Error adding attachment to DB: %s", e) raise Exception(f"Error adding attachment to DB: {e}") async def delete_attachment_from_db( @@ -353,7 +356,7 @@ class AttachmentRegistry: return {"message": f"Attachment {attachment_id} deleted successfully"} except Exception as e: - verbose_proxy_logger.exception(f"Error deleting attachment from DB: {e}") + verbose_proxy_logger.exception("Error deleting attachment from DB: %s", e) raise Exception(f"Error deleting attachment from DB: {e}") async def get_attachment_by_id_from_db( @@ -393,7 +396,7 @@ class AttachmentRegistry: updated_by=attachment.updated_by, ) except Exception as e: - verbose_proxy_logger.exception(f"Error getting attachment from DB: {e}") + verbose_proxy_logger.exception("Error getting attachment from DB: %s", e) raise Exception(f"Error getting attachment from DB: {e}") async def get_all_attachments_from_db( @@ -431,7 +434,7 @@ class AttachmentRegistry: for a in attachments ] except Exception as e: - verbose_proxy_logger.exception(f"Error getting attachments from DB: {e}") + verbose_proxy_logger.exception("Error getting attachments from DB: %s", e) raise Exception(f"Error getting attachments from DB: {e}") async def sync_attachments_from_db( @@ -463,11 +466,12 @@ class AttachmentRegistry: self._initialized = True verbose_proxy_logger.info( - f"Synced {len(attachments)} attachments from DB to in-memory registry " - f"({len(self._config_attachments)} config-defined attachments preserved)" + "Synced %s attachments from DB to in-memory registry (%s config-defined attachments preserved)", + len(attachments), + len(self._config_attachments), ) except Exception as e: - verbose_proxy_logger.exception(f"Error syncing attachments from DB: {e}") + verbose_proxy_logger.exception("Error syncing attachments from DB: %s", e) raise Exception(f"Error syncing attachments from DB: {e}") diff --git a/litellm/proxy/policy_engine/condition_evaluator.py b/litellm/proxy/policy_engine/condition_evaluator.py index 02268fcd721..2f3a4fa8015 100644 --- a/litellm/proxy/policy_engine/condition_evaluator.py +++ b/litellm/proxy/policy_engine/condition_evaluator.py @@ -48,7 +48,9 @@ class ConditionEvaluator: condition=condition.model, model=context.model, ): - verbose_proxy_logger.debug(f"Condition failed: model={context.model} did not match {condition.model}") + verbose_proxy_logger.debug( + "Condition failed: model=%s did not match %s", context.model, condition.model + ) return False return True diff --git a/litellm/proxy/policy_engine/init_policies.py b/litellm/proxy/policy_engine/init_policies.py index 1facec0898f..c3016afbd6c 100644 --- a/litellm/proxy/policy_engine/init_policies.py +++ b/litellm/proxy/policy_engine/init_policies.py @@ -124,7 +124,7 @@ async def init_policies( Raises: ValueError: If fail_on_error is True and validation errors are found """ - verbose_proxy_logger.info(f"Initializing {len(policies_config)} policies...") + verbose_proxy_logger.info("Initializing %s policies...", len(policies_config)) # Print policies to console on startup _print_policies_on_startup(policies_config, policy_attachments_config) @@ -146,13 +146,13 @@ async def init_policies( if validation_result.errors: for error in validation_result.errors: verbose_proxy_logger.error( - f"Policy validation error in '{error.policy_name}': [{error.error_type}] {error.message}" + "Policy validation error in '%s': [%s] %s", error.policy_name, error.error_type, error.message ) if validation_result.warnings: for warning in validation_result.warnings: verbose_proxy_logger.warning( - f"Policy validation warning in '{warning.policy_name}': [{warning.error_type}] {warning.message}" + "Policy validation warning in '%s': [%s] %s", warning.policy_name, warning.error_type, warning.message ) # Fail if there are errors and fail_on_error is True @@ -165,18 +165,18 @@ async def init_policies( # Load policies into registry (even with warnings) try: policy_registry.load_policies(policies_config) - verbose_proxy_logger.info(f"Successfully loaded {len(policies_config)} policies") + verbose_proxy_logger.info("Successfully loaded %s policies", len(policies_config)) except Exception as e: - verbose_proxy_logger.error(f"Failed to load policies: {e}") + verbose_proxy_logger.error("Failed to load policies: %s", e) raise # Load attachments if provided if policy_attachments_config: try: attachment_registry.load_attachments(policy_attachments_config) - verbose_proxy_logger.info(f"Successfully loaded {len(policy_attachments_config)} policy attachments") + verbose_proxy_logger.info("Successfully loaded %s policy attachments", len(policy_attachments_config)) except Exception as e: - verbose_proxy_logger.error(f"Failed to load policy attachments: {e}") + verbose_proxy_logger.error("Failed to load policy attachments: %s", e) raise return validation_result diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index 983d2da124b..7feb5123093 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -92,7 +92,12 @@ class PipelineExecutor: step_results.append(step_result) verbose_proxy_logger.debug( - f"Pipeline '{policy_name}' step {i}: guardrail={step.guardrail}, outcome={outcome}, action={action}" + "Pipeline '%s' step %s: guardrail=%s, outcome=%s, action=%s", + policy_name, + i, + step.guardrail, + outcome, + action, ) # Forward modified data to next step if pass_data is True @@ -158,7 +163,7 @@ class PipelineExecutor: """ callback = PipelineExecutor.find_guardrail_callback(step.guardrail) if callback is None: - verbose_proxy_logger.warning(f"Pipeline: guardrail '{step.guardrail}' not found in callbacks") + verbose_proxy_logger.warning("Pipeline: guardrail '%s' not found in callbacks", step.guardrail) return ("error", None, f"Guardrail '{step.guardrail}' not found", None) try: @@ -205,7 +210,7 @@ class PipelineExecutor: error_msg = _extract_error_message(e) return ("fail", None, error_msg, e) else: - verbose_proxy_logger.error(f"Pipeline: unexpected error from guardrail '{step.guardrail}': {e}") + verbose_proxy_logger.error("Pipeline: unexpected error from guardrail '%s': %s", step.guardrail, e) return ("error", None, str(e), e) @staticmethod diff --git a/litellm/proxy/policy_engine/policy_endpoints.py b/litellm/proxy/policy_engine/policy_endpoints.py index 1d0fd67cfc8..9959c2302b5 100644 --- a/litellm/proxy/policy_engine/policy_endpoints.py +++ b/litellm/proxy/policy_engine/policy_endpoints.py @@ -142,7 +142,7 @@ async def list_policies(version_status: str | None = None): policies = db_policies + config_policies return PolicyListDBResponse(policies=policies, total_count=len(policies)) except Exception as e: - verbose_proxy_logger.exception(f"Error listing policies: {e}") + verbose_proxy_logger.exception("Error listing policies: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -201,7 +201,7 @@ async def create_policy( ) return result except Exception as e: - verbose_proxy_logger.exception(f"Error creating policy: {e}") + verbose_proxy_logger.exception("Error creating policy: %s", e) if "unique constraint" in str(e).lower(): raise HTTPException( status_code=400, @@ -236,7 +236,7 @@ async def list_policy_versions(policy_name: str): prisma_client=prisma_client, ) except Exception as e: - verbose_proxy_logger.exception(f"Error listing policy versions: {e}") + verbose_proxy_logger.exception("Error listing policy versions: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -269,7 +269,7 @@ async def create_policy_version( created_by=created_by, ) except Exception as e: - verbose_proxy_logger.exception(f"Error creating policy version: {e}") + verbose_proxy_logger.exception("Error creating policy version: %s", e) if "not found" in str(e).lower() or "no production" in str(e).lower(): raise HTTPException(status_code=404, detail=str(e)) raise HTTPException(status_code=500, detail=str(e)) @@ -308,7 +308,7 @@ async def update_policy_version_status( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error updating version status: {e}") + verbose_proxy_logger.exception("Error updating version status: %s", e) if "invalid status" in str(e).lower() or "only draft" in str(e).lower() or "cannot promote" in str(e).lower(): raise HTTPException(status_code=400, detail=str(e)) if "not found" in str(e).lower(): @@ -341,7 +341,7 @@ async def compare_policy_versions( prisma_client=prisma_client, ) except Exception as e: - verbose_proxy_logger.exception(f"Error comparing versions: {e}") + verbose_proxy_logger.exception("Error comparing versions: %s", e) if "not found" in str(e).lower(): raise HTTPException(status_code=404, detail=str(e)) raise HTTPException(status_code=500, detail=str(e)) @@ -367,7 +367,7 @@ async def delete_all_policy_versions(policy_name: str): prisma_client=prisma_client, ) except Exception as e: - verbose_proxy_logger.exception(f"Error deleting all versions: {e}") + verbose_proxy_logger.exception("Error deleting all versions: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -408,7 +408,7 @@ async def get_policy(policy_id: str): except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error getting policy: {e}") + verbose_proxy_logger.exception("Error getting policy: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -467,7 +467,7 @@ async def update_policy( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error updating policy: {e}") + verbose_proxy_logger.exception("Error updating policy: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -516,7 +516,7 @@ async def delete_policy(policy_id: str): except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error deleting policy: {e}") + verbose_proxy_logger.exception("Error deleting policy: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -577,7 +577,7 @@ async def get_resolved_guardrails(policy_id: str): except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: - verbose_proxy_logger.exception(f"Error resolving guardrails: {e}") + verbose_proxy_logger.exception("Error resolving guardrails: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -640,7 +640,7 @@ async def test_pipeline( ) return result.model_dump() except Exception as e: - verbose_proxy_logger.exception(f"Error testing pipeline: {e}") + verbose_proxy_logger.exception("Error testing pipeline: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -699,7 +699,7 @@ async def list_policy_attachments(): attachments = db_attachments + config_attachments return PolicyAttachmentListResponse(attachments=attachments, total_count=len(attachments)) except Exception as e: - verbose_proxy_logger.exception(f"Error listing policy attachments: {e}") + verbose_proxy_logger.exception("Error listing policy attachments: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -791,7 +791,7 @@ async def create_policy_attachment( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error creating policy attachment: {e}") + verbose_proxy_logger.exception("Error creating policy attachment: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -830,7 +830,7 @@ async def get_policy_attachment(attachment_id: str): except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error getting policy attachment: {e}") + verbose_proxy_logger.exception("Error getting policy attachment: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -881,5 +881,5 @@ async def delete_policy_attachment(attachment_id: str): except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error deleting policy attachment: {e}") + verbose_proxy_logger.exception("Error deleting policy attachment: %s", e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/litellm/proxy/policy_engine/policy_registry.py b/litellm/proxy/policy_engine/policy_registry.py index 07a4c2abac6..6031f4a372e 100644 --- a/litellm/proxy/policy_engine/policy_registry.py +++ b/litellm/proxy/policy_engine/policy_registry.py @@ -185,15 +185,15 @@ class PolicyRegistry: try: policy = self._parse_policy(policy_name, policy_data) self._policies[policy_name] = policy - verbose_proxy_logger.debug(f"Loaded policy: {policy_name}") + verbose_proxy_logger.debug("Loaded policy: %s", policy_name) except Exception as e: - verbose_proxy_logger.error(f"Error loading policy '{policy_name}': {e}") + verbose_proxy_logger.error("Error loading policy '%s': %s", policy_name, e) raise ValueError(f"Invalid policy '{policy_name}': {e}") from e self._config_policies = dict(self._policies) self._sources = {policy_name: "config" for policy_name in self._policies} self._initialized = True - verbose_proxy_logger.info(f"Loaded {len(self._policies)} policies") + verbose_proxy_logger.info("Loaded %s policies", len(self._policies)) def _parse_policy(self, policy_name: str, policy_data: dict[str, Any]) -> Policy: """ @@ -336,7 +336,7 @@ class PolicyRegistry: if source == "config": self._config_policies = {**self._config_policies, policy_name: policy} self._initialized = True - verbose_proxy_logger.debug(f"Added/updated policy: {policy_name}") + verbose_proxy_logger.debug("Added/updated policy: %s", policy_name) def remove_policy(self, policy_name: str) -> bool: """ @@ -355,11 +355,11 @@ class PolicyRegistry: if config_fallback is not None: self._policies[policy_name] = config_fallback self._sources = {**self._sources, policy_name: "config"} - verbose_proxy_logger.debug(f"Removed policy: {policy_name}; restored config-defined version") + verbose_proxy_logger.debug("Removed policy: %s; restored config-defined version", policy_name) return True del self._policies[policy_name] self._sources = {name: source for name, source in self._sources.items() if name != policy_name} - verbose_proxy_logger.debug(f"Removed policy: {policy_name}") + verbose_proxy_logger.debug("Removed policy: %s", policy_name) return True # ───────────────────────────────────────────────────────────────────────── @@ -432,7 +432,7 @@ class PolicyRegistry: return _row_to_policy_db_response(created_policy) except Exception as e: - verbose_proxy_logger.exception(f"Error adding policy to DB: {e}") + verbose_proxy_logger.exception("Error adding policy to DB: %s", e) raise Exception(f"Error adding policy to DB: {e}") async def update_policy_in_db( @@ -496,7 +496,7 @@ class PolicyRegistry: return _row_to_policy_db_response(updated_policy) except Exception as e: - verbose_proxy_logger.exception(f"Error updating policy in DB: {e}") + verbose_proxy_logger.exception("Error updating policy in DB: %s", e) raise Exception(f"Error updating policy in DB: {e}") async def delete_policy_from_db( @@ -546,7 +546,7 @@ class PolicyRegistry: return result except Exception as e: - verbose_proxy_logger.exception(f"Error deleting policy from DB: {e}") + verbose_proxy_logger.exception("Error deleting policy from DB: %s", e) raise Exception(f"Error deleting policy from DB: {e}") async def get_policy_by_id_from_db( @@ -572,7 +572,7 @@ class PolicyRegistry: return _row_to_policy_db_response(policy) except Exception as e: - verbose_proxy_logger.exception(f"Error getting policy from DB: {e}") + verbose_proxy_logger.exception("Error getting policy from DB: %s", e) raise Exception(f"Error getting policy from DB: {e}") def get_policy_by_id_for_request(self, policy_id: str) -> tuple[str, Policy] | None: @@ -619,7 +619,7 @@ class PolicyRegistry: return [_row_to_policy_db_response(p) for p in policies] except Exception as e: - verbose_proxy_logger.exception(f"Error getting policies from DB: {e}") + verbose_proxy_logger.exception("Error getting policies from DB: %s", e) raise Exception(f"Error getting policies from DB: {e}") async def sync_policies_from_db( @@ -653,7 +653,8 @@ class PolicyRegistry: } for policy_name in set(db_policies) & set(self._config_policies): verbose_proxy_logger.warning( - f"Policy '{policy_name}' is defined in both config.yaml and the DB; the DB version takes precedence" + "Policy '%s' is defined in both config.yaml and the DB; the DB version takes precedence", + policy_name, ) config_sources: Mapping[str, Literal["db", "config"]] = {name: "config" for name in self._config_policies} db_sources: Mapping[str, Literal["db", "config"]] = {name: "db" for name in db_policies} @@ -683,12 +684,13 @@ class PolicyRegistry: self._initialized = True verbose_proxy_logger.info( - f"Synced {len(production)} production policies and {len(non_production)} " - "draft/published (by ID) from DB to in-memory registry " - f"({len(self._config_policies)} config-defined policies preserved)" + "Synced %s production policies and %s draft/published (by ID) from DB to in-memory registry (%s config-defined policies preserved)", + len(production), + len(non_production), + len(self._config_policies), ) except Exception as e: - verbose_proxy_logger.exception(f"Error syncing policies from DB: {e}") + verbose_proxy_logger.exception("Error syncing policies from DB: %s", e) raise Exception(f"Error syncing policies from DB: {e}") async def resolve_guardrails_from_db( @@ -741,7 +743,7 @@ class PolicyRegistry: return sorted(resolved_policy.guardrails) except Exception as e: - verbose_proxy_logger.exception(f"Error resolving guardrails from DB: {e}") + verbose_proxy_logger.exception("Error resolving guardrails from DB: %s", e) raise Exception(f"Error resolving guardrails from DB: {e}") async def get_versions_by_policy_name( @@ -771,7 +773,7 @@ class PolicyRegistry: total_count=len(versions), ) except Exception as e: - verbose_proxy_logger.exception(f"Error getting versions: {e}") + verbose_proxy_logger.exception("Error getting versions: %s", e) raise Exception(f"Error getting versions: {e}") async def create_new_version( @@ -857,7 +859,7 @@ class PolicyRegistry: created = await _policy_table(prisma_client).create(data=data) return _row_to_policy_db_response(created) except Exception as e: - verbose_proxy_logger.exception(f"Error creating new version: {e}") + verbose_proxy_logger.exception("Error creating new version: %s", e) raise Exception(f"Error creating new version: {e}") async def update_version_status( @@ -962,7 +964,7 @@ class PolicyRegistry: return _row_to_policy_db_response(updated) except Exception as e: - verbose_proxy_logger.exception(f"Error updating version status: {e}") + verbose_proxy_logger.exception("Error updating version status: %s", e) raise Exception(f"Error updating version status: {e}") async def compare_versions( @@ -1015,7 +1017,7 @@ class PolicyRegistry: field_diffs=field_diffs, ) except Exception as e: - verbose_proxy_logger.exception(f"Error comparing versions: {e}") + verbose_proxy_logger.exception("Error comparing versions: %s", e) raise Exception(f"Error comparing versions: {e}") async def delete_all_versions( @@ -1046,7 +1048,7 @@ class PolicyRegistry: } return {"message": message} except Exception as e: - verbose_proxy_logger.exception(f"Error deleting all versions: {e}") + verbose_proxy_logger.exception("Error deleting all versions: %s", e) raise Exception(f"Error deleting all versions: {e}") diff --git a/litellm/proxy/policy_engine/policy_resolve_endpoints.py b/litellm/proxy/policy_engine/policy_resolve_endpoints.py index e5c2693cf21..575a9cbacc2 100644 --- a/litellm/proxy/policy_engine/policy_resolve_endpoints.py +++ b/litellm/proxy/policy_engine/policy_resolve_endpoints.py @@ -298,7 +298,7 @@ async def resolve_policies_for_context( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error resolving policies: {e}") + verbose_proxy_logger.exception("Error resolving policies: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -408,5 +408,5 @@ async def estimate_attachment_impact( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error estimating attachment impact: {e}") + verbose_proxy_logger.exception("Error estimating attachment impact: %s", e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/litellm/proxy/policy_engine/policy_resolver.py b/litellm/proxy/policy_engine/policy_resolver.py index 65c8236a0b3..355bc10c003 100644 --- a/litellm/proxy/policy_engine/policy_resolver.py +++ b/litellm/proxy/policy_engine/policy_resolver.py @@ -47,7 +47,7 @@ class PolicyResolver: visited = set() if policy_name in visited: - verbose_proxy_logger.warning(f"Circular inheritance detected for policy '{policy_name}'") + verbose_proxy_logger.warning("Circular inheritance detected for policy '%s'", policy_name) return [] policy = policies.get(policy_name) @@ -106,7 +106,7 @@ class PolicyResolver: context=context, ): verbose_proxy_logger.debug( - f"Policy '{chain_policy_name}' condition did not match, skipping guardrails" + "Policy '%s' condition did not match, skipping guardrails", chain_policy_name ) continue @@ -163,8 +163,10 @@ class PolicyResolver: if not matching_policy_names: verbose_proxy_logger.debug( - f"No policies match context: team_alias={context.team_alias}, " - f"key_alias={context.key_alias}, model={context.model}" + "No policies match context: team_alias=%s, key_alias=%s, model=%s", + context.team_alias, + context.key_alias, + context.model, ) return [] @@ -178,10 +180,10 @@ class PolicyResolver: context=context, ) all_guardrails.update(resolved.guardrails) - verbose_proxy_logger.debug(f"Policy '{policy_name}' contributes guardrails: {resolved.guardrails}") + verbose_proxy_logger.debug("Policy '%s' contributes guardrails: %s", policy_name, resolved.guardrails) result = list(all_guardrails) - verbose_proxy_logger.debug(f"Final guardrails for context: {result}") + verbose_proxy_logger.debug("Final guardrails for context: %s", result) return result @@ -229,7 +231,7 @@ class PolicyResolver: if policy.pipeline is not None: pipelines.append((policy_name, policy.pipeline)) verbose_proxy_logger.debug( - f"Policy '{policy_name}' has pipeline with {len(policy.pipeline.steps)} steps" + "Policy '%s' has pipeline with %s steps", policy_name, len(policy.pipeline.steps) ) return pipelines diff --git a/litellm/proxy/policy_engine/policy_validator.py b/litellm/proxy/policy_engine/policy_validator.py index 67f7b37472c..c8799a71307 100644 --- a/litellm/proxy/policy_engine/policy_validator.py +++ b/litellm/proxy/policy_engine/policy_validator.py @@ -78,7 +78,7 @@ class PolicyValidator: guardrails = IN_MEMORY_GUARDRAIL_HANDLER.list_in_memory_guardrails() return {g.get("guardrail_name", "") for g in guardrails if g.get("guardrail_name")} except Exception as e: - verbose_proxy_logger.warning(f"Could not get guardrails from registry: {e}") + verbose_proxy_logger.warning("Could not get guardrails from registry: %s", e) return set() async def check_team_alias_exists(self, team_alias: str) -> bool: @@ -100,7 +100,7 @@ class PolicyValidator: ) return team is not None except Exception as e: - verbose_proxy_logger.warning(f"Could not check team alias '{team_alias}': {e}") + verbose_proxy_logger.warning("Could not check team alias '%s': %s", team_alias, e) return True # Assume valid on error async def check_key_alias_exists(self, key_alias: str) -> bool: @@ -122,7 +122,7 @@ class PolicyValidator: ) return key is not None except Exception as e: - verbose_proxy_logger.warning(f"Could not check key alias '{key_alias}': {e}") + verbose_proxy_logger.warning("Could not check key alias '%s': %s", key_alias, e) return True # Assume valid on error def check_model_exists(self, model: str) -> bool: @@ -151,7 +151,7 @@ class PolicyValidator: return False except Exception as e: - verbose_proxy_logger.warning(f"Could not check model '{model}': {e}") + verbose_proxy_logger.warning("Could not check model '%s': %s", model, e) return True # Assume valid on error @staticmethod diff --git a/litellm/proxy/prisma_migration.py b/litellm/proxy/prisma_migration.py index 0f6944fac79..b09a49cbff5 100644 --- a/litellm/proxy/prisma_migration.py +++ b/litellm/proxy/prisma_migration.py @@ -16,9 +16,9 @@ run_server(["--skip_server_startup"], standalone_mode=False) # run prisma generate verbose_proxy_logger.info("Running 'prisma generate'...") result = subprocess.run(["prisma", "generate"], capture_output=True, text=True) -verbose_proxy_logger.info(f"'prisma generate' stdout: {result.stdout}") # Log stdout +verbose_proxy_logger.info("'prisma generate' stdout: %s", result.stdout) # Log stdout exit_code = result.returncode if exit_code != 0: - verbose_proxy_logger.info(f"'prisma generate' failed with exit code {exit_code}.") - verbose_proxy_logger.error(f"'prisma generate' stderr: {result.stderr}") # Log stderr + verbose_proxy_logger.info("'prisma generate' failed with exit code %s.", exit_code) + verbose_proxy_logger.error("'prisma generate' stderr: %s", result.stderr) # Log stderr diff --git a/litellm/proxy/prometheus_cleanup.py b/litellm/proxy/prometheus_cleanup.py index 2a22b1c5fae..7cf27193b9a 100644 --- a/litellm/proxy/prometheus_cleanup.py +++ b/litellm/proxy/prometheus_cleanup.py @@ -21,9 +21,9 @@ def wipe_directory(directory: str) -> None: os.remove(filepath) deleted += 1 except OSError as e: - verbose_proxy_logger.warning(f"Failed to delete stale prometheus file {filepath}: {e}") + verbose_proxy_logger.warning("Failed to delete stale prometheus file %s: %s", filepath, e) if deleted: - verbose_proxy_logger.info(f"Prometheus cleanup: wiped {deleted} stale .db files from {directory}") + verbose_proxy_logger.info("Prometheus cleanup: wiped %s stale .db files from %s", deleted, directory) def mark_worker_exit(worker_pid: int) -> None: @@ -34,6 +34,6 @@ def mark_worker_exit(worker_pid: int) -> None: from prometheus_client import multiprocess multiprocess.mark_process_dead(worker_pid) - verbose_proxy_logger.info(f"Prometheus cleanup: marked worker {worker_pid} as dead") + verbose_proxy_logger.info("Prometheus cleanup: marked worker %s as dead", worker_pid) except Exception as e: - verbose_proxy_logger.warning(f"Failed to mark prometheus worker {worker_pid} as dead: {e}") + verbose_proxy_logger.warning("Failed to mark prometheus worker %s as dead: %s", worker_pid, e) diff --git a/litellm/proxy/prompts/init_prompts.py b/litellm/proxy/prompts/init_prompts.py index 67e961b2da8..752762f07b9 100644 --- a/litellm/proxy/prompts/init_prompts.py +++ b/litellm/proxy/prompts/init_prompts.py @@ -23,4 +23,4 @@ def init_prompts( if initialized_prompt: prompt_list.append(initialized_prompt) - verbose_proxy_logger.debug(f"\nPrompt List:{prompt_list}\n") + verbose_proxy_logger.debug("\nPrompt List:%s\n", prompt_list) diff --git a/litellm/proxy/prompts/prompt_endpoints.py b/litellm/proxy/prompts/prompt_endpoints.py index 89087a3fdd5..fd0437a4265 100644 --- a/litellm/proxy/prompts/prompt_endpoints.py +++ b/litellm/proxy/prompts/prompt_endpoints.py @@ -755,7 +755,7 @@ async def create_prompt( return initialized_prompt except Exception as e: - verbose_proxy_logger.exception(f"Error creating prompt: {e}") + verbose_proxy_logger.exception("Error creating prompt: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -871,7 +871,7 @@ async def update_prompt( except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.exception(f"Error updating prompt: {e}") + verbose_proxy_logger.exception("Error updating prompt: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -970,7 +970,7 @@ async def delete_prompt( except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.exception(f"Error deleting prompt: {e}") + verbose_proxy_logger.exception("Error deleting prompt: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -1111,7 +1111,7 @@ async def patch_prompt( except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.exception(f"Error patching prompt: {e}") + verbose_proxy_logger.exception("Error patching prompt: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -1252,7 +1252,7 @@ async def test_prompt( except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: - verbose_proxy_logger.exception(f"Error testing prompt: {e}") + verbose_proxy_logger.exception("Error testing prompt: %s", e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/litellm/proxy/prompts/prompt_registry.py b/litellm/proxy/prompts/prompt_registry.py index 9e00d4e7c63..78b449873a8 100644 --- a/litellm/proxy/prompts/prompt_registry.py +++ b/litellm/proxy/prompts/prompt_registry.py @@ -51,7 +51,7 @@ def get_prompt_initializer_from_integrations(): module_path = f"litellm.integrations.{item}" try: # Import the module - verbose_proxy_logger.debug(f"Discovering prompt integrations in: {module_path}") + verbose_proxy_logger.debug("Discovering prompt integrations in: %s", module_path) module = importlib.import_module(module_path) @@ -61,22 +61,22 @@ def get_prompt_initializer_from_integrations(): if isinstance(registry, dict): discovered_initializers.update(registry) verbose_proxy_logger.debug( - f"Found prompt_initializer_registry in {module_path}: {list(registry.keys())}" + "Found prompt_initializer_registry in %s: %s", module_path, list(registry.keys()) ) except ImportError as e: - verbose_proxy_logger.error(f"Could not import {module_path}: {e}") + verbose_proxy_logger.error("Could not import %s: %s", module_path, e) continue except Exception as e: - verbose_proxy_logger.error(f"Error processing {module_path}: {e}") + verbose_proxy_logger.error("Error processing %s: %s", module_path, e) continue verbose_proxy_logger.debug( - f"Discovered {len(discovered_initializers)} prompt initializers: {list(discovered_initializers.keys())}" + "Discovered %s prompt initializers: %s", len(discovered_initializers), list(discovered_initializers.keys()) ) except Exception as e: - verbose_proxy_logger.error(f"Error discovering prompt initializers: {e}") + verbose_proxy_logger.error("Error discovering prompt initializers: %s", e) return discovered_initializers diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 58b4ec88192..82cbb7fda48 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -163,8 +163,8 @@ def _with_query_value(url: str, key: str, value: str) -> str: def append_query_params(url: str | None, params: dict) -> str: from litellm._logging import verbose_proxy_logger - verbose_proxy_logger.debug(f"url: {url}") - verbose_proxy_logger.debug(f"params: {params}") + verbose_proxy_logger.debug("url: %s", url) + verbose_proxy_logger.debug("params: %s", params) if not isinstance(url, str) or url == "": # Preserve previous startup behavior when DATABASE_URL is absent. # Returning an empty string avoids urlparse type errors in test/dev flows. diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 2ff39164d80..16a63540d4d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -862,12 +862,16 @@ async def _initialize_shared_aiohttp_session(): session = ClientSession(connector=connector) verbose_proxy_logger.info( - f"SESSION REUSE: Created shared aiohttp session for connection pooling (ID: {id(session)}, " - f"limit={AIOHTTP_CONNECTOR_LIMIT}, limit_per_host={AIOHTTP_CONNECTOR_LIMIT_PER_HOST})" + "SESSION REUSE: Created shared aiohttp session for connection pooling (ID: %s, limit=%s, limit_per_host=%s)", + id(session), + AIOHTTP_CONNECTOR_LIMIT, + AIOHTTP_CONNECTOR_LIMIT_PER_HOST, ) return session except Exception as e: - verbose_proxy_logger.warning(f"Failed to create shared aiohttp session: {e}. Continuing without session reuse.") + verbose_proxy_logger.warning( + "Failed to create shared aiohttp session: %s. Continuing without session reuse.", e + ) return None @@ -918,7 +922,7 @@ async def proxy_startup_event(app: FastAPI): raise ## CHECK PREMIUM USER - verbose_proxy_logger.debug(f"litellm.proxy.proxy_server.py::startup() - CHECKING PREMIUM USER - {premium_user}") + verbose_proxy_logger.debug("litellm.proxy.proxy_server.py::startup() - CHECKING PREMIUM USER - %s", premium_user) if premium_user is False: premium_user = _license_check.is_premium() @@ -973,9 +977,9 @@ async def proxy_startup_event(app: FastAPI): async def _run_pw_migration(): try: result = await migrate_passwords_to_scrypt_async(prisma_client) - verbose_proxy_logger.info(f"Password migration: {result}") + verbose_proxy_logger.info("Password migration: %s", result) except Exception as e: - verbose_proxy_logger.warning(f"Password migration skipped: {e}") + verbose_proxy_logger.warning("Password migration skipped: %s", e) asyncio.create_task(_run_pw_migration()) @@ -1049,14 +1053,14 @@ async def proxy_startup_event(app: FastAPI): verbose_proxy_logger.debug("About to initialize semantic tool filter") _config = proxy_config.get_config_state() _litellm_settings = _config.get("litellm_settings", {}) - verbose_proxy_logger.debug(f"litellm_settings keys = {list(_litellm_settings.keys())}") + verbose_proxy_logger.debug("litellm_settings keys = %s", list(_litellm_settings.keys())) await ProxyStartupEvent._initialize_semantic_tool_filter( llm_router=llm_router, litellm_settings=_litellm_settings, ) verbose_proxy_logger.debug("After semantic tool filter initialization") except Exception as e: - verbose_proxy_logger.error(f"Semantic filter init failed: {e}", exc_info=True) + verbose_proxy_logger.error("Semantic filter init failed: %s", e, exc_info=True) ## JWT AUTH ## ProxyStartupEvent._initialize_jwt_auth( @@ -1133,7 +1137,7 @@ async def proxy_startup_event(app: FastAPI): await shared_aiohttp_session.close() verbose_proxy_logger.info("SESSION REUSE: Closed shared aiohttp session") except Exception as e: - verbose_proxy_logger.error(f"Error closing shared aiohttp session: {e}") + verbose_proxy_logger.error("Error closing shared aiohttp session: %s", e) # Shutdown event - stop RDS IAM token refresh background task if ( @@ -1144,14 +1148,14 @@ async def proxy_startup_event(app: FastAPI): try: await prisma_client.db.stop_token_refresh_task() except Exception as e: - verbose_proxy_logger.error(f"Error stopping token refresh task: {e}") + verbose_proxy_logger.error("Error stopping token refresh task: %s", e) # Shutdown event - stop Prisma DB health watchdog task if prisma_client is not None and hasattr(prisma_client, "stop_db_health_watchdog_task"): try: await prisma_client.stop_db_health_watchdog_task() except Exception as e: - verbose_proxy_logger.error(f"Error stopping DB health watchdog task: {e}") + verbose_proxy_logger.error("Error stopping DB health watchdog task: %s", e) await proxy_config.stop_config_sync_subscriber() @@ -1598,7 +1602,7 @@ try: # Primary signal: marker file created by Dockerfile marker_file = os.path.join(ui_dir, ".litellm_ui_ready") if os.path.exists(marker_file): - verbose_proxy_logger.debug(f"Found UI ready marker: {marker_file}") + verbose_proxy_logger.debug("Found UI ready marker: %s", marker_file) return True # Fallback signal: Detect restructuring pattern @@ -1617,11 +1621,11 @@ try: if os.path.exists(index_path): # Found at least one restructured route - this proves the pattern verbose_proxy_logger.debug( - f"Detected restructured UI via pattern: found {entry.name}/index.html" + "Detected restructured UI via pattern: found %s/index.html", entry.name ) return True except (PermissionError, OSError) as e: - verbose_proxy_logger.debug(f"Could not scan {ui_dir} for restructuring detection: {e}") + verbose_proxy_logger.debug("Could not scan %s for restructuring detection: %s", ui_dir, e) return False # No restructured routes found @@ -1641,7 +1645,7 @@ try: target_path, dirs_exist_ok=True, ) - verbose_proxy_logger.info(f"Successfully populated UI at {target_path}") + verbose_proxy_logger.info("Successfully populated UI at %s", target_path) return True, "" else: return False, "Source or target directory state invalid" @@ -1665,7 +1669,7 @@ try: # Validate packaged UI before proceeding if not _validate_ui_directory(packaged_ui_path): verbose_proxy_logger.error( - f"Packaged UI at {packaged_ui_path} is invalid or incomplete. UI may not function correctly." + "Packaged UI at %s is invalid or incomplete. UI may not function correctly.", packaged_ui_path ) # Decision tree for UI path selection: @@ -1684,20 +1688,20 @@ try: # Case 2: Runtime UI exists and is ready if has_content and is_pre_restructured: - verbose_proxy_logger.info(f"Using pre-restructured UI at {runtime_ui_path}") + verbose_proxy_logger.info("Using pre-restructured UI at %s", runtime_ui_path) ui_path = runtime_ui_path # Case 3: Runtime UI exists but needs restructuring elif has_content and not is_pre_restructured: verbose_proxy_logger.warning( - f"UI at {runtime_ui_path} has content but is not properly restructured. " - f"Will attempt to restructure in place." + "UI at %s has content but is not properly restructured. Will attempt to restructure in place.", + runtime_ui_path, ) ui_path = runtime_ui_path # Case 4: Runtime UI missing - try to populate else: - verbose_proxy_logger.info(f"UI not found at {runtime_ui_path}. Attempting to populate from packaged UI.") + verbose_proxy_logger.info("UI not found at %s. Attempting to populate from packaged UI.", runtime_ui_path) success, error = _try_populate_ui_directory(packaged_ui_path, runtime_ui_path) @@ -1707,20 +1711,20 @@ try: else: # Case 4b: Population failed - fall back to packaged UI verbose_proxy_logger.warning( - f"Failed to populate UI at {runtime_ui_path}: {error}. " - f"Falling back to packaged UI at {packaged_ui_path}. " - f"For read-only deployments, pre-build UI in Dockerfile " - f"or set LITELLM_UI_PATH to a writable emptyDir volume." + "Failed to populate UI at %s: %s. Falling back to packaged UI at %s. For read-only deployments, pre-build UI in Dockerfile or set LITELLM_UI_PATH to a writable emptyDir volume.", + runtime_ui_path, + error, + packaged_ui_path, ) ui_path = packaged_ui_path else: # Case 1: Using packaged UI directly (local development) - verbose_proxy_logger.info(f"Using packaged UI directory: {packaged_ui_path}") + verbose_proxy_logger.info("Using packaged UI directory: %s", packaged_ui_path) ui_path = packaged_ui_path # Validate final UI path if not _validate_ui_directory(ui_path): - verbose_proxy_logger.error(f"Selected UI path {ui_path} is invalid or incomplete. UI may not work correctly.") + verbose_proxy_logger.error("Selected UI path %s is invalid or incomplete. UI may not work correctly.", ui_path) # Only modify files if a custom server root path is set AND filesystem is writable if server_root_path and server_root_path != "/": @@ -1729,9 +1733,8 @@ try: if not is_writable: verbose_proxy_logger.warning( - f"Cannot apply server_root_path replacements to UI at {ui_path}: " - f"path is not writable. Ensure server_root_path is '/' or pre-process " - f"UI files in Dockerfile with custom server_root_path." + "Cannot apply server_root_path replacements to UI at %s: path is not writable. Ensure server_root_path is '/' or pre-process UI files in Dockerfile with custom server_root_path.", + ui_path, ) else: # Iterate through files in the UI directory @@ -1825,20 +1828,19 @@ try: is_writable = os.access(ui_path, os.W_OK) if is_pre_restructured: - verbose_proxy_logger.info(f"Skipping UI restructuring: {ui_path} is already pre-restructured") + verbose_proxy_logger.info("Skipping UI restructuring: %s is already pre-restructured", ui_path) elif not is_writable: verbose_proxy_logger.warning( - f"Cannot restructure UI at {ui_path}: path is not writable. " - f"UI may not work correctly for extensionless routes. " - f"Pre-build and restructure UI in Dockerfile for read-only deployments." + "Cannot restructure UI at %s: path is not writable. UI may not work correctly for extensionless routes. Pre-build and restructure UI in Dockerfile for read-only deployments.", + ui_path, ) else: _restructure_ui_html_files(ui_path) - verbose_proxy_logger.info(f"Restructured UI directory: {ui_path}") + verbose_proxy_logger.info("Restructured UI directory: %s", ui_path) except PermissionError as e: - verbose_proxy_logger.exception(f"Permission error while restructuring UI directory {ui_path}: {e}") + verbose_proxy_logger.exception("Permission error while restructuring UI directory %s: %s", ui_path, e) except Exception as e: - verbose_proxy_logger.exception(f"Error while restructuring UI directory {ui_path}: {e}") + verbose_proxy_logger.exception("Error while restructuring UI directory %s: %s", ui_path, e) except Exception: pass @@ -2826,7 +2828,7 @@ async def update_cache( hashed_token = token verbose_proxy_logger.debug("_update_key_cache: hashed_token=%s", hashed_token) existing_spend_obj = await user_api_key_cache.async_get_cache(key=hashed_token, model_type=UserAPIKeyAuth) - verbose_proxy_logger.debug(f"_update_key_cache: existing_spend_obj={existing_spend_obj}") + verbose_proxy_logger.debug("_update_key_cache: existing_spend_obj=%s", existing_spend_obj) if existing_spend_obj is None: return @@ -2889,7 +2891,7 @@ async def update_cache( if existing_spend_obj is None: return verbose_proxy_logger.debug( - f"_update_user_db: existing spend: {existing_spend_obj}; response_cost: {response_cost}" + "_update_user_db: existing spend: %s; response_cost: %s", existing_spend_obj, response_cost ) existing_spend = existing_spend_obj.spend or 0.0 @@ -2939,7 +2941,7 @@ async def update_cache( if existing_spend_obj is None: return verbose_proxy_logger.debug( - f"_update_end_user_db: existing spend: {existing_spend_obj}; response_cost: {response_cost}" + "_update_end_user_db: existing spend: %s; response_cost: %s", existing_spend_obj, response_cost ) existing_spend = existing_spend_obj.spend or 0.0 @@ -2981,7 +2983,7 @@ async def update_cache( if existing_spend_obj is None: return verbose_proxy_logger.debug( - f"_update_team_db: existing spend: {existing_spend_obj}; response_cost: {response_cost}" + "_update_team_db: existing spend: %s; response_cost: %s", existing_spend_obj, response_cost ) existing_spend: float = existing_spend_obj.spend or 0.0 @@ -3031,7 +3033,10 @@ async def update_cache( continue verbose_proxy_logger.debug( - f"_update_tag_cache: existing spend for tag={tag_name}: {existing_tag_obj}; response_cost: {response_cost}" + "_update_tag_cache: existing spend for tag=%s: %s; response_cost: %s", + tag_name, + existing_tag_obj, + response_cost, ) existing_spend = existing_tag_obj.spend or 0.0 @@ -3101,9 +3106,10 @@ def run_ollama_serve(): with open(os.devnull, "w") as devnull: subprocess.Popen(command, stdout=devnull, stderr=devnull) except Exception as e: - verbose_proxy_logger.debug(f""" - LiteLLM Warning: proxy started with `ollama` model\n`ollama serve` failed with Exception{e}. \nEnsure you run `ollama serve` - """) + verbose_proxy_logger.debug( + "\n LiteLLM Warning: proxy started with `ollama` model\n`ollama serve` failed with Exception%s. \nEnsure you run `ollama serve`\n ", + e, + ) def _get_process_rss_mb() -> float | None: @@ -4026,7 +4032,7 @@ class ProxyConfig: dict: Processed configuration dictionary. """ if depth > max_depth: - verbose_proxy_logger.warning(f"Maximum recursion depth ({max_depth}) reached while processing config.") + verbose_proxy_logger.warning("Maximum recursion depth (%s) reached while processing config.", max_depth) return config for key, value in config.items(): @@ -4230,7 +4236,9 @@ class ProxyConfig: return copy.deepcopy(self.config) except Exception as e: verbose_proxy_logger.debug( - f"ProxyConfig:get_config_state(): Error returning copy of config state. self.config={self.config}\nError: {e}" + "ProxyConfig:get_config_state(): Error returning copy of config state. self.config=%s\nError: %s", + self.config, + e, ) return {} @@ -4294,7 +4302,7 @@ class ProxyConfig: search_tool_typed: SearchToolTypedDict = SearchToolTypedDict(**search_tool) # type: ignore search_tools_parsed.append(search_tool_typed) except Exception as e: - verbose_proxy_logger.error(f"Error parsing search tool {search_tool_name}: {e}") + verbose_proxy_logger.error("Error parsing search tool %s: %s", search_tool_name, e) continue return search_tools_parsed if search_tools_parsed else None @@ -4484,7 +4492,7 @@ class ProxyConfig: ) ) if litellm.cache is not None: - verbose_proxy_logger.debug(f"{blue_color_code}Set Cache on LiteLLM Proxy{reset_color_code}") + verbose_proxy_logger.debug("%sSet Cache on LiteLLM Proxy%s", blue_color_code, reset_color_code) elif key == "cache" and value is False: pass elif key == "guardrails": @@ -4504,7 +4512,7 @@ class ProxyConfig: set_global_prompt_directory(value) verbose_proxy_logger.info( - f"{blue_color_code}Set Global Prompt Directory on LiteLLM Proxy{reset_color_code}" + "%sSet Global Prompt Directory on LiteLLM Proxy%s", blue_color_code, reset_color_code ) elif key == "global_bitbucket_config": from litellm.integrations.bitbucket import ( @@ -4513,14 +4521,14 @@ class ProxyConfig: set_global_bitbucket_config(value) verbose_proxy_logger.info( - f"{blue_color_code}Set Global BitBucket Config on LiteLLM Proxy{reset_color_code}" + "%sSet Global BitBucket Config on LiteLLM Proxy%s", blue_color_code, reset_color_code ) elif key == "global_gitlab_config": from litellm.integrations.gitlab import set_global_gitlab_config set_global_gitlab_config(value) verbose_proxy_logger.info( - f"{blue_color_code}Set Global Gitlab Config on LiteLLM Proxy{reset_color_code}" + "%sSet Global Gitlab Config on LiteLLM Proxy%s", blue_color_code, reset_color_code ) elif key == "priority_reservation_settings": from litellm.types.utils import PriorityReservationSettings @@ -4542,7 +4550,7 @@ class ProxyConfig: elif key == "post_call_rules": litellm.post_call_rules = [get_instance_fn(value=value, config_file_path=config_file_path)] - verbose_proxy_logger.debug(f"litellm.post_call_rules: {litellm.post_call_rules}") + verbose_proxy_logger.debug("litellm.post_call_rules: %s", litellm.post_call_rules) elif key == "max_budget": litellm.max_budget = float(value) elif key == "max_internal_user_budget": @@ -4558,7 +4566,11 @@ class ProxyConfig: else value ) verbose_proxy_logger.debug( - f"{blue_color_code} setting litellm.{key}={_redact_general_setting_value(key, litellm.default_internal_user_params, is_full_admin=False)}{reset_color_code}" + "%s setting litellm.%s=%s%s", + blue_color_code, + key, + _redact_general_setting_value(key, litellm.default_internal_user_params, is_full_admin=False), + reset_color_code, ) elif key == "custom_provider_map": from litellm.utils import custom_llm_setup @@ -4662,12 +4674,20 @@ class ProxyConfig: native_background_mode = background_mode.get("native_background_mode", []) polling_cache_ttl = background_mode.get("ttl", 3600) verbose_proxy_logger.debug( - f"{blue_color_code} Initialized polling via cache: enabled={polling_via_cache_enabled}, native_background_mode={native_background_mode}, ttl={polling_cache_ttl}{reset_color_code}" + "%s Initialized polling via cache: enabled=%s, native_background_mode=%s, ttl=%s%s", + blue_color_code, + polling_via_cache_enabled, + native_background_mode, + polling_cache_ttl, + reset_color_code, ) elif key == "max_ui_session_budget": litellm.max_ui_session_budget = float(value) if value is not None else None verbose_proxy_logger.debug( - f"{blue_color_code} setting litellm.max_ui_session_budget={litellm.max_ui_session_budget}{reset_color_code}" + "%s setting litellm.max_ui_session_budget=%s%s", + blue_color_code, + litellm.max_ui_session_budget, + reset_color_code, ) elif key == "default_team_settings": for idx, team_setting in enumerate(value): # run through pydantic validation @@ -4682,7 +4702,11 @@ class ProxyConfig: f"team_id missing from default_team_settings at index={idx}\npassed in value={type(team_setting)}" ) verbose_proxy_logger.debug( - f"{blue_color_code} setting litellm.{key}={_redact_general_setting_value(key, value, is_full_admin=False)}{reset_color_code}" + "%s setting litellm.%s=%s%s", + blue_color_code, + key, + _redact_general_setting_value(key, value, is_full_admin=False), + reset_color_code, ) setattr(litellm, key, value) elif key == "upperbound_key_generate_params": @@ -4696,7 +4720,9 @@ class ProxyConfig: elif key == "json_logs" and value is True: litellm.json_logs = True litellm._turn_on_json() - verbose_proxy_logger.debug(f"{blue_color_code} Enabled JSON logging via config{reset_color_code}") + verbose_proxy_logger.debug( + "%s Enabled JSON logging via config%s", blue_color_code, reset_color_code + ) elif key == "budget_reset_time": from litellm.proxy.common_utils.timezone_utils import ( parse_budget_reset_time, @@ -4706,7 +4732,11 @@ class ProxyConfig: setattr(litellm, key, value) else: verbose_proxy_logger.debug( - f"{blue_color_code} setting litellm.{key}={_redact_general_setting_value(key, value, is_full_admin=False)}{reset_color_code}" + "%s setting litellm.%s=%s%s", + blue_color_code, + key, + _redact_general_setting_value(key, value, is_full_admin=False), + reset_color_code, ) setattr(litellm, key, value) if key == "request_timeout": @@ -5035,7 +5065,7 @@ class ProxyConfig: ) else: verbose_proxy_logger.warning( - f"Key '{k}' is not a valid argument for Router.__init__(). Ignoring this key." + "Key '%s' is not a valid argument for Router.__init__(). Ignoring this key.", k ) router = litellm.Router( **router_params, @@ -5168,7 +5198,7 @@ class ProxyConfig: policy_attachments_config = config.get("policy_attachments", None) - verbose_proxy_logger.info(f"Policy engine: found {len(policies_config)} policies in config") + verbose_proxy_logger.info("Policy engine: found %s policies in config", len(policies_config)) # Initialize policies await init_policies( @@ -5191,7 +5221,7 @@ class ProxyConfig: # Ensure proxy_logging_obj.alerting is set for all alerting types _alerting_value = general_settings.get("alerting", None) - verbose_proxy_logger.debug(f"_load_alerting_settings: Calling update_values with alerting={_alerting_value}") + verbose_proxy_logger.debug("_load_alerting_settings: Calling update_values with alerting=%s", _alerting_value) proxy_logging_obj.update_values( alerting=_alerting_value, alerting_threshold=general_settings.get("alerting_threshold", 600), @@ -5409,7 +5439,7 @@ class ProxyConfig: else: verbose_proxy_logger.error( - f"Invalid model added to proxy db. Invalid litellm params. litellm_params={_litellm_params}" + "Invalid model added to proxy db. Invalid litellm params. litellm_params=%s", _litellm_params ) continue # skip to next model _model_info = self.get_model_info_with_id(model=m, db_model=True) ## 👈 FLAG = True for db_models @@ -5439,7 +5469,7 @@ class ProxyConfig: _litellm_params = LiteLLM_Params.model_validate(_litellm_params) else: verbose_proxy_logger.error( - f"Invalid model added to proxy db. Invalid litellm params. litellm_params={_litellm_params}" + "Invalid model added to proxy db. Invalid litellm params. litellm_params=%s", _litellm_params ) continue # skip to next model @@ -5488,13 +5518,13 @@ class ProxyConfig: models_list: list = new_models if isinstance(new_models, list) else [] if llm_router is None and master_key is not None: - verbose_proxy_logger.debug(f"len new_models: {len(models_list)}") + verbose_proxy_logger.debug("len new_models: %s", len(models_list)) _model_list: list = self.decrypt_model_list_from_db(new_models=models_list) # Only create router if we have models or search_tools to route # Router can function with model_list=[] if search_tools are configured if len(_model_list) > 0 or search_tools: - verbose_proxy_logger.debug(f"_model_list: {_model_list}") + verbose_proxy_logger.debug("_model_list: %s", _model_list) llm_router = litellm.Router( model_list=_model_list, router_general_settings=RouterGeneralSettings( @@ -5503,9 +5533,9 @@ class ProxyConfig: search_tools=search_tools, ignore_invalid_deployments=True, ) - verbose_proxy_logger.debug(f"updated llm_router: {llm_router}") + verbose_proxy_logger.debug("updated llm_router: %s", llm_router) else: - verbose_proxy_logger.debug(f"len new_models: {len(models_list)}") + verbose_proxy_logger.debug("len new_models: %s", len(models_list)) if search_tools is not None and llm_router is not None: llm_router.search_tools = search_tools ## DELETE MODEL LOGIC @@ -5515,7 +5545,7 @@ class ProxyConfig: self._add_deployment(db_models=models_list) except Exception as e: - verbose_proxy_logger.exception(f"Error adding/deleting model to llm_router: {e}") + verbose_proxy_logger.exception("Error adding/deleting model to llm_router: %s", e) if llm_router is not None: llm_model_list = llm_router.get_model_list() @@ -5800,7 +5830,10 @@ class ProxyConfig: item for item in _general_settings["alerting"] if item not in general_settings["alerting"] ] verbose_proxy_logger.debug( - f"Merging alerting values: YAML={general_settings['alerting']}, DB={_general_settings['alerting']}, Merged={_merged_alerting}" + "Merging alerting values: YAML=%s, DB=%s, Merged=%s", + general_settings["alerting"], + _general_settings["alerting"], + _merged_alerting, ) general_settings["alerting"] = _merged_alerting # Use update_values to properly set alerting for both slack and email @@ -5879,9 +5912,9 @@ class ProxyConfig: replace_existing=True, misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, ) - verbose_proxy_logger.info(f"Spend log cleanup rescheduled with cron: {cleanup_cron}") + verbose_proxy_logger.info("Spend log cleanup rescheduled with cron: %s", cleanup_cron) except ValueError: - verbose_proxy_logger.error(f"Invalid maximum_spend_logs_cleanup_cron value: {cleanup_cron}") + verbose_proxy_logger.error("Invalid maximum_spend_logs_cleanup_cron value: %s", cleanup_cron) else: # Interval-based scheduling (existing behavior) from litellm.litellm_core_utils.duration_parser import ( @@ -5900,7 +5933,7 @@ class ProxyConfig: replace_existing=True, misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, ) - verbose_proxy_logger.info(f"Spend log cleanup rescheduled with interval: {retention_interval}") + verbose_proxy_logger.info("Spend log cleanup rescheduled with interval: %s", retention_interval) except ValueError: verbose_proxy_logger.error("Invalid maximum_spend_logs_retention_interval value") @@ -6135,7 +6168,7 @@ class ProxyConfig: # If supported_db_objects is set, only load specified objects if not isinstance(supported_db_objects, list): verbose_proxy_logger.warning( - f"supported_db_objects is not a list, got {type(supported_db_objects)}. Loading all objects." + "supported_db_objects is not a list, got %s. Loading all objects.", type(supported_db_objects) ) return True @@ -6159,7 +6192,7 @@ class ProxyConfig: return new_models except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy_server.py::add_deployment() - Error getting new models from DB - {e}" + "litellm.proxy_server.py::add_deployment() - Error getting new models from DB - %s", e ) return None @@ -6216,7 +6249,7 @@ class ProxyConfig: await self._init_non_llm_objects_in_db(prisma_client=prisma_client) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - %s", e) return still_desired_ids @@ -6250,7 +6283,7 @@ class ProxyConfig: try: await subscriber.stop() except Exception as e: - verbose_proxy_logger.error(f"Error stopping config sync subscriber: {e}") + verbose_proxy_logger.error("Error stopping config sync subscriber: %s", e) async def _init_non_llm_objects_in_db(self, prisma_client: PrismaClient): """ @@ -6371,7 +6404,7 @@ class ProxyConfig: self._last_semantic_filter_config = mcp_semantic_filter_config.copy() except Exception as e: - verbose_proxy_logger.exception(f"Error initializing semantic filter settings from DB: {e}") + verbose_proxy_logger.exception("Error initializing semantic filter settings from DB: %s", e) async def _init_sso_settings_in_db(self, prisma_client: PrismaClient): """ @@ -6391,7 +6424,9 @@ class ProxyConfig: uppercase_sso_settings = {key.upper(): value for key, value in sso_settings.sso_settings.items()} self._decrypt_and_set_db_env_variables(environment_variables=uppercase_sso_settings) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_sso_settings_in_db - {e}") + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.py::ProxyConfig:_init_sso_settings_in_db - %s", e + ) async def _init_hashicorp_vault_config_override(self, prisma_client: PrismaClient): """ @@ -6492,7 +6527,7 @@ class ProxyConfig: f"Model cost map reload triggered by interval. Hours since last reload: {hours_since_last_reload:.2f}, Interval: {interval_hours}" ) except Exception as e: - verbose_proxy_logger.warning(f"Error parsing last reload time: {e}") + verbose_proxy_logger.warning("Error parsing last reload time: %s", e) # If we can't parse the last reload time, reload anyway should_reload = True else: @@ -6544,11 +6579,12 @@ class ProxyConfig: await evict_config_param("model_cost_map_reload_config") verbose_proxy_logger.info( - f"Model cost map reloaded successfully. Models count: {len(new_model_cost_map) if new_model_cost_map else 0}" + "Model cost map reloaded successfully. Models count: %s", + len(new_model_cost_map) if new_model_cost_map else 0, ) except Exception as e: - verbose_proxy_logger.exception(f"Error in _check_and_reload_model_cost_map: {e}") + verbose_proxy_logger.exception("Error in _check_and_reload_model_cost_map: %s", e) async def _check_and_reload_anthropic_beta_headers(self, prisma_client: PrismaClient): """ @@ -6592,7 +6628,7 @@ class ProxyConfig: f"Anthropic beta headers reload triggered by interval. Hours since last reload: {hours_since_last_reload:.2f}, Interval: {interval_hours}" ) except Exception as e: - verbose_proxy_logger.warning(f"Error parsing last reload time: {e}") + verbose_proxy_logger.warning("Error parsing last reload time: %s", e) # If we can't parse the last reload time, reload anyway should_reload = True else: @@ -6641,11 +6677,11 @@ class ProxyConfig: # Count providers in config provider_count = sum(1 for k in new_config.keys() if k != "provider_aliases" and k != "description") verbose_proxy_logger.info( - f"Anthropic beta headers config reloaded successfully. Providers: {provider_count}" + "Anthropic beta headers config reloaded successfully. Providers: %s", provider_count ) except Exception as e: - verbose_proxy_logger.exception(f"Error in _check_and_reload_anthropic_beta_headers: {e}") + verbose_proxy_logger.exception("Error in _check_and_reload_anthropic_beta_headers: %s", e) def _get_prompt_spec_for_db_prompt(self, db_prompt): """ @@ -6674,7 +6710,7 @@ class ProxyConfig: prompt_spec = self._get_prompt_spec_for_db_prompt(db_prompt=prompt) IN_MEMORY_PROMPT_REGISTRY.initialize_prompt(prompt=prompt_spec) except Exception as e: - verbose_proxy_logger.debug(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_prompts_in_db - {e}") + verbose_proxy_logger.debug("litellm.proxy.proxy_server.py::ProxyConfig:_init_prompts_in_db - %s", e) async def _init_guardrails_in_db(self, prisma_client: PrismaClient): from litellm.proxy.guardrails.guardrail_registry import ( @@ -6701,7 +6737,7 @@ class ProxyConfig: # pod. Config-loaded entries are never touched. IN_MEMORY_GUARDRAIL_HANDLER.reconcile_db_guardrails(db_guardrail_ids=db_guardrail_ids) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_guardrails_in_db - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:_init_guardrails_in_db - %s", e) async def _init_policies_in_db(self, prisma_client: PrismaClient): """ @@ -6725,7 +6761,7 @@ class ProxyConfig: verbose_proxy_logger.debug("Successfully synced policies and attachments from DB") except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_policies_in_db - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:_init_policies_in_db - %s", e) async def _init_tool_policy_in_db(self, prisma_client: PrismaClient): """ @@ -6739,7 +6775,7 @@ class ProxyConfig: await registry.sync_tool_policy_from_db(prisma_client=prisma_client) verbose_proxy_logger.debug("Successfully synced tool policy from DB") except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_tool_policy_in_db - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:_init_tool_policy_in_db - %s", e) async def _init_vector_stores_in_db(self, prisma_client: PrismaClient): from litellm.vector_stores.vector_store_registry import VectorStoreRegistry @@ -6757,7 +6793,7 @@ class ProxyConfig: litellm.vector_store_registry.add_vector_store_to_registry(vector_store=vector_store) except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.py::ProxyConfig:_init_vector_stores_in_db - {e}" + "litellm.proxy.proxy_server.py::ProxyConfig:_init_vector_stores_in_db - %s", e ) async def _init_vector_store_indexes_in_db(self, prisma_client: PrismaClient): @@ -6781,7 +6817,7 @@ class ProxyConfig: litellm.vector_store_index_registry.upsert_vector_store_index(vector_store_index=vector_store_index) except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.py::ProxyConfig:_init_vector_stores_in_db - {e}" + "litellm.proxy.proxy_server.py::ProxyConfig:_init_vector_stores_in_db - %s", e ) async def _init_mcp_servers_in_db(self): @@ -6806,7 +6842,7 @@ class ProxyConfig: await backfill_null_oauth2_flows(prisma_client) except Exception as e: # noqa: BLE001 verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db backfill - {e}" + "litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db backfill - %s", e ) try: @@ -6814,13 +6850,13 @@ class ProxyConfig: await backfill_discovery_stamped_issuers(prisma_client) except Exception as e: # noqa: BLE001 verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db issuer stamp backfill - {e}" + "litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db issuer stamp backfill - %s", e ) try: await global_mcp_server_manager.reload_servers_from_database() except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db - %s", e) async def init_mcp_servers_from_db(self) -> None: if self._should_load_db_object(object_type="mcp"): @@ -6848,7 +6884,7 @@ class ProxyConfig: await global_mcp_server_manager.reload_servers_from_database() except Exception as e: # noqa: BLE001 # scheduled job: a reload failure must not kill the recurring retry verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.py::ProxyConfig:reload_mcp_servers_from_db - {e}" + "litellm.proxy.proxy_server.py::ProxyConfig:reload_mcp_servers_from_db - %s", e ) async def _init_agents_in_db(self, prisma_client: PrismaClient): @@ -6860,7 +6896,7 @@ class ProxyConfig: db_agents = await AGENT_REGISTRY.get_all_agents_from_db(prisma_client=prisma_client) AGENT_REGISTRY.load_agents_from_db_and_config(db_agents=db_agents) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_agents_in_db - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:_init_agents_in_db - %s", e) async def _init_search_tools_in_db(self, prisma_client: PrismaClient): """ @@ -6885,13 +6921,15 @@ class ProxyConfig: ) verbose_proxy_logger.info( - f"Loading {len(search_tools)} search tool(s) into router " - f"({len(config_search_tools)} from config, {len(db_search_tools)} from database)" + "Loading %s search tool(s) into router (%s from config, %s from database)", + len(search_tools), + len(config_search_tools), + len(db_search_tools), ) if llm_router is not None and search_tools: await SearchAPIRouter.update_router_search_tools(router_instance=llm_router, search_tools=search_tools) - verbose_proxy_logger.info(f"Successfully loaded {len(search_tools)} search tool(s) into router") + verbose_proxy_logger.info("Successfully loaded %s search tool(s) into router", len(search_tools)) elif llm_router is not None: verbose_proxy_logger.debug("No search tools found in config or database, skipping router update") else: @@ -6900,7 +6938,9 @@ class ProxyConfig: ) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.py::ProxyConfig:_init_search_tools_in_db - {e}") + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.py::ProxyConfig:_init_search_tools_in_db - %s", e + ) @staticmethod def _merge_config_and_db_search_tools( @@ -6966,7 +7006,7 @@ class ProxyConfig: CredentialAccessor.upsert_credentials(credentials) # upsert credentials that are in the all-up list except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy_server.py::get_credentials() - Error getting credentials from DB - {e}" + "litellm.proxy_server.py::get_credentials() - Error getting credentials from DB - %s", e ) return [] @@ -7153,7 +7193,7 @@ async def async_assistants_data_generator(response, user_api_key_dict: UserAPIKe yield f"data: {done_message}\n\n" except Exception as e: verbose_proxy_logger.exception( - f"litellm.proxy.proxy_server.async_assistants_data_generator(): Exception occured - {e}" + "litellm.proxy.proxy_server.async_assistants_data_generator(): Exception occured - %s", e ) await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, @@ -7161,7 +7201,8 @@ async def async_assistants_data_generator(response, user_api_key_dict: UserAPIKe request_data=request_data, ) verbose_proxy_logger.debug( - f"\033[1;31mAn error occurred: {e}\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`" + "\x1b[1;31mAn error occurred: %s\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`", + e, ) if isinstance(e, HTTPException): raise e @@ -7632,14 +7673,15 @@ async def async_data_generator( client_disconnected = True raise except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.async_data_generator(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.async_data_generator(): Exception occured - %s", e) await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=request_data, ) verbose_proxy_logger.debug( - f"\033[1;31mAn error occurred: {e}\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`" + "\x1b[1;31mAn error occurred: %s\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`", + e, ) if isinstance(e, HTTPException): @@ -7881,8 +7923,9 @@ class ProxyStartupEvent: return verbose_proxy_logger.debug( - f"Initializing semantic tool filter: llm_router={llm_router is not None}, " - f"config={mcp_semantic_filter_config}" + "Initializing semantic tool filter: llm_router=%s, config=%s", + llm_router is not None, + mcp_semantic_filter_config, ) hook = await SemanticToolFilterHook.initialize_from_config( config=mcp_semantic_filter_config, @@ -8274,9 +8317,9 @@ class ProxyStartupEvent: replace_existing=True, misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, ) - verbose_proxy_logger.info(f"Spend log cleanup scheduled with cron: {cleanup_cron}") + verbose_proxy_logger.info("Spend log cleanup scheduled with cron: %s", cleanup_cron) except ValueError: - verbose_proxy_logger.error(f"Invalid maximum_spend_logs_cleanup_cron value: {cleanup_cron}") + verbose_proxy_logger.error("Invalid maximum_spend_logs_cleanup_cron value: %s", cleanup_cron) else: # Interval-based scheduling (existing behavior) retention_interval = general_settings.get("maximum_spend_logs_retention_interval", "1d") @@ -8318,7 +8361,7 @@ class ProxyStartupEvent: verbose_proxy_logger.info("Batch cost check job scheduled successfully") except Exception as e: - verbose_proxy_logger.debug(f"Failed to setup batch cost checking: {e}") + verbose_proxy_logger.debug("Failed to setup batch cost checking: %s", e) verbose_proxy_logger.debug( "Checking batch cost for LiteLLM Managed Files is an Enterprise Feature. Skipping..." ) @@ -8347,7 +8390,7 @@ class ProxyStartupEvent: verbose_proxy_logger.info("Responses cost check job scheduled successfully") except Exception as e: - verbose_proxy_logger.debug(f"Failed to setup responses cost checking: {e}") + verbose_proxy_logger.debug("Failed to setup responses cost checking: %s", e) verbose_proxy_logger.debug( "Checking responses cost for LiteLLM Managed Files is an Enterprise Feature. Skipping..." ) @@ -8359,8 +8402,8 @@ class ProxyStartupEvent: # Start the scheduler immediately without processing backlogs scheduler.start(paused=False) verbose_proxy_logger.info( - f"APScheduler started with memory leak prevention settings: " - f"removed jitter, increased intervals, misfire_grace_time={APSCHEDULER_MISFIRE_GRACE_TIME}" + "APScheduler started with memory leak prevention settings: removed jitter, increased intervals, misfire_grace_time=%s", + APSCHEDULER_MISFIRE_GRACE_TIME, ) @classmethod @@ -8448,7 +8491,7 @@ class ProxyStartupEvent: ) key_rotation_enabled: bool | None = str_to_bool(LITELLM_KEY_ROTATION_ENABLED) - verbose_proxy_logger.debug(f"key_rotation_enabled: {key_rotation_enabled}") + verbose_proxy_logger.debug("key_rotation_enabled: %s", key_rotation_enabled) if key_rotation_enabled is True: try: @@ -8465,7 +8508,8 @@ class ProxyStartupEvent: pod_lock_manager=pod_lock_manager, ) verbose_proxy_logger.debug( - f"Key rotation background job scheduled every {LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS} seconds (LITELLM_KEY_ROTATION_ENABLED=true)" + "Key rotation background job scheduled every %s seconds (LITELLM_KEY_ROTATION_ENABLED=true)", + LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS, ) scheduler.add_job( key_rotation_manager.process_rotations, @@ -8476,7 +8520,7 @@ class ProxyStartupEvent: else: verbose_proxy_logger.warning("Key rotation enabled but prisma_client not available") except Exception as e: - verbose_proxy_logger.warning(f"Failed to setup key rotation job: {e}") + verbose_proxy_logger.warning("Failed to setup key rotation job: %s", e) else: verbose_proxy_logger.debug("Key rotation disabled (set LITELLM_KEY_ROTATION_ENABLED=true to enable)") @@ -8503,7 +8547,7 @@ class ProxyStartupEvent: expired_ui_session_key_cleanup_enabled: bool | None = str_to_bool( LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_ENABLED ) - verbose_proxy_logger.debug(f"expired_ui_session_key_cleanup_enabled: {expired_ui_session_key_cleanup_enabled}") + verbose_proxy_logger.debug("expired_ui_session_key_cleanup_enabled: %s", expired_ui_session_key_cleanup_enabled) if expired_ui_session_key_cleanup_enabled is True: try: @@ -8519,11 +8563,8 @@ class ProxyStartupEvent: pod_lock_manager=pod_lock_manager, ) verbose_proxy_logger.debug( - "Expired UI session key cleanup background job scheduled " - "every " - f"{LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_INTERVAL_SECONDS} " - "seconds " - "(LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_ENABLED=true)" + "Expired UI session key cleanup background job scheduled every %s seconds (LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_ENABLED=true)", + LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_INTERVAL_SECONDS, ) scheduler.add_job( expired_ui_session_key_cleanup_manager.cleanup_expired_keys, @@ -8536,7 +8577,7 @@ class ProxyStartupEvent: "Expired UI session key cleanup enabled but prisma_client not available" ) except Exception as e: - verbose_proxy_logger.warning(f"Failed to setup expired UI session key cleanup job: {e}") + verbose_proxy_logger.warning("Failed to setup expired UI session key cleanup job: %s", e) else: verbose_proxy_logger.debug( "Expired UI session key cleanup disabled (set " @@ -9383,7 +9424,7 @@ async def completion( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.completion(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.completion(): Exception occured - %s", e) error_msg = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), @@ -9622,7 +9663,7 @@ async def moderations( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.moderations(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.moderations(): Exception occured - %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "message", str(e)), @@ -9768,7 +9809,7 @@ async def audio_speech( original_exception=e, request_data=data, ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.audio_speech(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.audio_speech(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) raise e @@ -9910,7 +9951,7 @@ async def audio_transcriptions( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.audio_transcription(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.audio_transcription(): Exception occured - %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "message", str(e.detail)), @@ -10196,7 +10237,7 @@ async def get_assistants( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.get_assistants(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.get_assistants(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10287,7 +10328,7 @@ async def create_assistant( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.create_assistant(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.create_assistant(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10376,7 +10417,7 @@ async def delete_assistant( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.delete_assistant(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.delete_assistant(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10465,7 +10506,7 @@ async def create_threads( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.create_threads(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.create_threads(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10552,7 +10593,7 @@ async def get_thread( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.get_thread(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.get_thread(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10643,7 +10684,7 @@ async def add_messages( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.add_messages(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.add_messages(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10730,7 +10771,7 @@ async def get_messages( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.get_messages(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.get_messages(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10831,7 +10872,7 @@ async def run_thread( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.run_thread(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.run_thread(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -10970,8 +11011,9 @@ async def _try_provider_token_count( code=result.status_code or 500, ) verbose_proxy_logger.warning( - f"Provider token counting failed ({result.status_code}): {result.error_message}. " - "Falling back to local tokenizer." + "Provider token counting failed (%s): %s. Falling back to local tokenizer.", + result.status_code, + result.error_message, ) return None return result @@ -11768,7 +11810,7 @@ async def _apply_search_filter_to_models( ) search_total_count = router_models_count + db_models_total_count except Exception as e: - verbose_proxy_logger.exception(f"Error querying database models with search: {e}") + verbose_proxy_logger.exception("Error querying database models with search: %s", e) search_total_count = router_models_count else: search_total_count = router_models_count @@ -11903,7 +11945,7 @@ def _sort_models( sorted_models = sorted(all_models, key=get_sort_key, reverse=reverse) return sorted_models except Exception as e: - verbose_proxy_logger.exception(f"Error sorting models by {sort_by}: {e}") + verbose_proxy_logger.exception("Error sorting models by %s: %s", sort_by, e) return all_models @@ -11951,7 +11993,12 @@ def _paginate_models_response( paginated_models = all_models[skip : skip + size] verbose_proxy_logger.debug( - f"Pagination: skip={skip}, take={size}, total_count={total_count}, total_pages={total_pages}, search={search}" + "Pagination: skip=%s, take=%s, total_count=%s, total_pages=%s, search=%s", + skip, + size, + total_count, + total_pages, + search, ) return { @@ -11979,11 +12026,11 @@ async def _load_team_object_for_model_filter(team_id: str, prisma_client: Prisma try: team_db_object = await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id}) if team_db_object is None: - verbose_proxy_logger.warning(f"Team {team_id} not found in database") + verbose_proxy_logger.warning("Team %s not found in database", team_id) return None return LiteLLM_TeamTable.model_validate(team_db_object.model_dump()) except Exception as e: - verbose_proxy_logger.exception(f"Error fetching team {team_id}: {e}") + verbose_proxy_logger.exception("Error fetching team %s: %s", team_id, e) return None @@ -12033,7 +12080,7 @@ async def _gather_team_accessible_model_ids( if db_model.model_id: team_accessible_model_ids.add(db_model.model_id) except Exception as e: - verbose_proxy_logger.debug(f"Error querying database models for team {team_id}: {e}") + verbose_proxy_logger.debug("Error querying database models for team %s: %s", team_id, e) return team_accessible_model_ids @@ -12171,7 +12218,7 @@ async def _find_model_by_id( if decrypted_models: found_model = decrypted_models[0] except Exception as e: - verbose_proxy_logger.exception(f"Error querying database for modelId {model_id}: {e}") + verbose_proxy_logger.exception("Error querying database for modelId %s: %s", model_id, e) # If model found, verify search filter if provided if found_model is not None: @@ -13787,7 +13834,7 @@ async def login_v2(request: Request): json_response.set_cookie(key="token", value=jwt_token) return json_response except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.login_v2(): Exception occurred - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.login_v2(): Exception occurred - %s", e) if isinstance(e, ProxyException): raise e elif isinstance(e, HTTPException): @@ -13864,7 +13911,7 @@ async def login_v3(request: Request): status_code=status.HTTP_200_OK, ) except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.login_v3(): Exception occurred - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.login_v3(): Exception occurred - %s", e) if isinstance(e, ProxyException): raise e elif isinstance(e, HTTPException): @@ -13937,7 +13984,7 @@ async def login_v3_exchange(request: Request): except ProxyException: raise except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.login_v3_exchange(): Exception occurred - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.login_v3_exchange(): Exception occurred - %s", e) raise ProxyException( message=str(e), type=ProxyErrorTypes.auth_error, @@ -14305,11 +14352,12 @@ async def get_image(): if not os.path.exists(assets_dir): try: os.makedirs(assets_dir, exist_ok=True) - verbose_proxy_logger.debug(f"Created assets directory at {assets_dir}") + verbose_proxy_logger.debug("Created assets directory at %s", assets_dir) except (PermissionError, OSError) as e: verbose_proxy_logger.warning( - f"Cannot create assets directory at {assets_dir}: {e}. " - f"Logo caching may not work. Using current directory for assets." + "Cannot create assets directory at %s: %s. Logo caching may not work. Using current directory for assets.", + assets_dir, + e, ) assets_dir = current_dir @@ -14764,7 +14812,7 @@ async def update_config( return {"message": "Config updated successfully"} except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.update_config(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.update_config(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( @@ -15592,7 +15640,7 @@ async def delete_callback( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.delete_callback(): Exception occurred - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.delete_callback(): Exception occurred - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) raise ProxyException( message="Error deleting callback: " + str(e), @@ -15716,7 +15764,7 @@ async def get_config( "available_callbacks": all_available_callbacks, } except Exception as e: - verbose_proxy_logger.exception(f"litellm.proxy.proxy_server.get_config(): Exception occured - {e}") + verbose_proxy_logger.exception("litellm.proxy.proxy_server.get_config(): Exception occured - %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"Authentication Error({e})"), @@ -15825,7 +15873,7 @@ async def reload_model_cost_map( await invalidate_config_param("model_cost_map_reload_config") models_count = len(new_model_cost_map) if new_model_cost_map else 0 - verbose_proxy_logger.info(f"Model cost map reloaded successfully in current pod. Models count: {models_count}") + verbose_proxy_logger.info("Model cost map reloaded successfully in current pod. Models count: %s", models_count) return { "message": f"Price data reloaded successfully! {models_count} models updated.", @@ -15834,7 +15882,7 @@ async def reload_model_cost_map( "timestamp": current_time.isoformat(), } except Exception as e: - verbose_proxy_logger.exception(f"Failed to reload model cost map: {e}") + verbose_proxy_logger.exception("Failed to reload model cost map: %s", e) raise HTTPException(status_code=500, detail=f"Failed to reload model cost map: {e}") @@ -15882,7 +15930,7 @@ async def schedule_model_cost_map_reload( ) await invalidate_config_param("model_cost_map_reload_config") - verbose_proxy_logger.info(f"Model cost map reload scheduled for every {hours} hours") + verbose_proxy_logger.info("Model cost map reload scheduled for every %s hours", hours) return { "message": f"Model cost map reload scheduled for every {hours} hours", @@ -15891,7 +15939,7 @@ async def schedule_model_cost_map_reload( "timestamp": datetime.utcnow().isoformat(), } except Exception as e: - verbose_proxy_logger.exception(f"Failed to schedule model cost map reload: {e}") + verbose_proxy_logger.exception("Failed to schedule model cost map reload: %s", e) raise HTTPException( status_code=500, detail=f"Failed to schedule model cost map reload: {e}", @@ -15936,7 +15984,7 @@ async def cancel_model_cost_map_reload( "timestamp": datetime.utcnow().isoformat(), } except Exception as e: - verbose_proxy_logger.exception(f"Failed to cancel model cost map reload: {e}") + verbose_proxy_logger.exception("Failed to cancel model cost map reload: %s", e) raise HTTPException(status_code=500, detail=f"Failed to cancel model cost map reload: {e}") @@ -15964,7 +16012,7 @@ async def get_model_cost_map_reload_status( try: global prisma_client, last_model_cost_map_reload - verbose_proxy_logger.info(f"Checking model cost map reload status. Last reload: {last_model_cost_map_reload}") + verbose_proxy_logger.info("Checking model cost map reload status. Last reload: %s", last_model_cost_map_reload) if prisma_client is None: verbose_proxy_logger.info("No database connection, returning not scheduled") @@ -16014,7 +16062,7 @@ async def get_model_cost_map_reload_status( if hours_since_last_reload < interval_hours: next_run = (last_reload_time + timedelta(hours=interval_hours)).isoformat() except Exception as e: - verbose_proxy_logger.warning(f"Error parsing last reload time: {e}") + verbose_proxy_logger.warning("Error parsing last reload time: %s", e) return { "scheduled": True, @@ -16023,7 +16071,7 @@ async def get_model_cost_map_reload_status( "next_run": next_run, } except Exception as e: - verbose_proxy_logger.exception(f"Failed to get model cost map reload status: {e}") + verbose_proxy_logger.exception("Failed to get model cost map reload status: %s", e) raise HTTPException( status_code=500, detail=f"Failed to get model cost map reload status: {e}", @@ -16071,7 +16119,7 @@ async def get_model_cost_map_source( "model_count": model_count, } except Exception as e: - verbose_proxy_logger.exception(f"Failed to get model cost map source info: {e}") + verbose_proxy_logger.exception("Failed to get model cost map source info: %s", e) raise HTTPException( status_code=500, detail=f"Failed to get model cost map source info: {e}", @@ -16140,7 +16188,7 @@ async def reload_anthropic_beta_headers( provider_count = sum(1 for k in new_config.keys() if k not in ["provider_aliases", "description"]) verbose_proxy_logger.info( - f"Anthropic beta headers config reloaded successfully in current pod. Providers: {provider_count}" + "Anthropic beta headers config reloaded successfully in current pod. Providers: %s", provider_count ) return { @@ -16150,7 +16198,7 @@ async def reload_anthropic_beta_headers( "timestamp": current_time.isoformat(), } except Exception as e: - verbose_proxy_logger.exception(f"Failed to reload anthropic beta headers: {e}") + verbose_proxy_logger.exception("Failed to reload anthropic beta headers: %s", e) raise HTTPException(status_code=500, detail=f"Failed to reload anthropic beta headers: {e}") @@ -16198,7 +16246,7 @@ async def schedule_anthropic_beta_headers_reload( ) await invalidate_config_param("anthropic_beta_headers_reload_config") - verbose_proxy_logger.info(f"Anthropic beta headers reload scheduled for every {hours} hours") + verbose_proxy_logger.info("Anthropic beta headers reload scheduled for every %s hours", hours) return { "message": f"Anthropic beta headers reload scheduled for every {hours} hours", @@ -16207,7 +16255,7 @@ async def schedule_anthropic_beta_headers_reload( "timestamp": datetime.utcnow().isoformat(), } except Exception as e: - verbose_proxy_logger.exception(f"Failed to schedule anthropic beta headers reload: {e}") + verbose_proxy_logger.exception("Failed to schedule anthropic beta headers reload: %s", e) raise HTTPException( status_code=500, detail=f"Failed to schedule anthropic beta headers reload: {e}", @@ -16252,7 +16300,7 @@ async def cancel_anthropic_beta_headers_reload( "timestamp": datetime.utcnow().isoformat(), } except Exception as e: - verbose_proxy_logger.exception(f"Failed to cancel anthropic beta headers reload: {e}") + verbose_proxy_logger.exception("Failed to cancel anthropic beta headers reload: %s", e) raise HTTPException( status_code=500, detail=f"Failed to cancel anthropic beta headers reload: {e}", @@ -16284,7 +16332,7 @@ async def get_anthropic_beta_headers_reload_status( global prisma_client, last_anthropic_beta_headers_reload verbose_proxy_logger.info( - f"Checking anthropic beta headers reload status. Last reload: {last_anthropic_beta_headers_reload}" + "Checking anthropic beta headers reload status. Last reload: %s", last_anthropic_beta_headers_reload ) if prisma_client is None: @@ -16335,7 +16383,7 @@ async def get_anthropic_beta_headers_reload_status( if hours_since_last_reload < interval_hours: next_run = (last_reload_time + timedelta(hours=interval_hours)).isoformat() except Exception as e: - verbose_proxy_logger.warning(f"Error parsing last reload time: {e}") + verbose_proxy_logger.warning("Error parsing last reload time: %s", e) return { "scheduled": True, @@ -16344,7 +16392,7 @@ async def get_anthropic_beta_headers_reload_status( "next_run": next_run, } except Exception as e: - verbose_proxy_logger.exception(f"Failed to get anthropic beta headers reload status: {e}") + verbose_proxy_logger.exception("Failed to get anthropic beta headers reload status: %s", e) raise HTTPException( status_code=500, detail=f"Failed to get anthropic beta headers reload status: {e}", diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index a4ef30fb3c4..129ff8df3bd 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -191,7 +191,7 @@ async def _save_vector_store_to_db_from_rag_ingest( elif hasattr(response, "vector_store_id"): vector_store_id = response.vector_store_id else: - verbose_proxy_logger.warning(f"Unable to extract vector_store_id from response type: {type(response)}") + verbose_proxy_logger.warning("Unable to extract vector_store_id from response type: %s", type(response)) return if vector_store_id is None or not isinstance(vector_store_id, str): @@ -229,7 +229,7 @@ async def _save_vector_store_to_db_from_rag_ingest( # Only create if it doesn't exist if existing_vector_store is None: - verbose_proxy_logger.info(f"Saving newly created vector store {vector_store_id} to database") + verbose_proxy_logger.info("Saving newly created vector store %s to database", vector_store_id) # Initialize metadata with first file initial_metadata = {"ingested_files": [file_entry]} @@ -250,9 +250,9 @@ async def _save_vector_store_to_db_from_rag_ingest( user_id=user_api_key_dict.user_id, ) - verbose_proxy_logger.info(f"Vector store {vector_store_id} saved to database successfully") + verbose_proxy_logger.info("Vector store %s saved to database successfully", vector_store_id) else: - verbose_proxy_logger.info(f"Vector store {vector_store_id} already exists, appending file to metadata") + verbose_proxy_logger.info("Vector store %s already exists, appending file to metadata", vector_store_id) # Update existing vector store with new file existing_metadata = existing_vector_store.vector_store_metadata or {} @@ -274,11 +274,13 @@ async def _save_vector_store_to_db_from_rag_ingest( ) verbose_proxy_logger.info( - f"Added file {file_entry.get('filename') or file_entry.get('file_url', 'Unknown')} to vector store {vector_store_id} metadata" + "Added file %s to vector store %s metadata", + file_entry.get("filename") or file_entry.get("file_url", "Unknown"), + vector_store_id, ) except Exception as db_error: # Log the error but don't fail the request since ingestion succeeded - verbose_proxy_logger.exception(f"Failed to save vector store {vector_store_id} to database: {db_error}") + verbose_proxy_logger.exception("Failed to save vector store %s to database: %s", vector_store_id, db_error) async def parse_rag_ingest_request( @@ -495,7 +497,7 @@ async def rag_ingest( proxy_config=proxy_config, ) - verbose_proxy_logger.debug(f"RAG Ingest - options: {ingest_options}") + verbose_proxy_logger.debug("RAG Ingest - options: %s", ingest_options) # Call ingest response = await litellm.aingest( @@ -509,7 +511,10 @@ async def rag_ingest( # Save vector store to database if it was newly created and prisma_client is available verbose_proxy_logger.debug( - f"RAG Ingest - Checking database save conditions: prisma_client={prisma_client is not None}, response={response is not None}, response_type={type(response)}" + "RAG Ingest - Checking database save conditions: prisma_client=%s, response=%s, response_type=%s", + prisma_client is not None, + response is not None, + type(response), ) if prisma_client is not None and response is not None: @@ -523,7 +528,7 @@ async def rag_ingest( ) else: verbose_proxy_logger.warning( - f"Skipping database save: prisma_client={prisma_client is not None}, response={response is not None}" + "Skipping database save: prisma_client=%s, response=%s", prisma_client is not None, response is not None ) return response @@ -531,7 +536,7 @@ async def rag_ingest( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"RAG Ingest failed: {e}") + verbose_proxy_logger.exception("RAG Ingest failed: %s", e) raise HTTPException( status_code=500, detail={"error": str(e)}, @@ -663,7 +668,7 @@ async def rag_query( proxy_config=proxy_config, ) - verbose_proxy_logger.debug(f"RAG Query - model: {model}, retrieval_config: {retrieval_config}") + verbose_proxy_logger.debug("RAG Query - model: %s, retrieval_config: %s", model, retrieval_config) # Call query response = await litellm.aquery( @@ -706,7 +711,7 @@ async def rag_query( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"RAG Query failed: {e}") + verbose_proxy_logger.exception("RAG Query failed: %s", e) raise HTTPException( status_code=500, detail={"error": str(e)}, diff --git a/litellm/proxy/rerank_endpoints/endpoints.py b/litellm/proxy/rerank_endpoints/endpoints.py index f1c138fa1bf..e91b3b18f01 100644 --- a/litellm/proxy/rerank_endpoints/endpoints.py +++ b/litellm/proxy/rerank_endpoints/endpoints.py @@ -103,7 +103,7 @@ async def rerank( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error(f"litellm.proxy.proxy_server.rerank(): Exception occured - {e}") + verbose_proxy_logger.error("litellm.proxy.proxy_server.rerank(): Exception occured - %s", e) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "message", str(e)), diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 9fa634dc12e..f03db48b4a7 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -116,7 +116,7 @@ async def responses_api( ResponsePollingHandler, ) - verbose_proxy_logger.info(f"Starting background response with polling for model={data.get('model')}") + verbose_proxy_logger.info("Starting background response with polling for model=%s", data.get("model")) # Run pre-call checks (rate limits, guardrails, budget) BEFORE creating # polling ID. This ensures rate-limited requests get a synchronous 429 @@ -233,7 +233,8 @@ async def responses_api( if not model_id: verbose_proxy_logger.warning( - f"No model_id found in response hidden params for response {response.id}, skipping managed object storage" + "No model_id found in response hidden params for response %s, skipping managed object storage", + response.id, ) raise Exception("No model_id found in response hidden params") # Store in managed objects table @@ -247,10 +248,14 @@ async def responses_api( ) verbose_proxy_logger.info( - f"Stored background response {response.id} in managed objects table with unified_id={response.id}" + "Stored background response %s in managed objects table with unified_id=%s", + response.id, + response.id, ) except Exception as e: - verbose_proxy_logger.error(f"Failed to store background response in managed objects table: {e}") + verbose_proxy_logger.error( + "Failed to store background response in managed objects table: %s", e + ) return response except ModifyResponseException as e: diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index b744396e850..7d81390866a 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -51,7 +51,7 @@ async def background_streaming_task( """ try: - verbose_proxy_logger.info(f"Starting background streaming for {polling_id}") + verbose_proxy_logger.info("Starting background streaming for %s", polling_id) # Update status to in_progress (OpenAI format) await polling_handler.update_state( @@ -146,8 +146,8 @@ async def background_streaming_task( # Handle StreamingResponse if not hasattr(response, "body_iterator"): verbose_proxy_logger.warning( - f"background_streaming_task: response for {polling_id} has no " - "body_iterator; this may indicate a misconfiguration or provider error" + "background_streaming_task: response for %s has no body_iterator; this may indicate a misconfiguration or provider error", + polling_id, ) if hasattr(response, "body_iterator"): @@ -293,7 +293,7 @@ async def background_streaming_task( await flush_state_if_needed() except json.JSONDecodeError as e: - verbose_proxy_logger.warning(f"Failed to parse streaming chunk: {e}") + verbose_proxy_logger.warning("Failed to parse streaming chunk: %s", e) # Final flush to ensure all accumulated state is saved await flush_state_if_needed(force=True) @@ -324,11 +324,16 @@ async def background_streaming_task( ) verbose_proxy_logger.info( - f"Finished background streaming for {polling_id}, status={final_status}, error={terminal_error}, incomplete_details={incomplete_details_data}, output_items={len(output_items)}" + "Finished background streaming for %s, status=%s, error=%s, incomplete_details=%s, output_items=%s", + polling_id, + final_status, + terminal_error, + incomplete_details_data, + len(output_items), ) except Exception as e: - verbose_proxy_logger.error(f"Error in background streaming task for {polling_id}: {e}") + verbose_proxy_logger.error("Error in background streaming task for %s: %s", polling_id, e) import traceback verbose_proxy_logger.error(traceback.format_exc()) diff --git a/litellm/proxy/response_polling/polling_handler.py b/litellm/proxy/response_polling/polling_handler.py index 4f2ad70cc7d..c39e5949f19 100644 --- a/litellm/proxy/response_polling/polling_handler.py +++ b/litellm/proxy/response_polling/polling_handler.py @@ -77,7 +77,7 @@ class ResponsePollingHandler: value=response.model_dump_json(), # Pydantic v2 method ttl=self.ttl, ) - verbose_proxy_logger.debug(f"Created initial polling state for {polling_id} with TTL={self.ttl}s") + verbose_proxy_logger.debug("Created initial polling state for %s with TTL=%ss", polling_id, self.ttl) return response @@ -141,7 +141,7 @@ class ResponsePollingHandler: # Get current state cached_state = await self.redis_cache.async_get_cache(cache_key) if not cached_state: - verbose_proxy_logger.warning(f"No cached state found for polling_id: {polling_id}") + verbose_proxy_logger.warning("No cached state found for polling_id: %s", polling_id) return # Parse existing ResponsesAPIResponse from cache @@ -209,7 +209,7 @@ class ResponsePollingHandler: output_count = len(state.get("output", [])) verbose_proxy_logger.debug( - f"Updated polling state for {polling_id}: status={state['status']}, output_items={output_count}" + "Updated polling state for %s: status=%s, output_items=%s", polling_id, state["status"], output_count ) async def get_state(self, polling_id: str) -> dict[str, Any] | None: @@ -277,7 +277,7 @@ def should_use_polling_for_request( # Check if model is in native_background_mode list - these use native provider background mode if native_background_mode and model in native_background_mode: - verbose_proxy_logger.debug(f"Model {model} is in native_background_mode list, skipping polling via cache") + verbose_proxy_logger.debug("Model %s is in native_background_mode list, skipping polling via cache", model) return False # "all" enables polling for all providers @@ -311,9 +311,9 @@ def should_use_polling_for_request( # If ANY deployment's provider matches, enable polling if dep_provider and dep_provider in polling_via_cache_enabled: - verbose_proxy_logger.debug(f"Polling enabled for model={model}, provider={dep_provider}") + verbose_proxy_logger.debug("Polling enabled for model=%s, provider=%s", model, dep_provider) return True except Exception as e: - verbose_proxy_logger.debug(f"Could not resolve provider for model {model}: {e}") + verbose_proxy_logger.debug("Could not resolve provider for model %s: %s", model, e) return False diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 5e775d1cbf7..97832b0b6c4 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -262,7 +262,7 @@ async def add_shared_session_to_data(data: dict) -> None: if session is not None and not session.closed: data["shared_session"] = session - verbose_proxy_logger.info(f"SESSION REUSE: Attached shared aiohttp session to request (ID: {id(session)})") + verbose_proxy_logger.info("SESSION REUSE: Attached shared aiohttp session to request (ID: %s)", id(session)) elif session is not None and session.closed: # Session was created at startup but has since closed — recreate it # Use lock to prevent concurrent recreation (avoids session/connector leak) @@ -278,7 +278,7 @@ async def add_shared_session_to_data(data: dict) -> None: # or closed — either way we need to recreate if session is not None: verbose_proxy_logger.warning( - f"SESSION REUSE: Shared aiohttp session is closed (ID: {id(session)}), recreating..." + "SESSION REUSE: Shared aiohttp session is closed (ID: %s), recreating...", id(session) ) else: verbose_proxy_logger.warning( diff --git a/litellm/proxy/search_endpoints/endpoints.py b/litellm/proxy/search_endpoints/endpoints.py index 0032083b09c..09814672ccf 100644 --- a/litellm/proxy/search_endpoints/endpoints.py +++ b/litellm/proxy/search_endpoints/endpoints.py @@ -170,14 +170,15 @@ async def search( team_object=team_object, ) except Exception as e: - verbose_proxy_logger.error(f"Search tool authorization failed for {search_tool_name_value}: {e}") + verbose_proxy_logger.error("Search tool authorization failed for %s: %s", search_tool_name_value, e) raise if llm_router is not None and hasattr(llm_router, "search_tools"): verbose_proxy_logger.debug( - f"Search endpoint - Looking for search_tool_name: {search_tool_name_value}. " - f"Available search tools in router: {[tool.get('search_tool_name') for tool in llm_router.search_tools]}. " - f"Total search tools: {len(llm_router.search_tools)}" + "Search endpoint - Looking for search_tool_name: %s. Available search tools in router: %s. Total search tools: %s", + search_tool_name_value, + [tool.get("search_tool_name") for tool in llm_router.search_tools], + len(llm_router.search_tools), ) matching_tools = [ @@ -302,5 +303,5 @@ async def list_search_tools( except Exception as e: from litellm._logging import verbose_proxy_logger - verbose_proxy_logger.exception(f"Error listing search tools: {e}") + verbose_proxy_logger.exception("Error listing search tools: %s", e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/litellm/proxy/search_endpoints/search_tool_management.py b/litellm/proxy/search_endpoints/search_tool_management.py index 6b0bbacd131..1b3462b48a4 100644 --- a/litellm/proxy/search_endpoints/search_tool_management.py +++ b/litellm/proxy/search_endpoints/search_tool_management.py @@ -161,7 +161,7 @@ async def list_search_tools( if parsed_tools: config_search_tools = parsed_tools except Exception as e: - verbose_proxy_logger.debug(f"Could not get config-defined search tools: {e}") + verbose_proxy_logger.debug("Could not get config-defined search tools: %s", e) for config_search_tool in config_search_tools: tool_name = config_search_tool.get("search_tool_name") @@ -214,7 +214,7 @@ async def list_search_tools( return ListSearchToolsResponse(search_tools=visible_search_tools) except Exception as e: - verbose_proxy_logger.exception(f"Error getting search tools: {e}") + verbose_proxy_logger.exception("Error getting search tools: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -278,13 +278,13 @@ async def create_search_tool(request: CreateSearchToolRequest): ) verbose_proxy_logger.debug( - f"Successfully added search tool '{result.get('search_tool_name')}' to database. " - f"Router will be updated by the cron job." + "Successfully added search tool '%s' to database. Router will be updated by the cron job.", + result.get("search_tool_name"), ) return result except Exception as e: - verbose_proxy_logger.exception(f"Error adding search tool to db: {e}") + verbose_proxy_logger.exception("Error adding search tool to db: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -361,15 +361,15 @@ async def update_search_tool(search_tool_id: str, request: UpdateSearchToolReque ) verbose_proxy_logger.debug( - f"Successfully updated search tool '{result.get('search_tool_name')}' in database. " - f"Router will be updated by the cron job." + "Successfully updated search tool '%s' in database. Router will be updated by the cron job.", + result.get("search_tool_name"), ) return result except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.exception(f"Error updating search tool: {e}") + verbose_proxy_logger.exception("Error updating search tool: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -425,7 +425,7 @@ async def delete_search_tool(search_tool_id: str): except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.exception(f"Error deleting search tool: {e}") + verbose_proxy_logger.exception("Error deleting search tool: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -498,7 +498,7 @@ async def get_search_tool_info(search_tool_id: str): except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.exception(f"Error getting search tool info: {e}") + verbose_proxy_logger.exception("Error getting search tool info: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -561,7 +561,7 @@ async def test_search_tool_connection(request: TestSearchToolConnectionRequest): if not search_provider: raise HTTPException(status_code=400, detail="search_provider is required in litellm_params") - verbose_proxy_logger.debug(f"Testing connection to search provider: {search_provider}") + verbose_proxy_logger.debug("Testing connection to search provider: %s", search_provider) # Make a simple test search query with max_results=1 to minimize cost test_query = "test" @@ -574,7 +574,7 @@ async def test_search_tool_connection(request: TestSearchToolConnectionRequest): timeout=10.0, # 10 second timeout for test ) - verbose_proxy_logger.debug(f"Successfully tested connection to {search_provider} search provider") + verbose_proxy_logger.debug("Successfully tested connection to %s search provider", search_provider) return { "status": "success", @@ -587,7 +587,7 @@ async def test_search_tool_connection(request: TestSearchToolConnectionRequest): error_message = str(e) error_type = type(e).__name__ - verbose_proxy_logger.exception(f"Failed to connect to search provider: {error_message}") + verbose_proxy_logger.exception("Failed to connect to search provider: %s", error_message) # Return error details in a structured format return { @@ -652,10 +652,10 @@ async def get_available_search_providers(): } ) except Exception as e: - verbose_proxy_logger.debug(f"Could not get config for search provider {provider.value}: {e}") + verbose_proxy_logger.debug("Could not get config for search provider %s: %s", provider.value, e) continue return {"providers": available_providers} except Exception as e: - verbose_proxy_logger.exception(f"Error getting available search providers: {e}") + verbose_proxy_logger.exception("Error getting available search providers: %s", e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/litellm/proxy/search_endpoints/search_tool_registry.py b/litellm/proxy/search_endpoints/search_tool_registry.py index d7e5efa6d1e..ad8c8b6fe42 100644 --- a/litellm/proxy/search_endpoints/search_tool_registry.py +++ b/litellm/proxy/search_endpoints/search_tool_registry.py @@ -78,7 +78,7 @@ class SearchToolRegistry: return search_tool_dict except Exception as e: - verbose_proxy_logger.exception(f"Error adding search tool to DB: {e}") + verbose_proxy_logger.exception("Error adding search tool to DB: %s", e) raise Exception(f"Error adding search tool to DB: {e}") async def delete_search_tool_from_db(self, search_tool_id: str, prisma_client: PrismaClient): @@ -109,7 +109,7 @@ class SearchToolRegistry: "search_tool_name": existing_tool.search_tool_name, } except Exception as e: - verbose_proxy_logger.exception(f"Error deleting search tool from DB: {e}") + verbose_proxy_logger.exception("Error deleting search tool from DB: %s", e) raise Exception(f"Error deleting search tool from DB: {e}") async def update_search_tool_in_db(self, search_tool_id: str, search_tool: SearchTool, prisma_client: PrismaClient): @@ -143,7 +143,7 @@ class SearchToolRegistry: # Convert to dict with ISO formatted datetimes return self._convert_prisma_to_dict(updated_search_tool) except Exception as e: - verbose_proxy_logger.exception(f"Error updating search tool in DB: {e}") + verbose_proxy_logger.exception("Error updating search tool in DB: %s", e) raise Exception(f"Error updating search tool in DB: {e}") @staticmethod @@ -176,7 +176,7 @@ class SearchToolRegistry: return search_tools except Exception as e: - verbose_proxy_logger.exception(f"Error getting search tools from DB: {e}") + verbose_proxy_logger.exception("Error getting search tools from DB: %s", e) raise Exception(f"Error getting search tools from DB: {e}") async def get_search_tool_by_id_from_db( @@ -204,7 +204,7 @@ class SearchToolRegistry: search_tool_dict = self._convert_prisma_to_dict(search_tool) return SearchTool(**search_tool_dict) # type: ignore except Exception as e: - verbose_proxy_logger.exception(f"Error getting search tool from DB: {e}") + verbose_proxy_logger.exception("Error getting search tool from DB: %s", e) raise Exception(f"Error getting search tool from DB: {e}") async def get_search_tool_by_name_from_db( @@ -232,5 +232,5 @@ class SearchToolRegistry: search_tool_dict = self._convert_prisma_to_dict(search_tool) return SearchTool(**search_tool_dict) # type: ignore except Exception as e: - verbose_proxy_logger.exception(f"Error getting search tool from DB: {e}") + verbose_proxy_logger.exception("Error getting search tool from DB: %s", e) raise Exception(f"Error getting search tool from DB: {e}") diff --git a/litellm/proxy/spend_tracking/cloudzero_endpoints.py b/litellm/proxy/spend_tracking/cloudzero_endpoints.py index 7b573b2fad7..fdeb176aa99 100644 --- a/litellm/proxy/spend_tracking/cloudzero_endpoints.py +++ b/litellm/proxy/spend_tracking/cloudzero_endpoints.py @@ -161,7 +161,7 @@ async def get_cloudzero_settings( # Re-raise HTTPExceptions as-is raise e except Exception as e: - verbose_proxy_logger.error(f"Error retrieving CloudZero settings: {e}") + verbose_proxy_logger.error("Error retrieving CloudZero settings: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to retrieve CloudZero settings: {e}"}, @@ -238,7 +238,7 @@ async def update_cloudzero_settings( ) raise e except Exception as e: - verbose_proxy_logger.error(f"Error updating CloudZero settings: {e}") + verbose_proxy_logger.error("Error updating CloudZero settings: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to update CloudZero settings: {e}"}, @@ -275,7 +275,7 @@ async def is_cloudzero_setup_in_db() -> bool: return cloudzero_config is not None and cloudzero_config.param_value is not None except Exception as e: - verbose_proxy_logger.error(f"Error checking CloudZero status: {e}") + verbose_proxy_logger.error("Error checking CloudZero status: %s", e) return False @@ -317,7 +317,7 @@ async def is_cloudzero_setup() -> bool: return False except Exception as e: - verbose_proxy_logger.error(f"Error checking CloudZero setup: {e}") + verbose_proxy_logger.error("Error checking CloudZero setup: %s", e) return False @@ -364,7 +364,7 @@ async def init_cloudzero_settings( return CloudZeroInitResponse(message="CloudZero settings initialized successfully", status="success") except Exception as e: - verbose_proxy_logger.error(f"Error initializing CloudZero settings: {e}") + verbose_proxy_logger.error("Error initializing CloudZero settings: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to initialize CloudZero settings: {e}"}, @@ -422,7 +422,7 @@ async def cloudzero_dry_run_export( ) except Exception as e: - verbose_proxy_logger.error(f"Error performing CloudZero dry run export: {e}") + verbose_proxy_logger.error("Error performing CloudZero dry run export: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to perform CloudZero dry run export: {e}"}, @@ -487,7 +487,7 @@ async def cloudzero_export( ) except Exception as e: - verbose_proxy_logger.error(f"Error performing CloudZero export: {e}") + verbose_proxy_logger.error("Error performing CloudZero export: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to perform CloudZero export: {e}"}, @@ -550,7 +550,7 @@ async def delete_cloudzero_settings( except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.error(f"Error deleting CloudZero settings: {e}") + verbose_proxy_logger.error("Error deleting CloudZero settings: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to delete CloudZero settings: {e}"}, diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 0bcc2b9994b..aa26a45ae0e 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1648,7 +1648,7 @@ async def _get_spend_report_for_time_range( return response, spend_per_tag except Exception as e: - verbose_proxy_logger.error(f"Exception in _get_daily_spend_reports {e}") + verbose_proxy_logger.error("Exception in _get_daily_spend_reports %s", e) @router.post( @@ -2261,7 +2261,7 @@ async def ui_view_spend_logs( total_is_capped=total_is_capped, ) except Exception as e: - verbose_proxy_logger.exception(f"Error in ui_view_spend_logs: {e}") + verbose_proxy_logger.exception("Error in ui_view_spend_logs: %s", e) raise handle_exception_on_proxy(e) @@ -2789,7 +2789,7 @@ async def global_spend_refresh(): } except Exception as e: - verbose_proxy_logger.exception(f"Failed to refresh materialized view - {e}") + verbose_proxy_logger.exception("Failed to refresh materialized view - %s", e) return { "message": "Failed to refresh materialized view", "status": "failure", @@ -2830,7 +2830,7 @@ async def global_spend_for_internal_user( return response except Exception as e: - verbose_proxy_logger.error(f"/global/spend/logs Error: {e}") + verbose_proxy_logger.error("/global/spend/logs Error: %s", e) raise e @@ -2906,7 +2906,7 @@ async def global_spend_logs( except Exception as e: error_trace = traceback.format_exc() error_str = str(e) + "\n" + error_trace - verbose_proxy_logger.error(f"/global/spend/logs Error: {error_str}") + verbose_proxy_logger.error("/global/spend/logs Error: %s", error_str) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"/global/spend/logs Error({error_str})"), @@ -3387,7 +3387,7 @@ async def provider_budgets() -> ProviderBudgetResponse: provider_budget_response_dict[_provider] = provider_budget_response_object return ProviderBudgetResponse(providers=provider_budget_response_dict) except Exception as e: - verbose_proxy_logger.exception(f"/provider/budgets: Exception occured - {e}") + verbose_proxy_logger.exception("/provider/budgets: Exception occured - %s", e) raise handle_exception_on_proxy(e) diff --git a/litellm/proxy/spend_tracking/vantage_endpoints.py b/litellm/proxy/spend_tracking/vantage_endpoints.py index ac45594de22..d965a150ac1 100644 --- a/litellm/proxy/spend_tracking/vantage_endpoints.py +++ b/litellm/proxy/spend_tracking/vantage_endpoints.py @@ -166,7 +166,7 @@ async def get_vantage_settings( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error retrieving Vantage settings: {e}") + verbose_proxy_logger.error("Error retrieving Vantage settings: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to retrieve Vantage settings: {e}"}, @@ -235,7 +235,7 @@ async def update_vantage_settings( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error updating Vantage settings: {e}") + verbose_proxy_logger.error("Error updating Vantage settings: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to update Vantage settings: {e}"}, @@ -257,7 +257,7 @@ async def is_vantage_setup_in_db() -> bool: return vantage_config is not None and vantage_config.param_value is not None except Exception as e: - verbose_proxy_logger.error(f"Error checking Vantage status: {e}") + verbose_proxy_logger.error("Error checking Vantage status: %s", e) return False @@ -280,7 +280,7 @@ async def is_vantage_setup() -> bool: return True return False except Exception as e: - verbose_proxy_logger.error(f"Error checking Vantage setup: {e}") + verbose_proxy_logger.error("Error checking Vantage setup: %s", e) return False @@ -324,7 +324,7 @@ async def init_vantage_settings( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error initializing Vantage settings: {e}") + verbose_proxy_logger.error("Error initializing Vantage settings: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to initialize Vantage settings: {e}"}, @@ -415,7 +415,7 @@ async def vantage_dry_run_export( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error performing Vantage dry run export: {e}") + verbose_proxy_logger.error("Error performing Vantage dry run export: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to perform Vantage dry run export: {e}"}, @@ -488,7 +488,7 @@ async def vantage_export( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error performing Vantage export: {e}") + verbose_proxy_logger.error("Error performing Vantage export: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to perform Vantage export: {e}"}, @@ -548,7 +548,7 @@ async def delete_vantage_settings( except HTTPException: raise except Exception as e: - verbose_proxy_logger.error(f"Error deleting Vantage settings: {e}") + verbose_proxy_logger.error("Error deleting Vantage settings: %s", e) raise HTTPException( status_code=500, detail={"error": f"Failed to delete Vantage settings: {e}"}, diff --git a/litellm/proxy/types_utils/utils.py b/litellm/proxy/types_utils/utils.py index e61fcdd859b..ae2bdbc5895 100644 --- a/litellm/proxy/types_utils/utils.py +++ b/litellm/proxy/types_utils/utils.py @@ -127,8 +127,11 @@ def _load_instance_from_remote_storage(remote_url: str, config_file_path: str | object_key = f"{module_path}.py" verbose_proxy_logger.debug( - f"Loading custom logger from {storage_type}: bucket={bucket_name}, " - f"object_key={object_key}, instance={instance_name}" + "Loading custom logger from %s: bucket=%s, object_key=%s, instance=%s", + storage_type, + bucket_name, + object_key, + instance_name, ) import tempfile @@ -170,9 +173,9 @@ def _load_instance_from_remote_storage(remote_url: str, config_file_path: str | try: os.remove(local_file_path) except Exception as cleanup_error: - verbose_proxy_logger.warning(f"Could not clean up temporary file {local_file_path}: {cleanup_error}") + verbose_proxy_logger.warning("Could not clean up temporary file %s: %s", local_file_path, cleanup_error) - verbose_proxy_logger.info(f"Successfully loaded custom logger from {remote_url}") + verbose_proxy_logger.info("Successfully loaded custom logger from %s", remote_url) return instance except Exception as e: @@ -190,7 +193,7 @@ async def _download_gcs_file_wrapper(bucket_name: str, object_key: str, local_fi except Exception as e: from litellm._logging import verbose_proxy_logger - verbose_proxy_logger.error(f"Error downloading from GCS: {e}") + verbose_proxy_logger.error("Error downloading from GCS: %s", e) return False diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 8ed848ac1bf..22759a37ae7 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -689,7 +689,10 @@ async def update_default_team_member_budget(teams: list[NewUserRequestTeam], use ) except Exception as e: verbose_proxy_logger.info( - f"Error updating team {team_id} with team member budget {max_budget_in_team} with error: {e}, skipping.." + "Error updating team %s with team member budget %s with error: %s, skipping..", + team_id, + max_budget_in_team, + e, ) continue @@ -1209,7 +1212,7 @@ async def update_mcp_semantic_filter_settings( if prisma_client is not None: await proxy_config._init_semantic_filter_settings_in_db(prisma_client=prisma_client) except Exception as e: - verbose_proxy_logger.warning(f"Failed to reinitialize MCP semantic filter settings immediately: {e}") + verbose_proxy_logger.warning("Failed to reinitialize MCP semantic filter settings immediately: %s", e) return result diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 5f18189b6b3..51c9b300b6e 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -196,7 +196,7 @@ def print_verbose(print_statement): """ import traceback - verbose_proxy_logger.debug(f"{print_statement}\n{traceback.format_exc()}") + verbose_proxy_logger.debug("%s\n%s", print_statement, traceback.format_exc()) if litellm.set_verbose: print(f"LiteLLM Proxy: {_redact_string(str(print_statement))}") # noqa: T201 @@ -722,11 +722,11 @@ class ProxyLogging: else: # Could not parse modified arguments, allow original call but warn verbose_proxy_logger.warning( - f"Could not parse modified arguments from guardrail response: {new_content}" + "Could not parse modified arguments from guardrail response: %s", new_content ) return None except Exception as e: - verbose_proxy_logger.error(f"Error parsing modified arguments: {e}") + verbose_proxy_logger.error("Error parsing modified arguments: %s", e) # Fallback: allow original call return None @@ -742,7 +742,7 @@ class ProxyLogging: """ import json - verbose_proxy_logger.debug(f"Extracting modified args from content: {masked_content}") + verbose_proxy_logger.debug("Extracting modified args from content: %s", masked_content) try: # The format should be: "Tool: \nArguments: " @@ -753,16 +753,16 @@ class ProxyLogging: # Get the arguments part - everything after "Arguments: " args_text = line[len("Arguments:") :].strip() - verbose_proxy_logger.debug(f"Found arguments text: {args_text}") + verbose_proxy_logger.debug("Found arguments text: %s", args_text) # Try to parse as JSON first try: modified_args = json.loads(args_text) - verbose_proxy_logger.debug(f"Successfully parsed JSON args: {modified_args}") + verbose_proxy_logger.debug("Successfully parsed JSON args: %s", modified_args) return modified_args except json.JSONDecodeError as e: # If JSON parsing fails, try to extract key-value pairs manually - verbose_proxy_logger.debug(f"Failed to parse JSON arguments: {args_text}, error: {e}") + verbose_proxy_logger.debug("Failed to parse JSON arguments: %s, error: %s", args_text, e) return self._parse_arguments_manually(args_text, request_obj.arguments) # If we can't find the Arguments: line, return None @@ -770,7 +770,7 @@ class ProxyLogging: return None except Exception as e: - verbose_proxy_logger.error(f"Error extracting modified arguments: {e}") + verbose_proxy_logger.error("Error extracting modified arguments: %s", e) return None def _parse_arguments_manually(self, args_text: str, original_args: dict) -> dict | None: @@ -799,7 +799,7 @@ class ProxyLogging: return modified_args except Exception as e: - verbose_proxy_logger.error(f"Error in manual argument parsing: {e}") + verbose_proxy_logger.error("Error in manual argument parsing: %s", e) return None def _convert_llm_result_to_mcp_during_response(self, llm_result, request_obj) -> Any | None: @@ -2174,10 +2174,10 @@ class ProxyLogging: except Exception as e: # Log non-HTTPException errors from callbacks but don't break the flow verbose_proxy_logger.exception( - f"[Non-Blocking] Error in async_post_call_failure_hook callback: {e}" + "[Non-Blocking] Error in async_post_call_failure_hook callback: %s", e ) except Exception as e: - verbose_proxy_logger.exception(f"[Non-Blocking] Error setting up post_call_failure_hook callback: {e}") + verbose_proxy_logger.exception("[Non-Blocking] Error setting up post_call_failure_hook callback: %s", e) return transformed_exception @@ -3019,7 +3019,7 @@ class PrismaClient: try: from prisma import Prisma # type: ignore except Exception as e: - verbose_proxy_logger.error(f"Failed to import Prisma client: {e}") + verbose_proxy_logger.error("Failed to import Prisma client: %s", e) verbose_proxy_logger.error("This usually means 'prisma generate' hasn't been run yet.") verbose_proxy_logger.error("Please run 'prisma generate' to generate the Prisma client.") raise Exception("Unable to find Prisma binaries. Please run 'prisma generate' first.") @@ -3283,7 +3283,8 @@ class PrismaClient: missing_views = expected_views_set - ret_view_names_set verbose_proxy_logger.warning( - f"\n\n\033[93mNot all views exist in db, needed for UI 'Usage' tab. Missing={missing_views}.\nRun 'create_views.py' from https://github.com/BerriAI/litellm/tree/main/db_scripts to create missing views.\033[0m\n" + "\n\n\x1b[93mNot all views exist in db, needed for UI 'Usage' tab. Missing=%s.\nRun 'create_views.py' from https://github.com/BerriAI/litellm/tree/main/db_scripts to create missing views.\x1b[0m\n", + missing_views, ) except Exception: @@ -3444,7 +3445,7 @@ class PrismaClient: if token is not None: if isinstance(token, str): hashed_token = _hash_token_if_needed(token=token) - verbose_proxy_logger.debug(f"PrismaClient: find_unique for token: {hashed_token}") + verbose_proxy_logger.debug("PrismaClient: find_unique for token: %s", hashed_token) if query_type == "find_unique" and hashed_token is not None: if token is None: raise HTTPException( @@ -3687,7 +3688,7 @@ class PrismaClient: if token is not None: if isinstance(token, str): hashed_token = _hash_token_if_needed(token=token) - verbose_proxy_logger.debug(f"PrismaClient: find_unique for token: {hashed_token}") + verbose_proxy_logger.debug("PrismaClient: find_unique for token: %s", hashed_token) if query_type == "find_unique": if token is None: raise HTTPException( @@ -3994,7 +3995,7 @@ class PrismaClient: """ Update existing data """ - verbose_proxy_logger.debug(f"PrismaClient: update_data, table_name: {table_name}") + verbose_proxy_logger.debug("PrismaClient: update_data, table_name: %s", table_name) start_time = time.time() try: db_data = self.jsonify_object(data=data) @@ -5062,7 +5063,7 @@ class PrismaClient: try: return await _fetch_row_count() except Exception as e: - verbose_proxy_logger.error(f"Error getting LiteLLM_SpendLogs row count: {e}") + verbose_proxy_logger.error("Error getting LiteLLM_SpendLogs row count: %s", e) return 0 @backoff.on_exception( @@ -5095,7 +5096,7 @@ class PrismaClient: value = float(response_time_ms) return value if value == value and value not in (float("inf"), float("-inf")) else None except (ValueError, TypeError): - verbose_proxy_logger.warning(f"Invalid response_time_ms value: {response_time_ms}") + verbose_proxy_logger.warning("Invalid response_time_ms value: %s", response_time_ms) return None def _clean_details(self, details: dict | None) -> dict | None: @@ -5105,7 +5106,7 @@ class PrismaClient: try: return safe_json_loads(safe_dumps(details)) except Exception as e: - verbose_proxy_logger.warning(f"Failed to clean details JSON: {e}") + verbose_proxy_logger.warning("Failed to clean details JSON: %s", e) return None async def save_health_check_result( @@ -5142,11 +5143,11 @@ class PrismaClient: # Add only non-None optional fields health_check_data.update({k: v for k, v in optional_fields.items() if v is not None}) - verbose_proxy_logger.debug(f"Saving health check data: {health_check_data}") + verbose_proxy_logger.debug("Saving health check data: %s", health_check_data) return await HealthCheckRepository(self).table.create(data=health_check_data) except Exception as e: - verbose_proxy_logger.error(f"Error saving health check result for model {model_name}: {e}") + verbose_proxy_logger.error("Error saving health check result for model %s: %s", model_name, e) return None async def get_health_check_history( @@ -5174,7 +5175,7 @@ class PrismaClient: ) return results except Exception as e: - verbose_proxy_logger.error(f"Error getting health check history: {e}") + verbose_proxy_logger.error("Error getting health check history: %s", e) return [] async def get_all_latest_health_checks(self): @@ -5194,7 +5195,7 @@ class PrismaClient: ], ) except Exception as e: - verbose_proxy_logger.error(f"Error getting all latest health checks: {e}") + verbose_proxy_logger.error("Error getting all latest health checks: %s", e) return [] @@ -5451,7 +5452,7 @@ class ProxyUpdateSpend: if len(logs_to_process) > 0 and base_url is not None and db_writer_client is not None: if not base_url.endswith("/"): base_url += "/" - verbose_proxy_logger.debug(f"base_url: {base_url}") + verbose_proxy_logger.debug("base_url: %s", base_url) json_data = json.dumps(logs_to_process) response = await db_writer_client.post( url=base_url + "spend/update", @@ -5475,7 +5476,7 @@ class ProxyUpdateSpend: statement_rows, isolation_budget, ) - verbose_proxy_logger.debug(f"Flushed {len(batch)} logs to the DB.") + verbose_proxy_logger.debug("Flushed %s logs to the DB.", len(batch)) # Explicitly clear batch memory del batch, batch_with_dates @@ -5483,7 +5484,7 @@ class ProxyUpdateSpend: async with prisma_client._spend_log_transactions_lock: remaining_count = len(prisma_client.spend_log_transactions) verbose_proxy_logger.debug( - f"{len(logs_to_process)} logs processed. Remaining in queue: {remaining_count}" + "%s logs processed. Remaining in queue: %s", len(logs_to_process), remaining_count ) break except DB_CONNECTION_ERROR_TYPES as e: @@ -5551,7 +5552,7 @@ async def update_spend( # Check queue size with lock protection async with prisma_client._spend_log_transactions_lock: queue_size = len(prisma_client.spend_log_transactions) - verbose_proxy_logger.debug(f"Spend Logs transactions: {queue_size}") + verbose_proxy_logger.debug("Spend Logs transactions: %s", queue_size) async with prisma_client._tool_usage_transactions_lock: tool_usage_queue_size = len(prisma_client.tool_usage_transactions) @@ -5611,7 +5612,7 @@ async def update_daily_tag_spend( # the active exception's traceback whenever the suppression env var # is unset, which would be a regression for operators who never saw # one here before. - verbose_proxy_logger.error(f"Error updating daily tag spend: {e}") + verbose_proxy_logger.error("Error updating daily tag spend: %s", e) async def update_spend_logs_job( @@ -5713,7 +5714,7 @@ async def _monitor_spend_logs_queue( current_interval = base_interval verbose_proxy_logger.info( - f"Starting spend logs queue monitor (threshold: {threshold}, poll_interval: {base_interval}s)" + "Starting spend logs queue monitor (threshold: %s, poll_interval: %ss)", threshold, base_interval ) while True: @@ -5729,13 +5730,17 @@ async def _monitor_spend_logs_queue( if queue_size > 0: if queue_size >= threshold: verbose_proxy_logger.debug( - f"Spend logs queue size ({queue_size}) reached threshold ({threshold}), triggering processing" + "Spend logs queue size (%s) reached threshold (%s), triggering processing", + queue_size, + threshold, ) # Reset to base interval when threshold is reached current_interval = base_interval else: verbose_proxy_logger.debug( - f"Spend logs queue size ({queue_size}) below threshold ({threshold}), processing with backoff" + "Spend logs queue size (%s) below threshold (%s), processing with backoff", + queue_size, + threshold, ) # Exponential backoff when below threshold but still processing current_interval = min(current_interval * backoff_multiplier, max_backoff) @@ -6121,7 +6126,7 @@ def handle_exception_on_proxy(e: Exception) -> ProxyException: """ from fastapi import status - verbose_proxy_logger.exception(f"Exception: {e}") + verbose_proxy_logger.exception("Exception: %s", e) if isinstance(e, HTTPException): return ProxyException( diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index 6176ae03d3d..d6d3392a7b4 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -207,11 +207,11 @@ def _resolve_embedding_config_from_router(embedding_model: str, llm_router) -> d # Only return config if we have at least api_key or api_base if embedding_config: verbose_proxy_logger.debug( - f"Resolved embedding config from router model {model_name}: {list(embedding_config.keys())}" + "Resolved embedding config from router model %s: %s", model_name, list(embedding_config.keys()) ) return embedding_config except Exception as e: - verbose_proxy_logger.debug(f"Error resolving embedding config from router for model {model_name}: {e}") + verbose_proxy_logger.debug("Error resolving embedding config from router for model %s: %s", model_name, e) continue return None @@ -295,11 +295,13 @@ async def _resolve_embedding_config_from_db(embedding_model: str, prisma_client) # Only return config if we have at least api_key or api_base if embedding_config: verbose_proxy_logger.debug( - f"Resolved embedding config from database model {model_name}: {list(embedding_config.keys())}" + "Resolved embedding config from database model %s: %s", + model_name, + list(embedding_config.keys()), ) return embedding_config except Exception as e: - verbose_proxy_logger.debug(f"Error resolving embedding config for model {model_name}: {e}") + verbose_proxy_logger.debug("Error resolving embedding config for model %s: %s", model_name, e) continue return None @@ -344,7 +346,7 @@ async def _resolve_embedding_config(embedding_model: str, prisma_client, llm_rou if llm_router is not None: router_config = _resolve_embedding_config_from_router(embedding_model=embedding_model, llm_router=llm_router) if router_config: - verbose_proxy_logger.debug(f"Resolved embedding config from router for model {embedding_model}") + verbose_proxy_logger.debug("Resolved embedding config from router for model %s", embedding_model) cache.set_cache(embedding_model, router_config) return router_config @@ -354,12 +356,12 @@ async def _resolve_embedding_config(embedding_model: str, prisma_client, llm_rou embedding_model=embedding_model, prisma_client=prisma_client ) if db_config: - verbose_proxy_logger.debug(f"Resolved embedding config from database for model {embedding_model}") + verbose_proxy_logger.debug("Resolved embedding config from database for model %s", embedding_model) cache.set_cache(embedding_model, db_config) return db_config verbose_proxy_logger.debug( - f"Could not resolve embedding config for model {embedding_model} from router or database" + "Could not resolve embedding config for model %s from router or database", embedding_model ) return None @@ -469,7 +471,7 @@ async def create_vector_store_in_db( if litellm.vector_store_registry is not None: litellm.vector_store_registry.add_vector_store_to_registry(vector_store=new_vector_store) - verbose_proxy_logger.info(f"Vector store {vector_store_id} created in database successfully") + verbose_proxy_logger.info("Vector store %s created in database successfully", vector_store_id) return new_vector_store @@ -542,7 +544,7 @@ async def new_vector_store( "vector_store": response_vs, } except Exception as e: - verbose_proxy_logger.exception(f"Error creating vector store: {e}") + verbose_proxy_logger.exception("Error creating vector store: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -604,7 +606,8 @@ async def list_vector_stores( # If vector store is in memory but NOT in database, it was deleted if vector_store_id not in db_vector_store_ids: verbose_proxy_logger.info( - f"Vector store {vector_store_id} exists in memory but not in database - marking for deletion from cache" + "Vector store %s exists in memory but not in database - marking for deletion from cache", + vector_store_id, ) vector_stores_to_delete_from_memory.append(vector_store_id) # If not in our map yet, add it (only in-memory, not in DB) @@ -615,7 +618,7 @@ async def list_vector_stores( # 1. Remove deleted vector stores from memory for vs_id in vector_stores_to_delete_from_memory: litellm.vector_store_registry.delete_vector_store_from_registry(vector_store_id=vs_id) - verbose_proxy_logger.debug(f"Removed deleted vector store {vs_id} from in-memory registry") + verbose_proxy_logger.debug("Removed deleted vector store %s from in-memory registry", vs_id) # 2. Update in-memory registry with database versions (for updates) for vector_store in vector_stores_from_db: @@ -647,7 +650,7 @@ async def list_vector_stores( return response except Exception as e: - verbose_proxy_logger.exception(f"Error listing vector stores: {e}") + verbose_proxy_logger.exception("Error listing vector stores: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -727,7 +730,7 @@ async def delete_vector_store( except HTTPException: raise except Exception as e: - verbose_proxy_logger.exception(f"Error deleting vector store: {e}") + verbose_proxy_logger.exception("Error deleting vector store: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -799,7 +802,7 @@ async def get_vector_store_info( # the catch-all below would otherwise rewrite them as 500. raise except Exception as e: - verbose_proxy_logger.exception(f"Error getting vector store info: {e}") + verbose_proxy_logger.exception("Error getting vector store info: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -868,7 +871,7 @@ async def update_vector_store( updated_data=updated_vs, ) verbose_proxy_logger.debug( - f"Updated vector store {vector_store_id} in both database and in-memory registry" + "Updated vector store %s in both database and in-memory registry", vector_store_id ) # The DB row is returned in full, so the response would otherwise @@ -888,5 +891,5 @@ async def update_vector_store( # as 500 with the original status code embedded in the detail. raise except Exception as e: - verbose_proxy_logger.exception(f"Error updating vector store: {e}") + verbose_proxy_logger.exception("Error updating vector store: %s", e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/litellm/proxy/vector_store_files_endpoints/endpoints.py b/litellm/proxy/vector_store_files_endpoints/endpoints.py index 06bcc524ea7..feaefece266 100644 --- a/litellm/proxy/vector_store_files_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_files_endpoints/endpoints.py @@ -71,7 +71,7 @@ def _update_request_data_with_managed_file_id( if decoded_id: # This is a unified managed file ID - verbose_logger.debug(f"Processing unified managed file ID: {file_id}") + verbose_logger.debug("Processing unified managed file ID: %s", file_id) # Parse the unified ID to extract components parsed_id = parse_unified_id(file_id) @@ -90,7 +90,9 @@ def _update_request_data_with_managed_file_id( pass verbose_logger.debug( - f"Decoded unified file ID - target_model_names: {target_model_names}, llm_output_file_id: {llm_output_file_id}" + "Decoded unified file ID - target_model_names: %s, llm_output_file_id: %s", + target_model_names, + llm_output_file_id, ) # Set the model for routing @@ -108,14 +110,17 @@ def _update_request_data_with_managed_file_id( file_id=llm_output_file_id, # Use the actual provider file ID ) verbose_logger.info( - f"Routing vector store file operation to model: {routing_model}, file_id: {file_id} -> {llm_output_file_id}" + "Routing vector store file operation to model: %s, file_id: %s -> %s", + routing_model, + file_id, + llm_output_file_id, ) return data, file_id # Return original managed file ID # If we extracted the provider file ID but no routing, still use it if llm_output_file_id: data["file_id"] = llm_output_file_id - verbose_logger.debug(f"Replaced unified file ID with provider file ID: {llm_output_file_id}") + verbose_logger.debug("Replaced unified file ID with provider file ID: %s", llm_output_file_id) return data, file_id # Return original managed file ID return data, file_id if decoded_id else None @@ -361,7 +366,7 @@ def _update_request_data_with_litellm_managed_vector_store_registry( if decoded_id: # This is a managed vector store - decode and extract routing information - verbose_logger.debug(f"Processing managed vector store ID: {vector_store_id}") + verbose_logger.debug("Processing managed vector store ID: %s", vector_store_id) parsed_id = parse_unified_id(vector_store_id) @@ -371,7 +376,10 @@ def _update_request_data_with_litellm_managed_vector_store_registry( target_model_names = parsed_id.get("target_model_names", []) verbose_logger.debug( - f"Decoded vector store - model_id: {model_id}, provider_resource_id: {provider_resource_id}, target_model_names: {target_model_names}" + "Decoded vector store - model_id: %s, provider_resource_id: %s, target_model_names: %s", + model_id, + provider_resource_id, + target_model_names, ) # Set the model for routing - this tells the router which deployment to use @@ -384,13 +392,13 @@ def _update_request_data_with_litellm_managed_vector_store_registry( if routing_model: data["model"] = routing_model - verbose_logger.info(f"Routing vector store files operation to model: {routing_model}") + verbose_logger.info("Routing vector store files operation to model: %s", routing_model) # Replace unified vector store ID with provider resource ID if provider_resource_id: data["vector_store_id"] = provider_resource_id verbose_logger.debug( - f"Replaced unified vector store ID with provider resource ID: {provider_resource_id}" + "Replaced unified vector store ID with provider resource ID: %s", provider_resource_id ) return data diff --git a/litellm/rag/ingestion/base_ingestion.py b/litellm/rag/ingestion/base_ingestion.py index 76e6a4c574c..201baa2f471 100644 --- a/litellm/rag/ingestion/base_ingestion.py +++ b/litellm/rag/ingestion/base_ingestion.py @@ -365,7 +365,7 @@ class BaseRAGIngestion(ABC): ) except Exception as e: - verbose_logger.exception(f"RAG Pipeline failed: {e}") + verbose_logger.exception("RAG Pipeline failed: %s", e) return RAGIngestResponse( id=self.ingest_id, status="failed", diff --git a/litellm/rag/ingestion/bedrock_ingestion.py b/litellm/rag/ingestion/bedrock_ingestion.py index 10dc3af4319..45a3b64810f 100644 --- a/litellm/rag/ingestion/bedrock_ingestion.py +++ b/litellm/rag/ingestion/bedrock_ingestion.py @@ -138,7 +138,7 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): def _auto_detect_config(self): """Auto-detect data source ID and S3 bucket from existing Knowledge Base.""" - verbose_logger.debug(f"Auto-detecting data source and S3 bucket for KB={self.knowledge_base_id}") + verbose_logger.debug("Auto-detecting data source and S3 bucket for KB=%s", self.knowledge_base_id) bedrock_agent = self._get_boto3_client("bedrock-agent") @@ -157,7 +157,7 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): self.data_source_id = self._data_source_id else: self.data_source_id = data_sources[0]["dataSourceId"] - verbose_logger.info(f"Auto-detected data source: {self.data_source_id}") + verbose_logger.info("Auto-detected data source: %s", self.data_source_id) # Get data source details for S3 bucket ds_details = bedrock_agent.get_data_source( @@ -171,7 +171,7 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): if bucket_arn: # Extract bucket name from ARN: arn:aws:s3:::bucket-name self.s3_bucket = self._s3_bucket or bucket_arn.split(":")[-1] - verbose_logger.info(f"Auto-detected S3 bucket: {self.s3_bucket}") + verbose_logger.info("Auto-detected S3 bucket: %s", self.s3_bucket) # Use inclusion prefix if available prefixes = s3_config.get("inclusionPrefixes", []) @@ -218,8 +218,10 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): self.data_source_id = self._create_data_source(kb_name) verbose_logger.info( - f"Created KB infrastructure: kb_id={self.knowledge_base_id}, " - f"ds_id={self.data_source_id}, bucket={self.s3_bucket}" + "Created KB infrastructure: kb_id=%s, ds_id=%s, bucket=%s", + self.knowledge_base_id, + self.data_source_id, + self.s3_bucket, ) def _create_s3_bucket(self, unique_id: str) -> str: @@ -227,7 +229,7 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): s3 = self._get_boto3_client("s3") bucket_name = f"litellm-kb-{unique_id}" - verbose_logger.debug(f"Creating S3 bucket: {bucket_name}") + verbose_logger.debug("Creating S3 bucket: %s", bucket_name) create_params: dict[str, Any] = {"Bucket": bucket_name} if self.aws_region_name != "us-east-1": @@ -236,7 +238,7 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): s3.create_bucket(**create_params) self._created_resources["s3_bucket"] = bucket_name - verbose_logger.info(f"Created S3 bucket: {bucket_name}") + verbose_logger.info("Created S3 bucket: %s", bucket_name) return bucket_name async def _create_opensearch_collection(self, unique_id: str, account_id: str, caller_arn: str) -> tuple[str, str]: @@ -244,7 +246,7 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): oss = self._get_boto3_client("opensearchserverless") collection_name = f"litellm-kb-{unique_id}" - verbose_logger.debug(f"Creating OpenSearch Serverless collection: {collection_name}") + verbose_logger.debug("Creating OpenSearch Serverless collection: %s", collection_name) # Create encryption policy oss.create_security_policy( @@ -290,7 +292,7 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): # This ensures the credentials being used have access to the collection # Normalize the caller ARN (convert assumed-role ARN to IAM role ARN if needed) normalized_caller_arn = _normalize_principal_arn(caller_arn, account_id) - verbose_logger.debug(f"Caller ARN: {caller_arn}, Normalized: {normalized_caller_arn}") + verbose_logger.debug("Caller ARN: %s, Normalized: %s", caller_arn, normalized_caller_arn) principals = [f"arn:aws:iam::{account_id}:root", normalized_caller_arn] # Deduplicate in case caller is root @@ -340,7 +342,7 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): raise TimeoutError("OpenSearch collection did not become active in time") collection_arn = status_response["collectionDetails"][0]["arn"] - verbose_logger.info(f"Created OpenSearch collection: {collection_name}") + verbose_logger.info("Created OpenSearch collection: %s", collection_name) # Wait for data access policy to propagate before returning # AWS recommends waiting 60+ seconds for policy propagation @@ -412,15 +414,17 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): for attempt in range(max_retries): try: client.indices.create(index=index_name, body=index_body) - verbose_logger.info(f"Created OpenSearch index: {index_name}") + verbose_logger.info("Created OpenSearch index: %s", index_name) return except Exception as e: last_error = e error_str = str(e) if "authorization_exception" in error_str.lower() or "security_exception" in error_str.lower(): verbose_logger.warning( - f"OpenSearch index creation attempt {attempt + 1}/{max_retries} failed due to authorization. " - f"Waiting {retry_delay}s for policy propagation..." + "OpenSearch index creation attempt %s/%s failed due to authorization. Waiting %ss for policy propagation...", + attempt + 1, + max_retries, + retry_delay, ) await asyncio.sleep(retry_delay) else: @@ -438,7 +442,7 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): iam = self._get_boto3_client("iam") role_name = f"litellm-bedrock-kb-{unique_id}" - verbose_logger.debug(f"Creating IAM role: {role_name}") + verbose_logger.debug("Creating IAM role: %s", role_name) trust_policy = { "Version": "2012-10-17", @@ -498,14 +502,14 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): # Wait for role to propagate (use asyncio.sleep to avoid blocking) await asyncio.sleep(10) - verbose_logger.info(f"Created IAM role: {role_arn}") + verbose_logger.info("Created IAM role: %s", role_arn) return role_arn async def _create_knowledge_base(self, kb_name: str, role_arn: str, collection_arn: str) -> str: """Create Bedrock Knowledge Base.""" bedrock_agent = self._get_boto3_client("bedrock-agent") - verbose_logger.debug(f"Creating Knowledge Base: {kb_name}") + verbose_logger.debug("Creating Knowledge Base: %s", kb_name) response = bedrock_agent.create_knowledge_base( name=kb_name, @@ -543,14 +547,14 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): else: raise TimeoutError("Knowledge Base did not become active in time") - verbose_logger.info(f"Created Knowledge Base: {kb_id}") + verbose_logger.info("Created Knowledge Base: %s", kb_id) return kb_id def _create_data_source(self, kb_name: str) -> str: """Create Data Source for the Knowledge Base.""" bedrock_agent = self._get_boto3_client("bedrock-agent") - verbose_logger.debug(f"Creating Data Source for KB: {self.knowledge_base_id}") + verbose_logger.debug("Creating Data Source for KB: %s", self.knowledge_base_id) response = bedrock_agent.create_data_source( knowledgeBaseId=self.knowledge_base_id, @@ -566,7 +570,7 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): ds_id = response["dataSource"]["dataSourceId"] self._created_resources["data_source"] = ds_id - verbose_logger.info(f"Created Data Source: {ds_id}") + verbose_logger.info("Created Data Source: %s", ds_id) return ds_id def _get_boto3_client(self, service_name: str): @@ -652,25 +656,25 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): s3_client = self._get_boto3_client("s3") s3_key = f"{self.s3_prefix.rstrip('/')}/{filename}" - verbose_logger.debug(f"Uploading file to s3://{self.s3_bucket}/{s3_key}") + verbose_logger.debug("Uploading file to s3://%s/%s", self.s3_bucket, s3_key) s3_client.put_object( Bucket=self.s3_bucket, Key=s3_key, Body=file_content, ContentType=content_type or "application/octet-stream", ) - verbose_logger.info(f"Uploaded file to s3://{self.s3_bucket}/{s3_key}") + verbose_logger.info("Uploaded file to s3://%s/%s", self.s3_bucket, s3_key) # Step 2: Start ingestion job bedrock_agent = self._get_boto3_client("bedrock-agent") - verbose_logger.debug(f"Starting ingestion job for KB={self.knowledge_base_id}, DS={self.data_source_id}") + verbose_logger.debug("Starting ingestion job for KB=%s, DS=%s", self.knowledge_base_id, self.data_source_id) ingestion_response = bedrock_agent.start_ingestion_job( knowledgeBaseId=self.knowledge_base_id, dataSourceId=self.data_source_id, ) job_id = ingestion_response["ingestionJob"]["ingestionJobId"] - verbose_logger.info(f"Started ingestion job: {job_id}") + verbose_logger.info("Started ingestion job: %s", job_id) # Step 3: Wait for ingestion (optional) - use asyncio.sleep to avoid blocking if self.wait_for_ingestion: @@ -684,22 +688,22 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): ingestionJobId=job_id, ) status = job_status["ingestionJob"]["status"] - verbose_logger.debug(f"Ingestion job {job_id} status: {status}") + verbose_logger.debug("Ingestion job %s status: %s", job_id, status) if status == "COMPLETE": stats = job_status["ingestionJob"].get("statistics", {}) verbose_logger.info( - f"Ingestion complete: {stats.get('numberOfNewDocumentsIndexed', 0)} docs indexed" + "Ingestion complete: %s docs indexed", stats.get("numberOfNewDocumentsIndexed", 0) ) break elif status == "FAILED": failure_reasons = job_status["ingestionJob"].get("failureReasons", []) - verbose_logger.error(f"Ingestion failed: {failure_reasons}") + verbose_logger.error("Ingestion failed: %s", failure_reasons) break elif status in ("STARTING", "IN_PROGRESS"): await asyncio.sleep(2) else: - verbose_logger.warning(f"Unknown ingestion status: {status}") + verbose_logger.warning("Unknown ingestion status: %s", status) break return str(self.knowledge_base_id) if self.knowledge_base_id else None, s3_key diff --git a/litellm/rag/ingestion/file_parsers/pdf_parser.py b/litellm/rag/ingestion/file_parsers/pdf_parser.py index 2b4e07b224f..0301c5d6e03 100644 --- a/litellm/rag/ingestion/file_parsers/pdf_parser.py +++ b/litellm/rag/ingestion/file_parsers/pdf_parser.py @@ -35,7 +35,7 @@ def extract_text_from_pdf(file_content: bytes) -> str | None: if text_parts: extracted_text = "\n\n".join(text_parts) - verbose_logger.debug(f"Extracted {len(extracted_text)} characters from PDF using pypdf") + verbose_logger.debug("Extracted %s characters from PDF using pypdf", len(extracted_text)) return extracted_text except ImportError: @@ -56,13 +56,13 @@ def extract_text_from_pdf(file_content: bytes) -> str | None: if text_parts: extracted_text = "\n\n".join(text_parts) - verbose_logger.debug(f"Extracted {len(extracted_text)} characters from PDF using PyPDF2") + verbose_logger.debug("Extracted %s characters from PDF using PyPDF2", len(extracted_text)) return extracted_text except ImportError: verbose_logger.debug("PyPDF2 not available, PDF extraction requires OCR or pypdf/PyPDF2 library") except Exception as e: - verbose_logger.debug(f"PDF text extraction failed: {e}") + verbose_logger.debug("PDF text extraction failed: %s", e) return None diff --git a/litellm/rag/ingestion/gemini_ingestion.py b/litellm/rag/ingestion/gemini_ingestion.py index 5722936b742..f007e6282b9 100644 --- a/litellm/rag/ingestion/gemini_ingestion.py +++ b/litellm/rag/ingestion/gemini_ingestion.py @@ -162,7 +162,7 @@ class GeminiRAGIngestion(BaseRAGIngestion): response_data = response.json() store_name = response_data.get("name", "") - verbose_logger.debug(f"Created File Search store: {store_name}") + verbose_logger.debug("Created File Search store: %s", store_name) return store_name async def _upload_to_file_search_store( @@ -259,7 +259,7 @@ class GeminiRAGIngestion(BaseRAGIngestion): "x-goog-api-key": api_key, } - verbose_logger.debug(f"Initiating resumable upload: {url}") + verbose_logger.debug("Initiating resumable upload: %s", url) client = get_async_httpx_client( llm_provider=httpxSpecialProvider.RAG, @@ -275,13 +275,13 @@ class GeminiRAGIngestion(BaseRAGIngestion): error_msg = f"Failed to initiate upload: {response.text}" verbose_logger.error(error_msg) raise Exception(error_msg) - verbose_logger.debug(f"Initiate resumable upload response: {response.headers}") + verbose_logger.debug("Initiate resumable upload response: %s", response.headers) # Extract upload URL from response headers upload_url = response.headers.get("x-goog-upload-url") if not upload_url: raise Exception("No upload URL returned in response headers") - verbose_logger.debug(f"Got upload URL: {upload_url}") + verbose_logger.debug("Got upload URL: %s", upload_url) return upload_url async def _upload_file_content( @@ -301,7 +301,7 @@ class GeminiRAGIngestion(BaseRAGIngestion): "X-Goog-Upload-Command": "upload, finalize", } - verbose_logger.debug(f"Uploading file content ({len(file_content)} bytes)") + verbose_logger.debug("Uploading file content (%s bytes)", len(file_content)) client = get_async_httpx_client( llm_provider=httpxSpecialProvider.RAG, @@ -323,9 +323,9 @@ class GeminiRAGIngestion(BaseRAGIngestion): response_data = response.json() # The response should contain the document name or file reference file_id = response_data.get("name", "") or response_data.get("file", {}).get("name", "") - verbose_logger.debug(f"Upload complete. File ID: {file_id}") + verbose_logger.debug("Upload complete. File ID: %s", file_id) return file_id except Exception as e: - verbose_logger.warning(f"Could not parse upload response: {e}") + verbose_logger.warning("Could not parse upload response: %s", e) # Return a placeholder if we can't get the ID return "uploaded" diff --git a/litellm/rag/ingestion/s3_vectors_ingestion.py b/litellm/rag/ingestion/s3_vectors_ingestion.py index 6abd0737ba6..b50e798a7c9 100644 --- a/litellm/rag/ingestion/s3_vectors_ingestion.py +++ b/litellm/rag/ingestion/s3_vectors_ingestion.py @@ -105,7 +105,7 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): try: model_name = self.embedding_config["model"] - verbose_logger.debug(f"Auto-detecting dimension by making test embedding request to {model_name}") + verbose_logger.debug("Auto-detecting dimension by making test embedding request to %s", model_name) # Make a test embedding request test_input = "test" @@ -117,12 +117,13 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): # Get dimension from the response if response.data and len(response.data) > 0: dimension = len(response.data[0]["embedding"]) - verbose_logger.debug(f"Auto-detected dimension {dimension} for embedding model {model_name}") + verbose_logger.debug("Auto-detected dimension %s for embedding model %s", dimension, model_name) return dimension except Exception as e: verbose_logger.warning( - f"Could not auto-detect dimension from embedding model: {e}. " - f"Using default dimension of {S3_VECTORS_DEFAULT_DIMENSION}." + "Could not auto-detect dimension from embedding model: %s. Using default dimension of %s.", + e, + S3_VECTORS_DEFAULT_DIMENSION, ) return S3_VECTORS_DEFAULT_DIMENSION @@ -236,7 +237,7 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): async def _ensure_vector_bucket_exists(self): """Create vector bucket if it doesn't exist using GetVectorBucket and CreateVectorBucket APIs.""" - verbose_logger.debug(f"Ensuring S3 vector bucket exists: {self.vector_bucket_name}") + verbose_logger.debug("Ensuring S3 vector bucket exists: %s", self.vector_bucket_name) # Validate bucket name (AWS S3 naming rules) if len(self.vector_bucket_name) < 3: @@ -259,34 +260,34 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): try: response = await self._sign_and_execute_request("POST", get_url, data=get_body) if response.status_code == 200: - verbose_logger.debug(f"Vector bucket {self.vector_bucket_name} exists") + verbose_logger.debug("Vector bucket %s exists", self.vector_bucket_name) return except Exception as e: - verbose_logger.debug(f"Bucket check failed (may not exist): {e}, attempting to create") + verbose_logger.debug("Bucket check failed (may not exist): %s, attempting to create", e) # Create vector bucket using CreateVectorBucket API try: - verbose_logger.debug(f"Creating vector bucket: {self.vector_bucket_name}") + verbose_logger.debug("Creating vector bucket: %s", self.vector_bucket_name) create_url = f"https://s3vectors.{self.aws_region_name}.api.aws/CreateVectorBucket" create_body = safe_dumps({"vectorBucketName": self.vector_bucket_name}) response = await self._sign_and_execute_request("POST", create_url, data=create_body) if response.status_code in (200, 201): - verbose_logger.info(f"Created vector bucket: {self.vector_bucket_name}") + verbose_logger.info("Created vector bucket: %s", self.vector_bucket_name) elif response.status_code == 409: # Bucket already exists (ConflictException) - verbose_logger.debug(f"Vector bucket {self.vector_bucket_name} already exists") + verbose_logger.debug("Vector bucket %s already exists", self.vector_bucket_name) else: - verbose_logger.error(f"CreateVectorBucket failed: {response.status_code} - {response.text}") + verbose_logger.error("CreateVectorBucket failed: %s - %s", response.status_code, response.text) response.raise_for_status() except Exception as e: - verbose_logger.exception(f"Error creating vector bucket: {e}") + verbose_logger.exception("Error creating vector bucket: %s", e) raise async def _ensure_vector_index_exists(self): """Create vector index if it doesn't exist using GetIndex and CreateIndex APIs.""" - verbose_logger.debug(f"Ensuring vector index exists: {self.vector_bucket_name}/{self.index_name}") + verbose_logger.debug("Ensuring vector index exists: %s/%s", self.vector_bucket_name, self.index_name) # Try to get index info using GetIndex API get_url = f"https://s3vectors.{self.aws_region_name}.api.aws/GetIndex" @@ -295,15 +296,18 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): try: response = await self._sign_and_execute_request("POST", get_url, data=get_body) if response.status_code == 200: - verbose_logger.debug(f"Vector index {self.index_name} exists") + verbose_logger.debug("Vector index %s exists", self.index_name) return except Exception as e: - verbose_logger.debug(f"Index check failed (may not exist): {e}, attempting to create") + verbose_logger.debug("Index check failed (may not exist): %s, attempting to create", e) # Create vector index using CreateIndex API try: verbose_logger.debug( - f"Creating vector index: {self.index_name} with dimension={self.dimension}, metric={self.distance_metric}" + "Creating vector index: %s with dimension=%s, metric=%s", + self.index_name, + self.dimension, + self.distance_metric, ) # Prepare index configuration per AWS API docs @@ -322,14 +326,14 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): response = await self._sign_and_execute_request("POST", create_url, data=safe_dumps(index_config)) if response.status_code in (200, 201): - verbose_logger.info(f"Created vector index: {self.index_name}") + verbose_logger.info("Created vector index: %s", self.index_name) elif response.status_code == 409: - verbose_logger.debug(f"Vector index {self.index_name} already exists") + verbose_logger.debug("Vector index %s already exists", self.index_name) else: - verbose_logger.error(f"CreateIndex failed: {response.status_code} - {response.text}") + verbose_logger.error("CreateIndex failed: %s - %s", response.status_code, response.text) response.raise_for_status() except Exception as e: - verbose_logger.exception(f"Error creating vector index: {e}") + verbose_logger.exception("Error creating vector index: %s", e) raise async def _put_vectors(self, vectors: list[dict[str, Any]]): @@ -339,7 +343,7 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): Args: vectors: List of vector objects with keys: "key", "data", "metadata" """ - verbose_logger.debug(f"Storing {len(vectors)} vectors in {self.vector_bucket_name}/{self.index_name}") + verbose_logger.debug("Storing %s vectors in %s/%s", len(vectors), self.vector_bucket_name, self.index_name) url = f"https://s3vectors.{self.aws_region_name}.api.aws/PutVectors" @@ -354,12 +358,12 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): response = await self._sign_and_execute_request("POST", url, data=safe_dumps(request_body)) if response.status_code in (200, 201): - verbose_logger.info(f"Successfully stored {len(vectors)} vectors in index {self.index_name}") + verbose_logger.info("Successfully stored %s vectors in index %s", len(vectors), self.index_name) else: - verbose_logger.error(f"PutVectors failed with status {response.status_code}: {response.text}") + verbose_logger.error("PutVectors failed with status %s: %s", response.status_code, response.text) response.raise_for_status() except Exception as e: - verbose_logger.exception(f"Error storing vectors: {e}") + verbose_logger.exception("Error storing vectors: %s", e) raise async def embed( @@ -381,7 +385,7 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): embedding_model = self.embedding_config.get("model", "text-embedding-3-small") - verbose_logger.debug(f"Generating embeddings for {len(chunks)} chunks using {embedding_model}") + verbose_logger.debug("Generating embeddings for %s chunks using %s", len(chunks), embedding_model) # Convert to list to ensure type compatibility input_chunks: list[str] = list(chunks) @@ -476,7 +480,7 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): Returns: Query results with vectors and metadata """ - verbose_logger.debug(f"Querying index {vector_store_id} with query: {query}") + verbose_logger.debug("Querying index %s with query: %s", vector_store_id, query) # Generate query embedding if not self.embedding_config: @@ -504,7 +508,7 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): if response.status_code == 200: results = response.json() - verbose_logger.debug(f"Query returned {len(results.get('vectors', []))} results") + verbose_logger.debug("Query returned %s results", len(results.get("vectors", []))) # Check if query terms appear in results if results.get("vectors"): @@ -517,8 +521,8 @@ class S3VectorsRAGIngestion(BaseRAGIngestion, BaseAWSLLM): # Return results even if exact match not found return results else: - verbose_logger.error(f"QueryVectors failed with status {response.status_code}: {response.text}") + verbose_logger.error("QueryVectors failed with status %s: %s", response.status_code, response.text) return None except Exception as e: - verbose_logger.exception(f"Error querying vectors: {e}") + verbose_logger.exception("Error querying vectors: %s", e) return None diff --git a/litellm/rag/ingestion/vertex_ai_ingestion.py b/litellm/rag/ingestion/vertex_ai_ingestion.py index ababca1a955..43c3ed1ac29 100644 --- a/litellm/rag/ingestion/vertex_ai_ingestion.py +++ b/litellm/rag/ingestion/vertex_ai_ingestion.py @@ -169,8 +169,8 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase): "vertexPredictionEndpoint": {"endpoint": embedding_model} } - verbose_logger.debug(f"Creating RAG corpus: {url}") - verbose_logger.debug(f"Request body: {json.dumps(request_body, indent=2)}") + verbose_logger.debug("Creating RAG corpus: %s", url) + verbose_logger.debug("Request body: %s", json.dumps(request_body, indent=2)) client = get_async_httpx_client( llm_provider=httpxSpecialProvider.RAG, @@ -191,7 +191,7 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase): raise Exception(error_msg) response_data = response.json() - verbose_logger.debug(f"Create corpus response: {json.dumps(response_data, indent=2)}") + verbose_logger.debug("Create corpus response: %s", json.dumps(response_data, indent=2)) # The response is a long-running operation # Check if it's already done or if we need to poll @@ -201,13 +201,13 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase): else: # Need to poll the operation operation_name = response_data.get("name", "") - verbose_logger.debug(f"Polling operation: {operation_name}") + verbose_logger.debug("Polling operation: %s", operation_name) corpus_name = await self._poll_operation( operation_name=operation_name, access_token=access_token, ) - verbose_logger.debug(f"Created RAG corpus: {corpus_name}") + verbose_logger.debug("Created RAG corpus: %s", corpus_name) return corpus_name async def _poll_operation( @@ -272,7 +272,7 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase): else: raise Exception(f"No corpus name in operation response: {operation_data}") - verbose_logger.debug(f"Operation not done yet, attempt {attempt + 1}/{max_retries}") + verbose_logger.debug("Operation not done yet, attempt %s/%s", attempt + 1, max_retries) await asyncio.sleep(retry_delay) raise Exception(f"Operation timed out after {max_retries} attempts") @@ -342,8 +342,8 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase): if chunk_overlap: chunking_config["chunk_overlap"] = chunk_overlap - verbose_logger.debug(f"Uploading file to RAG corpus: {url}") - verbose_logger.debug(f"Metadata: {json.dumps(metadata, indent=2)}") + verbose_logger.debug("Uploading file to RAG corpus: %s", url) + verbose_logger.debug("Metadata: %s", json.dumps(metadata, indent=2)) # Prepare multipart form data files = { @@ -381,10 +381,10 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase): if not file_id: file_id = response_data.get("name", "") - verbose_logger.debug(f"Upload complete. File ID: {file_id}") + verbose_logger.debug("Upload complete. File ID: %s", file_id) return file_id except Exception as e: - verbose_logger.warning(f"Could not parse upload response: {e}") + verbose_logger.warning("Could not parse upload response: %s", e) return "uploaded" async def _import_files_from_gcs( @@ -433,8 +433,8 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase): if max_embedding_qpm: request_body["importRagFilesConfig"]["maxEmbeddingRequestsPerMin"] = max_embedding_qpm - verbose_logger.debug(f"Importing files from GCS: {url}") - verbose_logger.debug(f"Request body: {json.dumps(request_body, indent=2)}") + verbose_logger.debug("Importing files from GCS: %s", url) + verbose_logger.debug("Request body: %s", json.dumps(request_body, indent=2)) client = get_async_httpx_client( llm_provider=httpxSpecialProvider.RAG, @@ -458,5 +458,5 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase): response_data = response.json() operation_name = response_data.get("name", "") - verbose_logger.debug(f"Import operation started: {operation_name}") + verbose_logger.debug("Import operation started: %s", operation_name) return operation_name diff --git a/litellm/repositories/config_repository.py b/litellm/repositories/config_repository.py index bc3e7fbaf9f..5d2a6b1421d 100644 --- a/litellm/repositories/config_repository.py +++ b/litellm/repositories/config_repository.py @@ -197,7 +197,7 @@ class ConfigRepository: param_name = response.param_name param_value = response.param_value - verbose_proxy_logger.debug(f"param_name={param_name}, param_value={param_value}") + verbose_proxy_logger.debug("param_name=%s, param_value=%s", param_name, param_value) if param_name is not None and param_value is not None: config = self._update_config_fields( diff --git a/litellm/rerank_api/main.py b/litellm/rerank_api/main.py index 2733fed744a..aa7ade5fdd9 100644 --- a/litellm/rerank_api/main.py +++ b/litellm/rerank_api/main.py @@ -159,7 +159,7 @@ def rerank( instruction=instruction, non_default_params=kwargs, ) - verbose_logger.debug(f"optional_rerank_params: {optional_rerank_params}") + verbose_logger.debug("optional_rerank_params: %s", optional_rerank_params) if isinstance(optional_params.timeout, str): optional_params.timeout = float(optional_params.timeout) @@ -534,5 +534,5 @@ def rerank( # Placeholder return return response except Exception as e: - verbose_logger.error(f"Error in rerank: {e}") + verbose_logger.error("Error in rerank: %s", e) raise exception_type(model=model, custom_llm_provider=custom_llm_provider, original_exception=e) diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index 8c0328fe5f0..357c2d25c22 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -376,7 +376,7 @@ async def acompletion_with_mcp( chunk = await self.follow_up_iterator.__anext__() from litellm._logging import verbose_logger - verbose_logger.debug(f"Follow-up chunk yielded: {chunk}") + verbose_logger.debug("Follow-up chunk yielded: %s", chunk) return chunk except StopAsyncIteration: self.follow_up_exhausted = True @@ -476,7 +476,7 @@ async def acompletion_with_mcp( from litellm._logging import verbose_logger verbose_logger.warning( - f"Follow-up response is not a CustomStreamWrapper: {type(follow_up_response)}" + "Follow-up response is not a CustomStreamWrapper: %s", type(follow_up_response) ) self.follow_up_stream = None diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 39881277a10..0777f86d52b 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -171,7 +171,7 @@ class LiteLLM_Proxy_MCP_Handler: ) return user_api_key_auth.model_copy(update={"object_permission": updated_op}) except Exception as _e: - verbose_logger.debug(f"Could not apply toolset permissions: {_e}") + verbose_logger.debug("Could not apply toolset permissions: %s", _e) return user_api_key_auth @staticmethod @@ -238,14 +238,16 @@ class LiteLLM_Proxy_MCP_Handler: # None means no grants configured → deny (consistent with # fetch_mcp_toolsets which returns [] for unconfigured keys) if granted is None or toolset.toolset_id not in granted: - verbose_logger.debug(f"Key does not have access to toolset '{name}', skipping.") + verbose_logger.debug( + "Key does not have access to toolset '%s', skipping.", name + ) continue resolved_toolset_ids.append(toolset.toolset_id) # Don't add to resolved_mcp_servers — toolset scope # restricts via object_permission, not server name filter. continue except Exception as _e: - verbose_logger.debug(f"Could not resolve '{name}' as toolset: {_e}") + verbose_logger.debug("Could not resolve '%s' as toolset: %s", name, _e) resolved_mcp_servers.append(name) # Apply all resolved toolsets at once (union), avoiding permission overwrite. @@ -664,7 +666,7 @@ class LiteLLM_Proxy_MCP_Handler: ) = LiteLLM_Proxy_MCP_Handler._extract_tool_call_details(tool_call) if not tool_name: - verbose_logger.warning(f"Tool call missing name: {tool_call}") + verbose_logger.warning("Tool call missing name: %s", tool_call) continue parsed_arguments = LiteLLM_Proxy_MCP_Handler._parse_tool_arguments(tool_arguments) @@ -844,7 +846,7 @@ class LiteLLM_Proxy_MCP_Handler: request_data=logging_request_data, error=e, ) - verbose_logger.error(f"BlockedPiiEntityError in MCP tool call: {e}") + verbose_logger.error("BlockedPiiEntityError in MCP tool call: %s", e) error_message = f"Tool call blocked: PII entity '{getattr(e, 'entity_type', 'unknown')}' detected by guardrail '{getattr(e, 'guardrail_name', 'unknown')}'. {e}" tool_results.append( { @@ -860,7 +862,7 @@ class LiteLLM_Proxy_MCP_Handler: request_data=logging_request_data, error=e, ) - verbose_logger.error(f"GuardrailRaisedException in MCP tool call: {e}") + verbose_logger.error("GuardrailRaisedException in MCP tool call: %s", e) error_message = ( f"Tool call blocked: Guardrail '{getattr(e, 'guardrail_name', 'unknown')}' violation. {e}" ) @@ -878,7 +880,7 @@ class LiteLLM_Proxy_MCP_Handler: request_data=logging_request_data, error=e, ) - verbose_logger.error(f"HTTPException in MCP tool call: {e}") + verbose_logger.error("HTTPException in MCP tool call: %s", e) error_message = f"Tool call failed: {str(e.detail) if hasattr(e, 'detail') else str(e)}" tool_results.append( { @@ -894,7 +896,7 @@ class LiteLLM_Proxy_MCP_Handler: request_data=logging_request_data, error=e, ) - verbose_logger.exception(f"Error executing MCP tool call: {e}") + verbose_logger.exception("Error executing MCP tool call: %s", e) tool_results.append( { "tool_call_id": tool_call_id, diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index c68628429da..c384dd86f5e 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -124,10 +124,10 @@ async def create_mcp_list_tools_events( ) events.append(output_item_done_event) - verbose_logger.debug(f"Created {len(events)} MCP discovery events") + verbose_logger.debug("Created %s MCP discovery events", len(events)) except Exception as e: - verbose_logger.error(f"Error creating MCP list tools events: {e}") + verbose_logger.error("Error creating MCP list tools events: %s", e) import traceback traceback.print_exc() @@ -513,7 +513,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): response_obj = getattr(chunk, "response", None) if response_obj and hasattr(response_obj, "id"): self._cached_response_id = response_obj.id - verbose_logger.debug(f"Cached response ID: {self._cached_response_id}") + verbose_logger.debug("Cached response ID: %s", self._cached_response_id) # After emitting response.output_item.added, transition to MCP discovery if not self.initial_events_emitted and hasattr(chunk, "type"): @@ -576,7 +576,9 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): response_obj = getattr(chunk, "response", None) if response_obj and hasattr(response_obj, "id"): if response_obj.id != self._cached_response_id: - verbose_logger.debug(f"Updating response ID from {response_obj.id} to {self._cached_response_id}") + verbose_logger.debug( + "Updating response ID from %s to %s", response_obj.id, self._cached_response_id + ) response_obj.id = self._cached_response_id # If auto-execution is enabled, check for completed responses @@ -607,7 +609,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): params_for_llm[key] = value # Copy all params as-is since tools are already processed tools_count = len(params_for_llm.get("tools", [])) if params_for_llm.get("tools") else 0 - verbose_logger.debug(f"Making LLM call with {tools_count} tools") + verbose_logger.debug("Making LLM call with %s tools", tools_count) response = await aresponses(**params_for_llm) # Set the base iterator @@ -617,15 +619,15 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): self.model = getattr(response, "model", self.model) self.litellm_metadata = getattr(response, "litellm_metadata", {}) self.custom_llm_provider = getattr(response, "custom_llm_provider", self.custom_llm_provider) - verbose_logger.debug(f"Created base iterator: {type(self.base_iterator)}") + verbose_logger.debug("Created base iterator: %s", type(self.base_iterator)) else: # Non-streaming response - this shouldn't happen but handle it - verbose_logger.warning(f"Got non-streaming response: {type(response)}") + verbose_logger.warning("Got non-streaming response: %s", type(response)) self.base_iterator = None self.phase = "finished" except Exception as e: - verbose_logger.error(f"Error creating initial response iterator: {e}") + verbose_logger.error("Error creating initial response iterator: %s", e) import traceback traceback.print_exc() @@ -742,7 +744,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): self._tool_results_for_response = self.collected_response except Exception as e: - verbose_logger.error(f"Error in tool execution: {e}") + verbose_logger.error("Error in tool execution: %s", e) import traceback traceback.print_exc() @@ -807,7 +809,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): self._cached_response_id = None except Exception as e: - verbose_logger.error(f"Error creating follow-up iterator: {e}") + verbose_logger.error("Error creating follow-up iterator: %s", e) import traceback traceback.print_exc() diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 6e951367995..3cd118fb1a4 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -557,7 +557,7 @@ class ResponsesAPIRequestUtils: response_id=decoded_response_id, ) except Exception as e: - verbose_logger.debug(f"Error decoding response_id '{response_id}': {e}") + verbose_logger.debug("Error decoding response_id '%s': %s", response_id, e) return DecodedResponseId( custom_llm_provider=None, model_id=None, @@ -670,7 +670,7 @@ class ResponsesAPIRequestUtils: response_id=original_container_id, ) except Exception as e: - verbose_logger.debug(f"Error decoding container_id '{container_id}': {e}") + verbose_logger.debug("Error decoding container_id '%s': %s", container_id, e) return DecodedResponseId( custom_llm_provider=None, model_id=None, diff --git a/litellm/router.py b/litellm/router.py index 6bf1bdfc670..7d6499cf7d2 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -721,7 +721,8 @@ class Router: self.retry_policy = retry_policy if self.retry_policy is not None: verbose_router_logger.info( - f"\033[32mRouter Custom Retry Policy Set:\n{self.retry_policy.model_dump(exclude_none=True)}\033[0m" + "\x1b[32mRouter Custom Retry Policy Set:\n%s\x1b[0m", + self.retry_policy.model_dump(exclude_none=True), ) self.model_group_retry_policy: dict[str, RetryPolicy] | None = model_group_retry_policy @@ -736,7 +737,8 @@ class Router: if self.allowed_fails_policy is not None: verbose_router_logger.info( - f"\033[32mRouter Custom Allowed Fails Policy Set:\n{self.allowed_fails_policy.model_dump(exclude_none=True)}\033[0m" + "\x1b[32mRouter Custom Allowed Fails Policy Set:\n%s\x1b[0m", + self.allowed_fails_policy.model_dump(exclude_none=True), ) self.alerting_config: AlertingConfig | None = alerting_config @@ -941,7 +943,7 @@ class Router: litellm.input_callback = [c for c in litellm.input_callback if id(c) not in selector_ids] def routing_strategy_init(self, routing_strategy: RoutingStrategy | str, routing_strategy_args: dict): - verbose_router_logger.info(f"Routing strategy: {routing_strategy}") + verbose_router_logger.info("Routing strategy: %s", routing_strategy) self._validate_routing_strategy(routing_strategy) self._reset_custom_routing_strategy() @@ -1721,7 +1723,7 @@ class Router: return _deployment_copy except Exception as e: - verbose_router_logger.debug(f"Error occurred while printing deployment - {e}") + verbose_router_logger.debug("Error occurred while printing deployment - %s", e) raise e ### COMPLETION, EMBEDDING, IMG GENERATION FUNCTIONS @@ -1732,7 +1734,7 @@ class Router: response = router.completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hey, how's it going?"}] """ try: - verbose_router_logger.debug(f"router.completion(model={model},..)") + verbose_router_logger.debug("router.completion(model=%s,..)", model) kwargs["model"] = model kwargs["messages"] = messages kwargs["original_function"] = self._completion @@ -1806,7 +1808,7 @@ class Router: **kwargs, } response = litellm.completion(**input_kwargs) - verbose_router_logger.info(f"litellm.completion(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) ## CHECK CONTENT FILTER ERROR ## if isinstance(response, ModelResponse): @@ -1829,7 +1831,7 @@ class Router: return response except Exception as e: - verbose_router_logger.info(f"litellm.completion(model={model_name})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.completion(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) # Set per-deployment num_retries on exception for retry logic if deployment is not None: self._set_deployment_num_retries_on_exception(e, deployment) @@ -1892,7 +1894,7 @@ class Router: messages = copy.deepcopy(messages) - verbose_router_logger.info(f"Starting silent experiment for model {silent_model}") + verbose_router_logger.info("Starting silent experiment for model %s", silent_model) silent_kwargs = self._get_silent_experiment_kwargs(**kwargs) @@ -1924,7 +1926,7 @@ class Router: finally: loop.close() except Exception as e: - verbose_router_logger.error(f"Silent experiment failed for model {silent_model}: {e}") + verbose_router_logger.error("Silent experiment failed for model %s: %s", silent_model, e) # fmt: off @@ -2160,7 +2162,7 @@ class Router: except Exception as fallback_error: # If fallback also fails, log and re-raise original error - verbose_router_logger.error(f"Fallback also failed: {fallback_error}") + verbose_router_logger.error("Fallback also failed: %s", fallback_error) # No fallback handled the mid-stream error, so surface the # real provider exception (e.g. RateLimitError) instead of # leaking the internal MidStreamFallbackError to the client @@ -2579,7 +2581,7 @@ class Router: else: yield fallback_response except Exception as fallback_error: - verbose_router_logger.error(f"Responses streaming fallback also failed: {fallback_error}") + verbose_router_logger.error("Responses streaming fallback also failed: %s", fallback_error) if ( isinstance(fallback_error, MidStreamFallbackError) and fallback_error.original_exception is not None @@ -2704,7 +2706,7 @@ class Router: yield None except Exception as fallback_error: - verbose_router_logger.error(f"Fallback also failed: {fallback_error}") + verbose_router_logger.error("Fallback also failed: %s", fallback_error) if ( isinstance(fallback_error, MidStreamFallbackError) and fallback_error.original_exception is not None @@ -2742,7 +2744,7 @@ class Router: messages = copy.deepcopy(messages) - verbose_router_logger.info(f"Starting silent experiment for model {silent_model}") + verbose_router_logger.info("Starting silent experiment for model %s", silent_model) silent_kwargs = self._get_silent_experiment_kwargs(**kwargs) # Override model_group to correctly attribute metrics to the silent model @@ -2755,7 +2757,7 @@ class Router: **silent_kwargs, ) except Exception as e: - verbose_router_logger.error(f"Silent experiment failed for model {silent_model}: {e}") + verbose_router_logger.error("Silent experiment failed for model %s: %s", silent_model, e) async def _acompletion( self, model: str, messages: list[dict[str, str]], **kwargs @@ -2879,7 +2881,7 @@ class Router: ) self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.acompletion(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.acompletion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) # debug how often this deployment picked self._track_deployment_metrics( deployment=deployment, @@ -2908,7 +2910,7 @@ class Router: self._set_failed_deployment_id_on_exception(e, deployment) raise e except Exception as e: - verbose_router_logger.info(f"litellm.acompletion(model={model_name})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.acompletion(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) if model_name is not None: self.fail_calls[model_name] += 1 # Set per-deployment num_retries on exception for retry logic @@ -3363,7 +3365,7 @@ class Router: result = await self.acompletion(model=model, messages=messages, stream=stream, **kwargs) # type: ignore return result except asyncio.CancelledError: - verbose_router_logger.debug(f"Received 'task.cancel'. Cancelling call w/ model={model}.") + verbose_router_logger.debug("Received 'task.cancel'. Cancelling call w/ model=%s.", model) raise except Exception as e: return e @@ -3665,7 +3667,7 @@ class Router: def _image_generation(self, prompt: str, model: str, **kwargs): model_name = "" try: - verbose_router_logger.debug(f"Inside _image_generation()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _image_generation()- model: %s; kwargs: %s", model, kwargs) deployment = self.get_available_deployment( model=model, messages=[{"role": "user", "content": "prompt"}], @@ -3694,10 +3696,10 @@ class Router: } ) self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.image_generation(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.info(f"litellm.image_generation(model={model_name})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -3725,7 +3727,7 @@ class Router: async def _aimage_generation(self, prompt: str, model: str, **kwargs): model_name = model try: - verbose_router_logger.debug(f"Inside _image_generation()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _image_generation()- model: %s; kwargs: %s", model, kwargs) parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) deployment = await self.async_get_available_deployment( model=model, @@ -3778,10 +3780,10 @@ class Router: response = await response self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.aimage_generation(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.aimage_generation(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.info(f"litellm.aimage_generation(model={model_name})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.aimage_generation(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -3831,7 +3833,7 @@ class Router: async def _atranscription(self, file: FileTypes, model: str, **kwargs): model_name = model try: - verbose_router_logger.debug(f"Inside _atranscription()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _atranscription()- model: %s; kwargs: %s", model, kwargs) parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) deployment = await self.async_get_available_deployment( model=model, @@ -3882,10 +3884,10 @@ class Router: response = await response self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.atranscription(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.atranscription(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.info(f"litellm.atranscription(model={model_name})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.atranscription(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -3945,7 +3947,7 @@ class Router: async def _aspeech(self, model: str, input: str, voice: str, **kwargs): model_name = model try: - verbose_router_logger.debug(f"Inside _aspeech()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _aspeech()- model: %s; kwargs: %s", model, kwargs) parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) deployment = await self.async_get_available_deployment( model=model, @@ -3996,10 +3998,10 @@ class Router: response = await response self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.aspeech(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.aspeech(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.info(f"litellm.aspeech(model={model_name})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.aspeech(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -4028,7 +4030,7 @@ class Router: async def _arerank(self, model: str, **kwargs): model_name = None try: - verbose_router_logger.debug(f"Inside _rerank()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _rerank()- model: %s; kwargs: %s", model, kwargs) deployment = await self.async_get_available_deployment( model=model, specific_deployment=kwargs.pop("specific_deployment", None), @@ -4054,10 +4056,10 @@ class Router: ) self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.arerank(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.arerank(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.info(f"litellm.arerank(model={model_name})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.arerank(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -4136,7 +4138,7 @@ class Router: async def _atext_completion(self, model: str, prompt: str, **kwargs): try: - verbose_router_logger.debug(f"Inside _atext_completion()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _atext_completion()- model: %s; kwargs: %s", model, kwargs) parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) deployment = await self.async_get_available_deployment( model=model, @@ -4188,10 +4190,10 @@ class Router: response = await response self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.atext_completion(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.atext_completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.info(f"litellm.atext_completion(model={model})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.atext_completion(model=%s)\x1b[31m Exception %s\x1b[0m", model, e) if model is not None: self.fail_calls[model] += 1 raise e @@ -4226,7 +4228,7 @@ class Router: async def _aadapter_completion(self, adapter_id: str, model: str, **kwargs): try: - verbose_router_logger.debug(f"Inside _aadapter_completion()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _aadapter_completion()- model: %s; kwargs: %s", model, kwargs) parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) deployment = await self.async_get_available_deployment( model=model, @@ -4278,10 +4280,10 @@ class Router: response = await response # type: ignore self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.aadapter_completion(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.aadapter_completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.info(f"litellm.aadapter_completion(model={model})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.aadapter_completion(model=%s)\x1b[31m Exception %s\x1b[0m", model, e) if model is not None: self.fail_calls[model] += 1 raise e @@ -4338,7 +4340,7 @@ class Router: kwargs=kwargs, metadata_variable_name="litellm_metadata", ) - verbose_router_logger.debug(f"Inside aguardrail() - guardrail_name: {guardrail_name}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside aguardrail() - guardrail_name: %s; kwargs: %s", guardrail_name, kwargs) response = await self.async_function_with_fallbacks(**kwargs) return response @@ -4363,7 +4365,7 @@ class Router: ) verbose_router_logger.debug( - f"Selected guardrail deployment: {selected_guardrail.get('litellm_params', {}).get('guardrail')}" + "Selected guardrail deployment: %s", selected_guardrail.get("litellm_params", {}).get("guardrail") ) # Pass the selected guardrail config to the original function @@ -4413,7 +4415,9 @@ class Router: kwargs["original_generic_function"] = original_function kwargs["original_function"] = self._ageneric_api_call_with_fallbacks_helper self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs, metadata_variable_name="litellm_metadata") - verbose_router_logger.debug(f"Inside ageneric_api_call_with_fallbacks() - model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug( + "Inside ageneric_api_call_with_fallbacks() - model: %s; kwargs: %s", model, kwargs + ) response = await self.async_function_with_fallbacks(**kwargs) return response @@ -4536,11 +4540,13 @@ class Router: response = await response # type: ignore self.success_calls[model_name] += 1 - verbose_router_logger.info(f"ageneric_api_call_with_fallbacks(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("ageneric_api_call_with_fallbacks(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.info(f"ageneric_api_call_with_fallbacks(model={model})\033[31m Exception {e}\033[0m") + verbose_router_logger.info( + "ageneric_api_call_with_fallbacks(model=%s)\x1b[31m Exception %s\x1b[0m", model, e + ) if model is not None: self.fail_calls[model] += 1 raise e @@ -4607,7 +4613,7 @@ class Router: metadata_variable_name = _get_router_metadata_variable_name(function_name="generic_api_call") try: verbose_router_logger.debug( - f"Inside _generic_api_call() - handler: {handler_name}, model: {model}; kwargs: {kwargs}" + "Inside _generic_api_call() - handler: %s, model: %s; kwargs: %s", handler_name, model, kwargs ) self._update_kwargs_before_fallbacks( model=model, @@ -4657,10 +4663,10 @@ class Router: ) self.success_calls[model_name] += 1 - verbose_router_logger.info(f"{handler_name}(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("%s(model=%s)\x1b[32m 200 OK\x1b[0m", handler_name, model_name) return response except Exception as e: - verbose_router_logger.info(f"{handler_name}(model={model})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("%s(model=%s)\x1b[31m Exception %s\x1b[0m", handler_name, model, e) if model is not None: self.fail_calls[model] += 1 raise e @@ -4685,7 +4691,7 @@ class Router: def _embedding(self, input: str | list, model: str, **kwargs): model_name = None try: - verbose_router_logger.debug(f"Inside embedding()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside embedding()- model: %s; kwargs: %s", model, kwargs) deployment = self.get_available_deployment( model=model, input=input, @@ -4722,10 +4728,10 @@ class Router: } ) self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.embedding(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.embedding(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.info(f"litellm.embedding(model={model_name})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.embedding(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -4758,7 +4764,7 @@ class Router: async def _aembedding(self, input: str | list, model: str, **kwargs): model_name = None try: - verbose_router_logger.debug(f"Inside _aembedding()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _aembedding()- model: %s; kwargs: %s", model, kwargs) parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) deployment = await self.async_get_available_deployment( model=model, @@ -4809,10 +4815,10 @@ class Router: response = await response self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.aembedding(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.aembedding(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.info(f"litellm.aembedding(model={model_name})\033[31m Exception {e}\033[0m") + verbose_router_logger.info("litellm.aembedding(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) if model_name is not None: self.fail_calls[model_name] += 1 raise e @@ -4849,7 +4855,7 @@ class Router: try: from litellm.router_utils.common_utils import add_model_file_id_mappings - verbose_router_logger.debug(f"Inside _atext_completion()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _atext_completion()- model: %s; kwargs: %s", model, kwargs) parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) healthy_deployments = await self.async_get_healthy_deployments( model=model, @@ -4940,7 +4946,7 @@ class Router: response = await response # type: ignore self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.acreate_file(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.acreate_file(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response @@ -4965,7 +4971,7 @@ class Router: return returned_response except Exception as e: verbose_router_logger.exception( - f"litellm.acreate_file(model={model}, {kwargs})\033[31m Exception {e}\033[0m" + "litellm.acreate_file(model=%s, %s)\x1b[31m Exception %s\x1b[0m", model, kwargs, e ) if model is not None: self.fail_calls[model] += 1 @@ -5056,11 +5062,13 @@ class Router: response = await response self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.avector_store_create(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.avector_store_create(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response except Exception as e: - verbose_router_logger.exception(f"litellm.avector_store_create(model={model})\033[31m Exception {e}\033[0m") + verbose_router_logger.exception( + "litellm.avector_store_create(model=%s)\x1b[31m Exception %s\x1b[0m", model, e + ) if model is not None: self.fail_calls[model] += 1 raise e @@ -5110,7 +5118,7 @@ class Router: **kwargs, ) -> LiteLLMBatch: try: - verbose_router_logger.debug(f"Inside _acreate_batch()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _acreate_batch()- model: %s; kwargs: %s", model, kwargs) parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) deployment = await self.async_get_available_deployment( model=model, @@ -5170,12 +5178,12 @@ class Router: response = await response # type: ignore self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.acreate_batch(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.acreate_batch(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response # type: ignore except Exception as e: verbose_router_logger.exception( - f"litellm._acreate_batch(model={model}, {kwargs})\033[31m Exception {e}\033[0m" + "litellm._acreate_batch(model=%s, %s)\x1b[31m Exception %s\x1b[0m", model, kwargs, e ) if model is not None: self.fail_calls[model] += 1 @@ -5325,7 +5333,7 @@ class Router: **kwargs, ) -> LiteLLMBatch: try: - verbose_router_logger.debug(f"Inside _acancel_batch()- model: {model}; kwargs: {kwargs}") + verbose_router_logger.debug("Inside _acancel_batch()- model: %s; kwargs: %s", model, kwargs) parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) deployment = await self.async_get_available_deployment( model=model, @@ -5392,12 +5400,12 @@ class Router: response = await response # type: ignore self.success_calls[model_name] += 1 - verbose_router_logger.info(f"litellm.acancel_batch(model={model_name})\033[32m 200 OK\033[0m") + verbose_router_logger.info("litellm.acancel_batch(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response # type: ignore except Exception as e: verbose_router_logger.exception( - f"litellm._acancel_batch(model={model}, {kwargs})\033[31m Exception {e}\033[0m" + "litellm._acancel_batch(model=%s, %s)\x1b[31m Exception %s\x1b[0m", model, kwargs, e ) if model is not None: self.fail_calls[model] += 1 @@ -6061,8 +6069,10 @@ class Router: return None verbose_router_logger.debug( - f"Weighted failover: exclude={excluded!r}, remaining={len(remaining)} " - f"for model_group={original_model_group!r}" + "Weighted failover: exclude=%r, remaining=%s for model_group=%r", + excluded, + len(remaining), + original_model_group, ) meta["_failover_excluded_ids"] = list(excluded) @@ -6107,7 +6117,7 @@ class Router: """ Common utilities for async_function_with_fallbacks """ - verbose_router_logger.debug(f"Traceback{traceback.format_exc()}") + verbose_router_logger.debug("Traceback%s", traceback.format_exc()) original_exception = e fallback_model_group = None original_model_group: str | None = kwargs.get("model") # type: ignore @@ -6284,7 +6294,7 @@ class Router: if litellm.expose_router_debug_in_errors: e.message += f"\n{error_message}" if fallbacks is not None and model_group is not None: - verbose_router_logger.debug(f"inside model fallbacks: {mask_sensitive_structure(fallbacks)}") + verbose_router_logger.debug("inside model fallbacks: %s", mask_sensitive_structure(fallbacks)) ( fallback_model_group, generic_fallback_idx, @@ -6299,7 +6309,9 @@ class Router: if fallback_model_group is None: masked_fallbacks = mask_sensitive_structure(fallbacks) verbose_router_logger.info( - f"No fallback model group found for original model_group={model_group}. Fallbacks={masked_fallbacks}" + "No fallback model group found for original model_group=%s. Fallbacks=%s", + model_group, + masked_fallbacks, ) if hasattr(original_exception, "message") and litellm.expose_router_debug_in_errors: original_exception.message += f"No fallback model group found for original model_group={model_group}. Fallbacks={masked_fallbacks}" # type: ignore @@ -6374,7 +6386,7 @@ class Router: else: response = await self.async_function_with_retries(*args, **kwargs) if verbose_router_logger.isEnabledFor(logging.DEBUG): - verbose_router_logger.debug(f"Async Response: {response}") + verbose_router_logger.debug("Async Response: %s", response) response = add_fallback_headers_to_response( response=response, attempted_fallbacks=0, @@ -6466,7 +6478,7 @@ class Router: _metadata.update({"model_group_size": len(model_list)}) verbose_router_logger.debug( - f"async function w/ retries: original_function - {original_function}, num_retries - {num_retries}" + "async function w/ retries: original_function - %s, num_retries - %s", original_function, num_retries ) ## ADD RETRY TRACKING TO METADATA - used for spend logs retry tracking _metadata["attempted_retries"] = 0 @@ -6538,7 +6550,7 @@ class Router: else: raise - verbose_router_logger.debug(f"Retrying request with num_retries: {num_retries}") + verbose_router_logger.debug("Retrying request with num_retries: %s", num_retries) # decides how long to sleep before retry retry_after = self._time_to_sleep_before_retry( e=original_exception, @@ -6655,7 +6667,7 @@ class Router: if mock_testing_rate_limit_error is not None and mock_testing_rate_limit_error is True: verbose_router_logger.info( - f"litellm.router.py::_mock_rate_limit_error() - Raising mock RateLimitError for model={model_group}" + "litellm.router.py::_mock_rate_limit_error() - Raising mock RateLimitError for model=%s", model_group ) raise litellm.RateLimitError( model=model_group, @@ -6945,7 +6957,7 @@ class Router: except Exception as e: verbose_router_logger.debug( - f"litellm.router.Router::deployment_callback_on_success(): Exception occured - {e}" + "litellm.router.Router::deployment_callback_on_success(): Exception occured - %s", e ) def sync_deployment_callback_on_success( @@ -7198,7 +7210,9 @@ class Router: return True verbose_router_logger.debug( - f"Content Policy Error occurred. No available fallbacks. Returning original response. model={model}, content_policy_fallbacks={content_policy_fallbacks}" + "Content Policy Error occurred. No available fallbacks. Returning original response. model=%s, content_policy_fallbacks=%s", + model, + content_policy_fallbacks, ) return False @@ -7537,7 +7551,9 @@ class Router: ## Check if LLM Deployment is allowed for this deployment if self.deployment_is_active_for_environment(deployment=deployment) is not True: verbose_router_logger.warning( - f"Ignoring deployment {deployment.model_name} as it is not active for environment {deployment.model_info['supported_environments']}" + "Ignoring deployment %s as it is not active for environment %s", + deployment.model_name, + deployment.model_info["supported_environments"], ) return None @@ -7561,7 +7577,7 @@ class Router: except Exception as e: if self.ignore_invalid_deployments: verbose_router_logger.exception( - f"Error creating deployment: {e}, ignoring and continuing with other deployments." + "Error creating deployment: %s, ignoring and continuing with other deployments.", e ) return None else: @@ -8014,7 +8030,7 @@ class Router: _model_info=_model_info, ) - verbose_router_logger.debug(f"\nInitialized Model List {self.get_model_names()}") + verbose_router_logger.debug("\nInitialized Model List %s", self.get_model_names()) self.model_names = {m["model_name"] for m in model_list} # Note: model_name_to_deployment_indices is already built incrementally @@ -8436,8 +8452,10 @@ class Router: except Exception as e: if self.ignore_invalid_deployments: verbose_router_logger.warning( - f"Error upserting deployment {deployment.model_name} (id={deployment.model_info.id}): {e}. " - "Dropping it and continuing with other deployments." + "Error upserting deployment %s (id=%s): %s. Dropping it and continuing with other deployments.", + deployment.model_name, + deployment.model_info.id, + e, ) return None else: @@ -8710,7 +8728,7 @@ class Router: ) if not credential_values: verbose_router_logger.warning( - f"Credential '{deployment.litellm_params.litellm_credential_name}' not found in credential_list" + "Credential '%s' not found in credential_list", deployment.litellm_params.litellm_credential_name ) credentials.update(credential_values) # Remove the credential name since we've resolved it @@ -8795,7 +8813,8 @@ class Router: ## SET MODEL TO 'model=' - if base_model is None + not azure if custom_llm_provider == "azure" and base_model is None: verbose_router_logger.error( - f"Could not identify azure model '{_model}'. Set azure 'base_model' for accurate max tokens, cost tracking, etc.- https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models" + "Could not identify azure model '%s'. Set azure 'base_model' for accurate max tokens, cost tracking, etc.- https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models", + _model, ) elif custom_llm_provider != "azure": model = _model @@ -9011,7 +9030,7 @@ class Router: custom_llm_provider=litellm_params.custom_llm_provider, ) except litellm.exceptions.BadRequestError as e: - verbose_router_logger.error(f"litellm.router.py::get_model_group_info() - {e}") + verbose_router_logger.error("litellm.router.py::get_model_group_info() - %s", e) if model_info is None: supported_openai_params = litellm.get_supported_openai_params( @@ -10073,7 +10092,7 @@ class Router: relink_lar1_from_args = True setattr(self, var, value) else: - verbose_router_logger.debug(f"Setting {var} is not allowed") + verbose_router_logger.debug("Setting %s is not allowed", var) if relink_lar1_from_args and self._normalize_strategy(self.routing_strategy) == "lar1": from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy @@ -10082,7 +10101,7 @@ class Router: if rebuild_routing_groups: self._init_routing_groups(self._routing_groups_input) - verbose_router_logger.debug(f"Updated Router settings: {self.get_settings()}") + verbose_router_logger.debug("Updated Router settings: %s", self.get_settings()) def _get_client(self, deployment, kwargs, client_type=None): """ @@ -10173,7 +10192,7 @@ class Router: - [TODO] function call and model doesn't support function calling """ - verbose_router_logger.debug(f"Starting Pre-call checks for deployments in model={model}") + verbose_router_logger.debug("Starting Pre-call checks for deployments in model=%s", model) # Optimized: Use list() shallow copy instead of deepcopy # We only pop from the list, not modify deployment dicts - 100x+ faster on hot path (every request) @@ -10225,7 +10244,8 @@ class Router: ) except Exception as e: verbose_router_logger.error( - f"litellm.router.py::_pre_call_checks: failed to count tokens. Returning initial list of deployments. Got - {e}" + "litellm.router.py::_pre_call_checks: failed to count tokens. Returning initial list of deployments. Got - %s", + e, ) return _returned_deployments if input_tokens > max_input_tokens: @@ -10236,7 +10256,7 @@ class Router: ) continue except Exception as e: - verbose_router_logger.exception(f"An error occurs - {e}") + verbose_router_logger.exception("An error occurs - %s", e) model_id = _model_info.get("id", "") ## RPM CHECK ## @@ -10297,7 +10317,7 @@ class Router: for k, v in non_default_params.items(): if k not in supported_openai_params and k in special_params: # if not -> invalid model - verbose_router_logger.debug(f"INVALID MODEL INDEX @ REQUEST KWARG FILTERING, k={k}") + verbose_router_logger.debug("INVALID MODEL INDEX @ REQUEST KWARG FILTERING, k=%s", k) invalid_model_indices.add(idx) if len(invalid_model_indices) == len(_returned_deployments): @@ -10490,7 +10510,7 @@ class Router: _access_group_filter_emptied_candidates = True if verbose_router_logger.isEnabledFor(logging.DEBUG): - verbose_router_logger.debug(f"initial list of deployments: {healthy_deployments}") + verbose_router_logger.debug("initial list of deployments: %s", healthy_deployments) if len(healthy_deployments) == 0: # Check for default fallbacks if no deployments are found for the requested model @@ -10501,7 +10521,7 @@ class Router: fallback_model = self._get_first_default_fallback() if fallback_model: verbose_router_logger.info( - f"Model '{model}' not found. Attempting to use default fallback model '{fallback_model}'." + "Model '%s' not found. Attempting to use default fallback model '%s'.", model, fallback_model ) # Re-assign model to the fallback and try to get deployments again model = fallback_model @@ -10618,7 +10638,7 @@ class Router: ) if verbose_router_logger.isEnabledFor(logging.DEBUG): - verbose_router_logger.debug(f"healthy_deployments after team filter: {healthy_deployments}") + verbose_router_logger.debug("healthy_deployments after team filter: %s", healthy_deployments) healthy_deployments = filter_web_search_deployments( healthy_deployments=healthy_deployments, @@ -10626,7 +10646,7 @@ class Router: ) if verbose_router_logger.isEnabledFor(logging.DEBUG): - verbose_router_logger.debug(f"healthy_deployments after web search filter: {healthy_deployments}") + verbose_router_logger.debug("healthy_deployments after web search filter: %s", healthy_deployments) if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: @@ -10647,7 +10667,7 @@ class Router: litellm_router_instance=self, parent_otel_span=parent_otel_span ) if verbose_router_logger.isEnabledFor(logging.DEBUG): - verbose_router_logger.debug(f"cooldown deployments: {cooldown_deployments}") + verbose_router_logger.debug("cooldown deployments: %s", cooldown_deployments) _pre_cooldown_deployments = healthy_deployments healthy_deployments = self._filter_cooldown_deployments( healthy_deployments=healthy_deployments, @@ -10810,7 +10830,10 @@ class Router: ) raise exception verbose_router_logger.info( - f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment)} for model: {model}" + "get_available_deployment for model: %s, Selected deployment: %s for model: %s", + model, + self.print_deployment(deployment), + model, ) end_time = time.time() @@ -10939,7 +10962,9 @@ class Router: raise exception verbose_router_logger.info( - f"async_get_available_deployment_for_pass_through model: {model}, selected deployment: {self.print_deployment(deployment)}" + "async_get_available_deployment_for_pass_through model: %s, selected deployment: %s", + model, + self.print_deployment(deployment), ) end_time = time.perf_counter() @@ -11312,7 +11337,7 @@ class Router: ) if deployment is None: - verbose_router_logger.info(f"get_available_deployment for model: {model}, No deployment available") + verbose_router_logger.info("get_available_deployment for model: %s, No deployment available", model) model_ids = self.get_model_ids(model_name=model) _cooldown_time = self.cooldown_cache.get_min_cooldown( model_ids=model_ids, parent_otel_span=parent_otel_span @@ -11325,7 +11350,10 @@ class Router: cooldown_list=_cooldown_list, ) verbose_router_logger.info( - f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment)} for model: {model}" + "get_available_deployment for model: %s, Selected deployment: %s for model: %s", + model, + self.print_deployment(deployment), + model, ) return deployment @@ -11452,7 +11480,7 @@ class Router: if deployment is None: verbose_router_logger.info( - f"get_available_deployment_for_pass_through model: {model}, no available deployments" + "get_available_deployment_for_pass_through model: %s, no available deployments", model ) model_ids = self.get_model_ids(model_name=model) _cooldown_time = self.cooldown_cache.get_min_cooldown( @@ -11467,7 +11495,9 @@ class Router: ) verbose_router_logger.info( - f"get_available_deployment_for_pass_through model: {model}, selected deployment: {self.print_deployment(deployment)}" + "get_available_deployment_for_pass_through model: %s, selected deployment: %s", + model, + self.print_deployment(deployment), ) return deployment @@ -11485,7 +11515,7 @@ class Router: List of healthy deployments """ if verbose_router_logger.isEnabledFor(logging.DEBUG): - verbose_router_logger.debug(f"cooldown deployments: {cooldown_deployments}") + verbose_router_logger.debug("cooldown deployments: %s", cooldown_deployments) # Convert to set for O(1) lookup and use list comprehension for O(n) filtering cooldown_set = set(cooldown_deployments) return [deployment for deployment in healthy_deployments if deployment["model_info"]["id"] not in cooldown_set] @@ -11587,7 +11617,7 @@ class Router: List[Dict]: Only includes a list of deployments that support pass-through """ verbose_router_logger.debug( - f"Filter pass-through deployments from {len(healthy_deployments)} healthy deployments" + "Filter pass-through deployments from %s healthy deployments", len(healthy_deployments) ) pass_through_deployments = [ @@ -11596,7 +11626,7 @@ class Router: if deployment.get("litellm_params", {}).get("use_in_pass_through", False) ] - verbose_router_logger.debug(f"Found {len(pass_through_deployments)} deployments with pass-through enabled") + verbose_router_logger.debug("Found %s deployments with pass-through enabled", len(pass_through_deployments)) return pass_through_deployments @@ -11611,7 +11641,7 @@ class Router: if model_id is not None: self._update_usage(model_id, parent_otel_span) # update in-memory cache for tracking except Exception as e: - verbose_router_logger.error(f"Error in _track_deployment_metrics: {e}") + verbose_router_logger.error("Error in _track_deployment_metrics: %s", e) def get_num_retries_from_retry_policy(self, exception: Exception, model_group: str | None = None): return _get_num_retries_from_retry_policy( diff --git a/litellm/router_strategy/auto_router/auto_router.py b/litellm/router_strategy/auto_router/auto_router.py index d959eb3ef73..4ea3389381c 100644 --- a/litellm/router_strategy/auto_router/auto_router.py +++ b/litellm/router_strategy/auto_router/auto_router.py @@ -147,7 +147,7 @@ class AutoRouter(CustomLogger): message_content = self._extract_text_from_messages(messages) route_choice: RouteChoice | list[RouteChoice] | None = routelayer(text=message_content) - verbose_router_logger.debug(f"route_choice: {route_choice}") + verbose_router_logger.debug("route_choice: %s", route_choice) if isinstance(route_choice, RouteChoice): model = route_choice.name or self.default_model elif isinstance(route_choice, list): diff --git a/litellm/router_strategy/base_routing_strategy.py b/litellm/router_strategy/base_routing_strategy.py index 70e1c12665d..9752c96fa5b 100644 --- a/litellm/router_strategy/base_routing_strategy.py +++ b/litellm/router_strategy/base_routing_strategy.py @@ -97,7 +97,7 @@ class BaseRoutingStrategy(ABC): default_sync_interval ) # Wait for DEFAULT_REDIS_SYNC_INTERVAL seconds before next sync except Exception as e: - verbose_router_logger.error(f"Error in periodic sync task: {e}") + verbose_router_logger.error("Error in periodic sync task: %s", e) await asyncio.sleep( default_sync_interval ) # Still wait DEFAULT_REDIS_SYNC_INTERVAL seconds on error before retrying @@ -146,7 +146,7 @@ class BaseRoutingStrategy(ABC): return return_result except Exception as e: - verbose_router_logger.error(f"Error syncing in-memory cache with Redis: {e}") + verbose_router_logger.error("Error syncing in-memory cache with Redis: %s", e) self.redis_increment_operation_queue = [] def add_to_in_memory_keys_to_update(self, key: str): @@ -226,4 +226,4 @@ class BaseRoutingStrategy(ABC): await self.dual_cache.in_memory_cache.async_set_cache(key=key, value=merged) except Exception as e: - verbose_router_logger.exception(f"Error syncing in-memory cache with Redis: {e}") + verbose_router_logger.exception("Error syncing in-memory cache with Redis: %s", e) diff --git a/litellm/router_strategy/budget_limiter.py b/litellm/router_strategy/budget_limiter.py index 3b8a75f4e49..0d8980b6f16 100644 --- a/litellm/router_strategy/budget_limiter.py +++ b/litellm/router_strategy/budget_limiter.py @@ -499,7 +499,7 @@ class RouterBudgetLimiting(CustomLogger): spend_key=spend_key, response_cost=response_cost, ttl=ttl_for_increment ) - verbose_router_logger.debug(f"Incremented spend for {spend_key} by {response_cost}") + verbose_router_logger.debug("Incremented spend for %s by %s", spend_key, response_cost) async def periodic_sync_in_memory_spend_with_redis(self): """ @@ -514,7 +514,7 @@ class RouterBudgetLimiting(CustomLogger): DEFAULT_REDIS_SYNC_INTERVAL ) # Wait for DEFAULT_REDIS_SYNC_INTERVAL seconds before next sync except Exception as e: - verbose_router_logger.error(f"Error in periodic sync task: {e}") + verbose_router_logger.error("Error in periodic sync task: %s", e) await asyncio.sleep( DEFAULT_REDIS_SYNC_INTERVAL ) # Still wait DEFAULT_REDIS_SYNC_INTERVAL seconds on error before retrying @@ -545,7 +545,7 @@ class RouterBudgetLimiting(CustomLogger): self.redis_increment_operation_queue = [] except Exception as e: - verbose_router_logger.error(f"Error syncing in-memory cache with Redis: {e}") + verbose_router_logger.error("Error syncing in-memory cache with Redis: %s", e) async def _sync_in_memory_spend_with_redis(self): """ @@ -597,10 +597,10 @@ class RouterBudgetLimiting(CustomLogger): for key, value in redis_values.items(): if value is not None: await self.dual_cache.in_memory_cache.async_set_cache(key=key, value=float(value)) - verbose_router_logger.debug(f"Updated in-memory cache for {key}: {value}") + verbose_router_logger.debug("Updated in-memory cache for %s: %s", key, value) except Exception as e: - verbose_router_logger.error(f"Error syncing in-memory cache with Redis: {e}") + verbose_router_logger.error("Error syncing in-memory cache with Redis: %s", e) def _get_budget_config_for_deployment( self, @@ -639,7 +639,7 @@ class RouterBudgetLimiting(CustomLogger): litellm_params=provider_resolution_params, ) except Exception: - verbose_router_logger.error(f"Error getting LLM provider for deployment: {deployment}") + verbose_router_logger.error("Error getting LLM provider for deployment: %s", deployment) return None return custom_llm_provider @@ -772,7 +772,7 @@ class RouterBudgetLimiting(CustomLogger): ) ) - verbose_router_logger.debug(f"Initalized Provider budget config: {self.provider_budget_config}") + verbose_router_logger.debug("Initalized Provider budget config: %s", self.provider_budget_config) def _init_deployment_budgets( self, @@ -788,7 +788,10 @@ class RouterBudgetLimiting(CustomLogger): _budget_duration = _litellm_params.get("budget_duration") verbose_router_logger.debug( - f"Init Deployment Budget: max_budget: {_max_budget}, budget_duration: {_budget_duration}, model_id: {_model_id}" + "Init Deployment Budget: max_budget: %s, budget_duration: %s, model_id: %s", + _max_budget, + _budget_duration, + _model_id, ) if _max_budget is not None and _budget_duration is not None and _model_id is not None: _budget_config = GenericBudgetInfo( @@ -799,7 +802,7 @@ class RouterBudgetLimiting(CustomLogger): self.deployment_budget_config = {} self.deployment_budget_config[_model_id] = _budget_config - verbose_router_logger.debug(f"Initialized Deployment Budget Config: {self.deployment_budget_config}") + verbose_router_logger.debug("Initialized Deployment Budget Config: %s", self.deployment_budget_config) def register_deployment_budget( self, @@ -837,4 +840,4 @@ class RouterBudgetLimiting(CustomLogger): ) self.tag_budget_config[_tag] = _generic_budget_config - verbose_router_logger.debug(f"Initialized Tag Budget Config: {self.tag_budget_config}") + verbose_router_logger.debug("Initialized Tag Budget Config: %s", self.tag_budget_config) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 11a1b686c2e..1d8176d1534 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -456,7 +456,7 @@ class ComplexityRouter(CustomLogger): self._model_tiers: dict[str, tuple[ComplexityTier, ...]] = {} self._adaptive_init_attempted = False - verbose_router_logger.debug(f"ComplexityRouter initialized for {model_name} with tiers: {self.config.tiers}") + verbose_router_logger.debug("ComplexityRouter initialized for %s with tiers: %s", model_name, self.config.tiers) def _estimate_tokens(self, text: str) -> int: """ @@ -746,7 +746,7 @@ class ComplexityRouter(CustomLogger): ) except Exception as e: # noqa: BLE001 -- external LLM call can fail in many distinct ways (timeout, provider error, validation, parse error); any failure must fall back to the heuristic scorer verbose_router_logger.warning( - f"ComplexityRouter: LLM classifier failed ({e}), falling back to heuristic scoring" + "ComplexityRouter: LLM classifier failed (%s), falling back to heuristic scoring", e ) tier, score, signals, cause = self._score_and_classify(prompt, system_prompt) return ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause) @@ -1324,7 +1324,7 @@ class ComplexityRouter(CustomLogger): semantic_tier = await self._semantic_tier_override(user_message, request_kwargs) except Exception as e: # noqa: BLE001 -- embedding call can fail many ways (timeout, provider/network/parse error); any failure must fall back to scoring, never fail the request verbose_router_logger.warning( - f"ComplexityRouter: semantic keyword matching failed ({e}), falling back to complexity scoring" + "ComplexityRouter: semantic keyword matching failed (%s), falling back to complexity scoring", e ) return None if semantic_tier is None: @@ -1468,7 +1468,7 @@ class ComplexityRouter(CustomLogger): escalated = routed_model != pinned_model cause: RoutingDecisionCause = "session_affinity_escalation" if escalated else "session_affinity_pin" verbose_router_logger.info( - f"ComplexityRouter: routing decision cause={cause}, routed_model={routed_model}" + "ComplexityRouter: routing decision cause=%s, routed_model=%s", cause, routed_model ) has_original_messages = messages is not None and len(messages) > 0 return PreRoutingHookResponse( @@ -1578,8 +1578,11 @@ class ComplexityRouter(CustomLogger): "semantic_keyword_match" if self.config.semantic_keyword_matching else "literal_keyword_match" ) verbose_router_logger.info( - f"ComplexityRouter: routing decision cause={keyword_cause}, escalated={keyword_escalated}, " - f"tier={routed_tier.value}, routed_model={routed_model}" + "ComplexityRouter: routing decision cause=%s, escalated=%s, tier=%s, routed_model=%s", + keyword_cause, + keyword_escalated, + routed_tier.value, + routed_model, ) return PreRoutingHookResponse( model=routed_model, @@ -1613,15 +1616,22 @@ class ComplexityRouter(CustomLogger): chosen_key = getattr(self, "_adaptive_chosen_model_key", "adaptive_router_chosen_model") kwargs_metadata[chosen_key] = routed_model verbose_router_logger.info( - f"ComplexityRouter[adaptive]: routing decision cause={outcome.cause}, " - f"tier={tier.value}, score={score_repr}, " - f"signals={signals}, routed_model={routed_model}" + "ComplexityRouter[adaptive]: routing decision cause=%s, tier=%s, score=%s, signals=%s, routed_model=%s", + outcome.cause, + tier.value, + score_repr, + signals, + routed_model, ) else: routed_model = await self._pick_model_for_tier(tier, messages, resolved_messages, request_kwargs) verbose_router_logger.info( - f"ComplexityRouter: routing decision cause={outcome.cause}, tier={tier.value}, " - f"score={score_repr}, signals={signals}, routed_model={routed_model}" + "ComplexityRouter: routing decision cause=%s, tier=%s, score=%s, signals=%s, routed_model=%s", + outcome.cause, + tier.value, + score_repr, + signals, + routed_model, ) classifier_model = ( diff --git a/litellm/router_strategy/lar1_routing.py b/litellm/router_strategy/lar1_routing.py index acd7ac63225..99a26926558 100644 --- a/litellm/router_strategy/lar1_routing.py +++ b/litellm/router_strategy/lar1_routing.py @@ -68,12 +68,12 @@ def _normalize_thresholds(thresholds: dict[str, float] | None) -> dict[str, floa def _parse_lar1_metadata(request_kwargs: dict) -> LAR1Metadata: lar1_raw = request_kwargs.get("metadata", {}).get("lar1", {}) if not isinstance(lar1_raw, dict): - verbose_router_logger.warning(f"[LAR-1] Invalid lar1 metadata type: {type(lar1_raw).__name__}. Using defaults") + verbose_router_logger.warning("[LAR-1] Invalid lar1 metadata type: %s. Using defaults", type(lar1_raw).__name__) return LAR1Metadata() try: return LAR1Metadata.model_validate(lar1_raw) except ValidationError as exc: - verbose_router_logger.warning(f"[LAR-1] Invalid lar1 metadata: {exc}. Using defaults") + verbose_router_logger.warning("[LAR-1] Invalid lar1 metadata: %s. Using defaults", exc) return LAR1Metadata() @@ -123,11 +123,11 @@ class LAR1RoutingStrategy(CustomRoutingStrategyBase): if selected is None: return None if exact_match: - verbose_router_logger.info(f"[LAR-1] confidence={confidence} -> {target}") + verbose_router_logger.info("[LAR-1] confidence=%s -> %s", confidence, target) else: actual_type = selected.get("model_info", {}).get("type", "unknown") verbose_router_logger.warning( - f"[LAR-1] No deployment for type '{target}', fallback to deployment type '{actual_type}'" + "[LAR-1] No deployment for type '%s', fallback to deployment type '%s'", target, actual_type ) return selected diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index ba7d32c42ad..7a2970d053a 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -91,7 +91,7 @@ class LowestCostLoggingHandler(CustomLogger): self.logged_success += 1 except Exception as e: verbose_logger.exception( - f"litellm.router_strategy.lowest_cost.py::log_success_event(): Exception occured - {e}" + "litellm.router_strategy.lowest_cost.py::log_success_event(): Exception occured - %s", e ) async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -170,7 +170,7 @@ class LowestCostLoggingHandler(CustomLogger): self.logged_success += 1 except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {e}" + "litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - %s", e ) async def async_get_available_deployments( @@ -269,7 +269,11 @@ class LowestCostLoggingHandler(CustomLogger): item_tpm = item_map.get(precise_minute, {}).get("tpm", 0) verbose_router_logger.debug( - f"item_cost: {item_cost}, item_tpm: {item_tpm}, item_rpm: {item_rpm}, model_id: {_deployment.get('model_info', {}).get('id')}" + "item_cost: %s, item_tpm: %s, item_rpm: %s, model_id: %s", + item_cost, + item_tpm, + item_rpm, + _deployment.get("model_info", {}).get("id"), ) # -------------- # diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index 0adcdebcbf2..294409d042c 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -160,7 +160,7 @@ class LowestLatencyLoggingHandler(CustomLogger): self.logged_success += 1 except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {e}" + "litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - %s", e ) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): @@ -217,7 +217,7 @@ class LowestLatencyLoggingHandler(CustomLogger): return except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {e}" + "litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - %s", e ) async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -350,7 +350,7 @@ class LowestLatencyLoggingHandler(CustomLogger): self.logged_success += 1 except Exception as e: verbose_logger.exception( - f"litellm.router_strategy.lowest_latency.py::async_log_success_event(): Exception occured - {e}" + "litellm.router_strategy.lowest_latency.py::async_log_success_event(): Exception occured - %s", e ) def _get_available_deployments( diff --git a/litellm/router_strategy/lowest_tpm_rpm.py b/litellm/router_strategy/lowest_tpm_rpm.py index f8e7e93eb54..7375658f982 100644 --- a/litellm/router_strategy/lowest_tpm_rpm.py +++ b/litellm/router_strategy/lowest_tpm_rpm.py @@ -73,7 +73,7 @@ class LowestTPMLoggingHandler(CustomLogger): self.logged_success += 1 except Exception as e: verbose_router_logger.error( - f"litellm.router_strategy.lowest_tpm_rpm.py::async_log_success_event(): Exception occured - {e}" + "litellm.router_strategy.lowest_tpm_rpm.py::async_log_success_event(): Exception occured - %s", e ) verbose_router_logger.debug(traceback.format_exc()) @@ -135,7 +135,7 @@ class LowestTPMLoggingHandler(CustomLogger): self.logged_success += 1 except Exception as e: verbose_router_logger.exception( - f"litellm.router_strategy.lowest_tpm_rpm.py::async_log_success_event(): Exception occured - {e}" + "litellm.router_strategy.lowest_tpm_rpm.py::async_log_success_event(): Exception occured - %s", e ) verbose_router_logger.debug(traceback.format_exc()) @@ -151,7 +151,9 @@ class LowestTPMLoggingHandler(CustomLogger): """ # get list of potential deployments verbose_router_logger.debug( - f"get_available_deployments - Usage Based. model_group: {model_group}, healthy_deployments: {healthy_deployments}" + "get_available_deployments - Usage Based. model_group: %s, healthy_deployments: %s", + model_group, + healthy_deployments, ) current_minute = datetime.now().strftime("%H-%M") tpm_key = f"{model_group}:tpm:{current_minute}" @@ -160,12 +162,12 @@ class LowestTPMLoggingHandler(CustomLogger): tpm_dict = self.router_cache.get_cache(key=tpm_key) rpm_dict = self.router_cache.get_cache(key=rpm_key) - verbose_router_logger.debug(f"tpm_key={tpm_key}, tpm_dict: {tpm_dict}, rpm_dict: {rpm_dict}") + verbose_router_logger.debug("tpm_key=%s, tpm_dict: %s, rpm_dict: %s", tpm_key, tpm_dict, rpm_dict) try: input_tokens = token_counter(messages=messages, text=input) except Exception: input_tokens = 0 - verbose_router_logger.debug(f"input_tokens={input_tokens}") + verbose_router_logger.debug("input_tokens=%s", input_tokens) # ----------------------- # Find lowest used model # ---------------------- diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py index a81428fd5fa..1d4703f6026 100644 --- a/litellm/router_strategy/lowest_tpm_rpm_v2.py +++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py @@ -245,7 +245,7 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): self.logged_success += 1 except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.lowest_tpm_rpm_v2.py::log_success_event(): Exception occured - {e}" + "litellm.proxy.hooks.lowest_tpm_rpm_v2.py::log_success_event(): Exception occured - %s", e ) async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -289,7 +289,7 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): self.logged_success += 1 except Exception as e: verbose_logger.exception( - f"litellm.proxy.hooks.lowest_tpm_rpm_v2.py::async_log_success_event(): Exception occured - {e}" + "litellm.proxy.hooks.lowest_tpm_rpm_v2.py::async_log_success_event(): Exception occured - %s", e ) def _return_potential_deployments( @@ -375,7 +375,7 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): input_tokens = token_counter(messages=messages, text=input) except Exception: input_tokens = 0 - verbose_router_logger.debug(f"input_tokens={input_tokens}") + verbose_router_logger.debug("input_tokens=%s", input_tokens) # ----------------------- # Find lowest used model # ---------------------- @@ -420,7 +420,9 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): """ # get list of potential deployments verbose_router_logger.debug( - f"get_available_deployments - Usage Based. model_group: {model_group}, healthy_deployments: {healthy_deployments}" + "get_available_deployments - Usage Based. model_group: %s, healthy_deployments: %s", + model_group, + healthy_deployments, ) dt = get_utc_datetime() @@ -535,7 +537,9 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): """ # get list of potential deployments verbose_router_logger.debug( - f"get_available_deployments - Usage Based. model_group: {model_group}, healthy_deployments: {healthy_deployments}" + "get_available_deployments - Usage Based. model_group: %s, healthy_deployments: %s", + model_group, + healthy_deployments, ) dt = get_utc_datetime() diff --git a/litellm/router_strategy/quality_router/quality_router.py b/litellm/router_strategy/quality_router/quality_router.py index da6825a5741..a84ee70864a 100644 --- a/litellm/router_strategy/quality_router/quality_router.py +++ b/litellm/router_strategy/quality_router/quality_router.py @@ -85,9 +85,10 @@ class QualityRouter(CustomLogger): self._tier_to_models_cache: dict[int, list[str]] | None = None verbose_router_logger.debug( - f"QualityRouter initialized for {model_name} with " - f"available_models={self.config.available_models}, " - f"default_model={self.config.default_model}" + "QualityRouter initialized for %s with available_models=%s, default_model=%s", + model_name, + self.config.available_models, + self.config.default_model, ) @property @@ -371,10 +372,11 @@ class QualityRouter(CustomLogger): if keyword_match is not None: routed_model, matched_keyword = keyword_match verbose_router_logger.info( - f"QualityRouter: keyword override matched='{matched_keyword}' " - f"routed_model={routed_model} " - f"(quality_tier={self._model_quality.get(routed_model)}, " - f"input_cost_per_token={self._model_cost.get(routed_model)})" + "QualityRouter: keyword override matched='%s' routed_model=%s (quality_tier=%s, input_cost_per_token=%s)", + matched_keyword, + routed_model, + self._model_quality.get(routed_model), + self._model_cost.get(routed_model), ) self._stash_decision( request_kwargs, diff --git a/litellm/router_strategy/simple_shuffle.py b/litellm/router_strategy/simple_shuffle.py index ab2fab09a0d..1a2abac4b05 100644 --- a/litellm/router_strategy/simple_shuffle.py +++ b/litellm/router_strategy/simple_shuffle.py @@ -44,7 +44,7 @@ def simple_shuffle( weight = healthy_deployments[0].get("litellm_params").get(weight_by, None) if weight is not None: weights = [m["litellm_params"].get(weight_by, 0) for m in healthy_deployments] - verbose_router_logger.debug(f"\nweight {weights}") + verbose_router_logger.debug("\nweight %s", weights) total_weight = sum(weights) if total_weight <= 0: # All remaining candidates have weight 0 for this metric (e.g. @@ -54,13 +54,16 @@ def simple_shuffle( # through to the uniform random pick at the end. continue weights = [weight / total_weight for weight in weights] - verbose_router_logger.debug(f"\n weights {weights} by {weight_by}") + verbose_router_logger.debug("\n weights %s by %s", weights, weight_by) # Perform weighted random pick selected_index = random.choices(range(len(weights)), weights=weights)[0] - verbose_router_logger.debug(f"\n selected index, {selected_index}") + verbose_router_logger.debug("\n selected index, %s", selected_index) deployment = healthy_deployments[selected_index] verbose_router_logger.info( - f"get_available_deployment for model: {model}, Selected deployment: {llm_router_instance.print_deployment(deployment) or deployment[0]} for model: {model}" + "get_available_deployment for model: %s, Selected deployment: %s for model: %s", + model, + llm_router_instance.print_deployment(deployment) or deployment[0], + model, ) return deployment or deployment[0] diff --git a/litellm/router_utils/batch_utils.py b/litellm/router_utils/batch_utils.py index eefacb7c9e6..c968e8dfde8 100644 --- a/litellm/router_utils/batch_utils.py +++ b/litellm/router_utils/batch_utils.py @@ -52,7 +52,7 @@ def parse_jsonl_with_embedded_newlines(content: str) -> list[dict]: json_object = json.loads(buffer.strip()) json_objects.append(json_object) except json.JSONDecodeError as e: - verbose_logger.error(f"error parsing final buffer: {buffer[:100]}..., error: {e}") + verbose_logger.error("error parsing final buffer: %s..., error: %s", buffer[:100], e) raise e return json_objects @@ -128,7 +128,7 @@ def replace_model_in_jsonl(file_content: FileTypes, new_model_name: str) -> File # that followed it). Returning the partial `output` would silently # drop those rows; return the unchanged original so the provider # rejects the batch loudly instead of accepting a truncated one. - verbose_logger.error(f"error parsing trailing batch content: {buffer[:100]}...") + verbose_logger.error("error parsing trailing batch content: %s...", buffer[:100]) if hasattr(source, "seek"): try: source.seek(0) # type: ignore[attr-defined] diff --git a/litellm/router_utils/cooldown_cache.py b/litellm/router_utils/cooldown_cache.py index 4e9a11a4bfd..4681609b9d7 100644 --- a/litellm/router_utils/cooldown_cache.py +++ b/litellm/router_utils/cooldown_cache.py @@ -58,7 +58,7 @@ class CooldownCache: return cooldown_key, cooldown_data except Exception as e: - verbose_logger.error(f"CooldownCache::_common_add_cooldown_logic - Exception occurred - {e}") + verbose_logger.error("CooldownCache::_common_add_cooldown_logic - Exception occurred - %s", e) raise e def add_deployment_to_cooldown( @@ -92,7 +92,7 @@ class CooldownCache: ttl=_cooldown_time, ) except Exception as e: - verbose_logger.error(f"CooldownCache::add_deployment_to_cooldown - Exception occurred - {e}") + verbose_logger.error("CooldownCache::add_deployment_to_cooldown - Exception occurred - %s", e) raise e @staticmethod diff --git a/litellm/router_utils/cooldown_callbacks.py b/litellm/router_utils/cooldown_callbacks.py index acd1c5b47ad..94e2847121b 100644 --- a/litellm/router_utils/cooldown_callbacks.py +++ b/litellm/router_utils/cooldown_callbacks.py @@ -34,7 +34,8 @@ async def router_cooldown_event_callback( _deployment = litellm_router_instance.get_deployment(model_id=deployment_id) if _deployment is None: verbose_logger.warning( - f"in router_cooldown_event_callback but _deployment is None for deployment_id={deployment_id}. Doing nothing" + "in router_cooldown_event_callback but _deployment is None for deployment_id=%s. Doing nothing", + deployment_id, ) return _litellm_params = _deployment["litellm_params"] diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 380689e653b..ca5fe198abb 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -281,7 +281,7 @@ def _set_cooldown_deployments( return False exception_status_int = cast_exception_status_to_int(exception_status) - verbose_router_logger.debug(f"Attempting to add {deployment} to cooldown list") + verbose_router_logger.debug("Attempting to add %s to cooldown list", deployment) if _should_cooldown_deployment( litellm_router_instance=litellm_router_instance, @@ -331,7 +331,7 @@ async def _async_get_cooldown_deployments( ): cached_value_deployment_ids = [cv[0] for cv in cooldown_models] - verbose_router_logger.debug(f"retrieve cooldown models: {cooldown_models}") + verbose_router_logger.debug("retrieve cooldown models: %s", cooldown_models) return cached_value_deployment_ids @@ -347,7 +347,7 @@ async def _async_get_cooldown_deployments_with_debug_info( model_ids=model_ids, parent_otel_span=parent_otel_span ) - verbose_router_logger.debug(f"retrieve cooldown models: {cooldown_models}") + verbose_router_logger.debug("retrieve cooldown models: %s", cooldown_models) return cooldown_models @@ -432,7 +432,7 @@ def cast_exception_status_to_int(exception_status: str | int) -> int: exception_status = int(exception_status) except Exception: verbose_router_logger.debug( - f"Unable to cast exception status to int {exception_status}. Defaulting to status=500." + "Unable to cast exception status to int %s. Defaulting to status=500.", exception_status ) exception_status = 500 return exception_status diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 3fad860fa7d..e2fcd9109d2 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -127,7 +127,7 @@ async def run_async_fallback( try: # LOGGING kwargs = litellm_router.log_retry(kwargs=kwargs, e=original_exception) - verbose_router_logger.info(f"Falling back to model_group = {mask_sensitive_structure(mg)}") + verbose_router_logger.info("Falling back to model_group = %s", mask_sensitive_structure(mg)) if isinstance(mg, str): kwargs["model"] = mg elif isinstance(mg, dict): @@ -190,7 +190,7 @@ async def log_success_fallback_event(original_model_group: str, kwargs: dict, or original_exception=original_exception, ) except Exception as e: - verbose_router_logger.error(f"Error in log_success_fallback_event: {e}") + verbose_router_logger.error("Error in log_success_fallback_event: %s", e) async def log_failure_fallback_event(original_model_group: str, kwargs: dict, original_exception: Exception): @@ -218,7 +218,7 @@ async def log_failure_fallback_event(original_model_group: str, kwargs: dict, or original_exception=original_exception, ) except Exception as e: - verbose_router_logger.error(f"Error in log_failure_fallback_event: {e}") + verbose_router_logger.error("Error in log_failure_fallback_event: %s", e) def _check_non_standard_fallback_format(fallbacks: list[Any] | None) -> bool: diff --git a/litellm/router_utils/handle_error.py b/litellm/router_utils/handle_error.py index b38d6605ed2..c25238a89b3 100644 --- a/litellm/router_utils/handle_error.py +++ b/litellm/router_utils/handle_error.py @@ -74,7 +74,7 @@ async def async_raise_no_deployment_exception( """ Raises a RouterRateLimitError if no deployment is found for the given model. """ - verbose_router_logger.info(f"get_available_deployment for model: {model}, No deployment available") + verbose_router_logger.info("get_available_deployment for model: %s, No deployment available", model) model_ids = litellm_router_instance.get_model_ids(model_name=model) _cooldown_time = litellm_router_instance.cooldown_cache.get_min_cooldown( model_ids=model_ids, parent_otel_span=parent_otel_span @@ -84,7 +84,7 @@ async def async_raise_no_deployment_exception( parent_otel_span=parent_otel_span, ) verbose_router_logger.info( - f"No deployment found for model: {model}, cooldown_list with debug info: {_cooldown_list}" + "No deployment found for model: %s, cooldown_list with debug info: %s", model, _cooldown_list ) cooldown_list_ids = [cooldown_model[0] for cooldown_model in (_cooldown_list or [])] diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index 42704cea826..c8161529626 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -150,7 +150,7 @@ class PatternMatchRouter: matched_pattern=pattern_match, deployments=llm_deployments ) except Exception as e: - verbose_router_logger.debug(f"Error in PatternMatchRouter.route: {e}") + verbose_router_logger.debug("Error in PatternMatchRouter.route: %s", e) return None # No matching pattern found diff --git a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py index 9561eafa900..4dd6944d791 100644 --- a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py @@ -551,16 +551,20 @@ def io_token_reconcile_success( ) else: verbose_router_logger.debug( - "[IO TOKEN LIMIT] usage missing; keeping reservation " - f"(itpm_reserved={itpm_reserved}, otpm_reserved={otpm_reserved})" + "[IO TOKEN LIMIT] usage missing; keeping reservation (itpm_reserved=%s, otpm_reserved=%s)", + itpm_reserved, + otpm_reserved, ) finally: _clear_reservation_from_kwargs(kwargs) verbose_router_logger.debug( - f"[IO TOKEN LIMIT] reconciled " - f"(usage_resolved={usage_resolved}, itpm_reserved={itpm_reserved}, " - f"billable_input={billable_input}, otpm_reserved={otpm_reserved}, output={completion_tokens})" + "[IO TOKEN LIMIT] reconciled (usage_resolved=%s, itpm_reserved=%s, billable_input=%s, otpm_reserved=%s, output=%s)", + usage_resolved, + itpm_reserved, + billable_input, + otpm_reserved, + completion_tokens, ) @@ -606,16 +610,20 @@ async def async_io_token_reconcile_success( ) else: verbose_router_logger.debug( - "[IO TOKEN LIMIT] usage missing; keeping reservation " - f"(itpm_reserved={itpm_reserved}, otpm_reserved={otpm_reserved})" + "[IO TOKEN LIMIT] usage missing; keeping reservation (itpm_reserved=%s, otpm_reserved=%s)", + itpm_reserved, + otpm_reserved, ) finally: _clear_reservation_from_kwargs(kwargs) verbose_router_logger.debug( - f"[IO TOKEN LIMIT] reconciled " - f"(usage_resolved={usage_resolved}, itpm_reserved={itpm_reserved}, " - f"billable_input={billable_input}, otpm_reserved={otpm_reserved}, output={completion_tokens})" + "[IO TOKEN LIMIT] reconciled (usage_resolved=%s, itpm_reserved=%s, billable_input=%s, otpm_reserved=%s, output=%s)", + usage_resolved, + itpm_reserved, + billable_input, + otpm_reserved, + completion_tokens, ) @@ -639,7 +647,7 @@ def io_token_refund_failure( ttl=RoutingArgsTTL, ) _clear_reservation_from_kwargs(kwargs) - verbose_router_logger.debug(f"[IO TOKEN LIMIT] refunded ITPM={itpm_reserved} OTPM={otpm_reserved}") + verbose_router_logger.debug("[IO TOKEN LIMIT] refunded ITPM=%s OTPM=%s", itpm_reserved, otpm_reserved) def refund_stale_reservation_before_retry(dual_cache: DualCache, kwargs: dict[str, Any] | None) -> None: @@ -693,7 +701,7 @@ async def async_io_token_refund_failure( parent_otel_span=parent_otel_span, ) _clear_reservation_from_kwargs(kwargs) - verbose_router_logger.debug(f"[IO TOKEN LIMIT] refunded ITPM={itpm_reserved} OTPM={otpm_reserved}") + verbose_router_logger.debug("[IO TOKEN LIMIT] refunded ITPM=%s OTPM=%s", itpm_reserved, otpm_reserved) def build_io_token_rate_limit_headers( diff --git a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py index da8b452fa8a..416b53f3267 100644 --- a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py @@ -75,8 +75,8 @@ class ModelRateLimitingCheck(CustomLogger): return self._io_token_conflict_warned_ids.add(str(model_id)) verbose_router_logger.warning( - f"Deployment '{model_id}' configures itpm/otpm alongside tpm/rpm; " - "both limit types are enforced on this deployment" + "Deployment '%s' configures itpm/otpm alongside tpm/rpm; both limit types are enforced on this deployment", + model_id, ) def _refund_io_token_reservation_if_any(self) -> None: @@ -212,7 +212,7 @@ class ModelRateLimitingCheck(CustomLogger): self._refund_io_token_reservation_if_any() raise except Exception as e: - verbose_router_logger.debug(f"Error in ModelRateLimitingCheck.pre_call_check: {e}") + verbose_router_logger.debug("Error in ModelRateLimitingCheck.pre_call_check: %s", e) # Don't fail the request if rate limit check fails return deployment @@ -300,7 +300,7 @@ class ModelRateLimitingCheck(CustomLogger): await self._async_refund_io_token_reservation_if_any(parent_otel_span=parent_otel_span) raise except Exception as e: - verbose_router_logger.debug(f"Error in ModelRateLimitingCheck.async_pre_call_check: {e}") + verbose_router_logger.debug("Error in ModelRateLimitingCheck.async_pre_call_check: %s", e) # Don't fail the request if rate limit check fails return deployment @@ -341,7 +341,7 @@ class ModelRateLimitingCheck(CustomLogger): model = standard_logging_object.get("hidden_params", {}).get("litellm_model_name") verbose_router_logger.debug( - f"[TPM TRACKING] model_id={model_id}, total_tokens={total_tokens}, model={model}" + "[TPM TRACKING] model_id=%s, total_tokens=%s, model=%s", model_id, total_tokens, model ) if not model or not total_tokens: @@ -351,7 +351,7 @@ class ModelRateLimitingCheck(CustomLogger): current_minute = dt.strftime("%H-%M") tpm_key = f"{model_id}:{model}:tpm:{current_minute}" - verbose_router_logger.debug(f"[TPM TRACKING] Incrementing {tpm_key} by {total_tokens}") + verbose_router_logger.debug("[TPM TRACKING] Incrementing %s by %s", tpm_key, total_tokens) await self.dual_cache.async_increment_cache( key=tpm_key, @@ -360,7 +360,7 @@ class ModelRateLimitingCheck(CustomLogger): ) except Exception as e: - verbose_router_logger.debug(f"Error in ModelRateLimitingCheck.async_log_success_event: {e}") + verbose_router_logger.debug("Error in ModelRateLimitingCheck.async_log_success_event: %s", e) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): from litellm.litellm_core_utils.core_helpers import ( @@ -418,7 +418,7 @@ class ModelRateLimitingCheck(CustomLogger): ) except Exception as e: - verbose_router_logger.debug(f"Error in ModelRateLimitingCheck.log_success_event: {e}") + verbose_router_logger.debug("Error in ModelRateLimitingCheck.log_success_event: %s", e) def log_failure_event(self, kwargs, response_obj, start_time, end_time): with contextlib.suppress(Exception): diff --git a/litellm/router_utils/search_api_router.py b/litellm/router_utils/search_api_router.py index 0ce0d4229c1..5708b772970 100644 --- a/litellm/router_utils/search_api_router.py +++ b/litellm/router_utils/search_api_router.py @@ -57,7 +57,7 @@ class SearchAPIRouter: try: from litellm.types.router import SearchToolTypedDict - verbose_router_logger.debug(f"Adding {len(search_tools)} search tools to router") + verbose_router_logger.debug("Adding %s search tools to router", len(search_tools)) # Convert search tools to the format expected by the router router_search_tools: list = [] @@ -74,10 +74,10 @@ class SearchAPIRouter: # Update the router's search_tools list router_instance.search_tools = router_search_tools - verbose_router_logger.info(f"Successfully updated router with {len(router_search_tools)} search tool(s)") + verbose_router_logger.info("Successfully updated router with %s search tool(s)", len(router_search_tools)) except Exception as e: - verbose_router_logger.exception(f"Error updating router with search tools: {e}") + verbose_router_logger.exception("Error updating router with search tools: %s", e) raise e @staticmethod @@ -149,7 +149,10 @@ class SearchAPIRouter: available_search_tool_names = [tool.get("search_tool_name") for tool in router_instance.search_tools] verbose_router_logger.debug( - f"Inside SearchAPIRouter.async_search_with_fallbacks() - search_tool_name: {search_tool_name}, Available Search Tools: {available_search_tool_names}, kwargs: {kwargs}" + "Inside SearchAPIRouter.async_search_with_fallbacks() - search_tool_name: %s, Available Search Tools: %s, kwargs: %s", + search_tool_name, + available_search_tool_names, + kwargs, ) # Use the existing retry/fallback infrastructure @@ -212,7 +215,7 @@ class SearchAPIRouter: tool_litellm_params=litellm_params, ) - verbose_router_logger.debug(f"Selected search tool with provider: {search_provider}") + verbose_router_logger.debug("Selected search tool with provider: %s", search_provider) # Call the original search function with the provider config response = await original_generic_function( @@ -226,6 +229,6 @@ class SearchAPIRouter: except Exception as e: verbose_router_logger.error( - f"Error in SearchAPIRouter.async_search_with_fallbacks_helper for {search_tool_name}: {e}" + "Error in SearchAPIRouter.async_search_with_fallbacks_helper for %s: %s", search_tool_name, e ) raise e diff --git a/litellm/sandbox/main.py b/litellm/sandbox/main.py index eae7e5c097a..3210e327fef 100644 --- a/litellm/sandbox/main.py +++ b/litellm/sandbox/main.py @@ -141,4 +141,4 @@ async def acode_interpreter_tool( try: await config.adelete_sandbox(container=container, api_key=api_key, api_base=api_base, **forwarded) except Exception as e: - litellm._logging.verbose_logger.debug(f"sandbox: failed to delete ephemeral container: {e}") + litellm._logging.verbose_logger.debug("sandbox: failed to delete ephemeral container: %s", e) diff --git a/litellm/search/main.py b/litellm/search/main.py index 932a73c0955..af9b1f7a745 100644 --- a/litellm/search/main.py +++ b/litellm/search/main.py @@ -247,7 +247,7 @@ def search( if search_provider_config is None: raise ValueError(f"Search is not supported for provider: {search_provider}") - verbose_logger.debug(f"Search call - provider: {search_provider}") + verbose_logger.debug("Search call - provider: %s", search_provider) # Build optional_params from explicit parameters optional_params = _build_search_optional_params( @@ -265,7 +265,7 @@ def search( if key not in optional_params: optional_params[key] = value - verbose_logger.debug(f"Search optional_params: {optional_params}") + verbose_logger.debug("Search optional_params: %s", optional_params) # Validate environment and get headers headers = search_provider_config.validate_environment( diff --git a/litellm/secret_managers/cyberark_secret_manager.py b/litellm/secret_managers/cyberark_secret_manager.py index 6e7eb742088..0e8aa2a5b3a 100644 --- a/litellm/secret_managers/cyberark_secret_manager.py +++ b/litellm/secret_managers/cyberark_secret_manager.py @@ -143,17 +143,17 @@ class CyberArkSecretManager(BaseSecretManager): content=policy_yaml, ) resp.raise_for_status() - verbose_logger.debug(f"Created policy entry for variable: {secret_name}") + verbose_logger.debug("Created policy entry for variable: %s", secret_name) except httpx.HTTPStatusError as e: # Variable might already exist, which is fine if e.response.status_code in [409, 422]: - verbose_logger.debug(f"Variable {secret_name} already exists or policy conflict (expected)") + verbose_logger.debug("Variable %s already exists or policy conflict (expected)", secret_name) else: verbose_logger.warning( - f"Could not ensure variable exists: {e.response.status_code} - {e.response.text}" + "Could not ensure variable exists: %s - %s", e.response.status_code, e.response.text ) except Exception as e: - verbose_logger.warning(f"Error ensuring variable exists: {e}") + verbose_logger.warning("Error ensuring variable exists: %s", e) def get_url(self, secret_name: str) -> str: """ @@ -207,12 +207,12 @@ class CyberArkSecretManager(BaseSecretManager): except httpx.HTTPStatusError as e: if e.response.status_code == 404: - verbose_logger.debug(f"Secret {secret_name} not found in CyberArk Conjur") + verbose_logger.debug("Secret %s not found in CyberArk Conjur", secret_name) else: - verbose_logger.exception(f"Error reading secret from CyberArk Conjur: {e}") + verbose_logger.exception("Error reading secret from CyberArk Conjur: %s", e) return None except Exception as e: - verbose_logger.exception(f"Error reading secret from CyberArk Conjur: {e}") + verbose_logger.exception("Error reading secret from CyberArk Conjur: %s", e) return None def sync_read_secret( @@ -250,12 +250,12 @@ class CyberArkSecretManager(BaseSecretManager): except httpx.HTTPStatusError as e: if e.response.status_code == 404: - verbose_logger.debug(f"Secret {secret_name} not found in CyberArk Conjur") + verbose_logger.debug("Secret %s not found in CyberArk Conjur", secret_name) else: - verbose_logger.exception(f"Error reading secret from CyberArk Conjur: {e}") + verbose_logger.exception("Error reading secret from CyberArk Conjur: %s", e) return None except Exception as e: - verbose_logger.exception(f"Error reading secret from CyberArk Conjur: {e}") + verbose_logger.exception("Error reading secret from CyberArk Conjur: %s", e) return None async def async_write_secret( @@ -303,7 +303,7 @@ class CyberArkSecretManager(BaseSecretManager): "message": f"Secret {secret_name} written successfully", } except Exception as e: - verbose_logger.exception(f"Error writing secret to CyberArk Conjur: {e}") + verbose_logger.exception("Error writing secret to CyberArk Conjur: %s", e) return {"status": "error", "message": str(e)} async def async_delete_secret( diff --git a/litellm/secret_managers/get_azure_ad_token_provider.py b/litellm/secret_managers/get_azure_ad_token_provider.py index ed348865859..fc3841008f3 100644 --- a/litellm/secret_managers/get_azure_ad_token_provider.py +++ b/litellm/secret_managers/get_azure_ad_token_provider.py @@ -69,7 +69,7 @@ def get_azure_ad_token_provider( if azure_credential else None or os.environ.get("AZURE_CREDENTIAL") or infer_credential_type_from_environment() ) - verbose_logger.info(f"For Azure AD Token Provider, choosing credential type: {cred}") + verbose_logger.info("For Azure AD Token Provider, choosing credential type: %s", cred) credential: ( ClientSecretCredential | ManagedIdentityCredential | CertificateCredential | DefaultAzureCredential | Any | None ) = None diff --git a/litellm/secret_managers/hashicorp_secret_manager.py b/litellm/secret_managers/hashicorp_secret_manager.py index 3f15a4fe5f5..12dae2af706 100644 --- a/litellm/secret_managers/hashicorp_secret_manager.py +++ b/litellm/secret_managers/hashicorp_secret_manager.py @@ -135,7 +135,7 @@ class HashicorpSecretManager(BaseSecretManager): _lease_duration = auth_data["lease_duration"] verbose_logger.debug( - f"Successfully obtained Vault token via AppRole auth. Lease duration: {_lease_duration}s" + "Successfully obtained Vault token via AppRole auth. Lease duration: %ss", _lease_duration ) # Cache the token with its lease duration @@ -337,7 +337,7 @@ class HashicorpSecretManager(BaseSecretManager): return _value except Exception as e: - verbose_logger.exception(f"Error reading secret from Hashicorp Vault: {e}") + verbose_logger.exception("Error reading secret from Hashicorp Vault: %s", e) return None def sync_read_secret( @@ -368,7 +368,7 @@ class HashicorpSecretManager(BaseSecretManager): return _value except Exception as e: - verbose_logger.exception(f"Error reading secret from Hashicorp Vault: {e}") + verbose_logger.exception("Error reading secret from Hashicorp Vault: %s", e) return None async def async_write_secret( @@ -415,7 +415,7 @@ class HashicorpSecretManager(BaseSecretManager): response.raise_for_status() return response.json() except Exception as e: - verbose_logger.exception(f"Error writing secret to Hashicorp Vault: {e}") + verbose_logger.exception("Error writing secret to Hashicorp Vault: %s", e) return {"status": "error", "message": str(e)} async def async_rotate_secret( @@ -459,20 +459,20 @@ class HashicorpSecretManager(BaseSecretManager): # Secret exists, we can proceed except httpx.HTTPStatusError as e: if e.response.status_code == 404: - verbose_logger.exception(f"Current secret {current_secret_name} not found") + verbose_logger.exception("Current secret %s not found", current_secret_name) return { "status": "error", "message": f"Current secret {current_secret_name} not found", } verbose_logger.exception( - f"Error checking current secret existence: {e.response.text if hasattr(e, 'response') else str(e)}" + "Error checking current secret existence: %s", e.response.text if hasattr(e, "response") else str(e) ) return { "status": "error", "message": f"HTTP error occurred while checking current secret: {e.response.text if hasattr(e, 'response') else str(e)}", } except Exception as e: - verbose_logger.exception(f"Error checking current secret existence: {e}") + verbose_logger.exception("Error checking current secret existence: %s", e) return { "status": "error", "message": f"Error checking current secret: {e}", @@ -506,7 +506,9 @@ class HashicorpSecretManager(BaseSecretManager): new_secret_value_from_vault = json_resp.get("data", {}).get("data", {}).get(data_key, None) if new_secret_value_from_vault != new_secret_value: verbose_logger.exception( - f"New secret value mismatch. Expected: {new_secret_value}, Got: {new_secret_value_from_vault}" + "New secret value mismatch. Expected: %s, Got: %s", + new_secret_value, + new_secret_value_from_vault, ) return { "status": "error", @@ -514,20 +516,20 @@ class HashicorpSecretManager(BaseSecretManager): } except httpx.HTTPStatusError as e: if e.response.status_code == 404: - verbose_logger.exception(f"Failed to verify new secret {new_secret_name}") + verbose_logger.exception("Failed to verify new secret %s", new_secret_name) return { "status": "error", "message": f"Failed to verify new secret {new_secret_name}", } verbose_logger.exception( - f"Error verifying new secret: {e.response.text if hasattr(e, 'response') else str(e)}" + "Error verifying new secret: %s", e.response.text if hasattr(e, "response") else str(e) ) return { "status": "error", "message": f"HTTP error occurred while verifying new secret: {e.response.text if hasattr(e, 'response') else str(e)}", } except Exception as e: - verbose_logger.exception(f"Error verifying new secret: {e}") + verbose_logger.exception("Error verifying new secret: %s", e) return { "status": "error", "message": f"Error verifying new secret: {e}", @@ -546,7 +548,9 @@ class HashicorpSecretManager(BaseSecretManager): if isinstance(delete_response, dict) and delete_response.get("status") == "error": # Log the error but don't fail the rotation since new secret was created successfully verbose_logger.warning( - f"Failed to delete old secret {current_secret_name} after rotation: {delete_response.get('message')}" + "Failed to delete old secret %s after rotation: %s", + current_secret_name, + delete_response.get("message"), ) else: # Clear cache for the old secret only if deletion was successful @@ -561,7 +565,7 @@ class HashicorpSecretManager(BaseSecretManager): verbose_logger.exception("Timeout error occurred during secret rotation") return {"status": "error", "message": "Timeout error occurred"} except Exception as e: - verbose_logger.exception(f"Error rotating secret in Hashicorp Vault: {e}") + verbose_logger.exception("Error rotating secret in Hashicorp Vault: %s", e) return {"status": "error", "message": str(e)} async def async_delete_secret( @@ -604,7 +608,7 @@ class HashicorpSecretManager(BaseSecretManager): "message": f"Secret {target['secret_name']} deleted successfully", } except Exception as e: - verbose_logger.exception(f"Error deleting secret from Hashicorp Vault: {e}") + verbose_logger.exception("Error deleting secret from Hashicorp Vault: %s", e) return {"status": "error", "message": str(e)} def _get_secret_value_from_json_response(self, json_resp: dict | None) -> str | None: diff --git a/litellm/secret_managers/main.py b/litellm/secret_managers/main.py index 2982d30274b..a3094bc06a1 100644 --- a/litellm/secret_managers/main.py +++ b/litellm/secret_managers/main.py @@ -335,7 +335,10 @@ def get_secret( ) except Exception as e: # check if it's in os.environ verbose_logger.error( - f"Defaulting to os.environ value for key={secret_name}. An exception occurred - {e}.\n\n{traceback.format_exc()}" + "Defaulting to os.environ value for key=%s. An exception occurred - %s.\n\n%s", + secret_name, + e, + traceback.format_exc(), ) secret = os.getenv(secret_name) try: diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/zscaler_ai_guard.py b/litellm/types/proxy/guardrails/guardrail_hooks/zscaler_ai_guard.py index f522f5b470a..c4eb5c39f1c 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/zscaler_ai_guard.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/zscaler_ai_guard.py @@ -102,7 +102,7 @@ class ZscalerAIGuardConfigModel(GuardrailConfigModel): policy_id = int(env_policy) except ValueError: verbose_proxy_logger.warning( - f"ZSCALER_AI_GUARD_POLICY_ID env var is not a valid integer: {env_policy}" + "ZSCALER_AI_GUARD_POLICY_ID env var is not a valid integer: %s", env_policy ) # Check for configuration issues diff --git a/litellm/types/videos/utils.py b/litellm/types/videos/utils.py index f1b20618a74..1b701260f2f 100644 --- a/litellm/types/videos/utils.py +++ b/litellm/types/videos/utils.py @@ -105,7 +105,7 @@ def decode_video_id_with_provider(encoded_video_id: str) -> DecodedVideoId: video_id=decoded_video_id, ) except Exception as e: - verbose_logger.debug(f"Error decoding video_id '{encoded_video_id}': {e}") + verbose_logger.debug("Error decoding video_id '%s': %s", encoded_video_id, e) return DecodedVideoId( custom_llm_provider=None, model_id=None, @@ -182,7 +182,7 @@ def decode_character_id_with_provider(encoded_character_id: str) -> DecodedChara character_id=decoded_character_id, ) except Exception as e: - verbose_logger.debug(f"Error decoding character_id '{encoded_character_id}': {e}") + verbose_logger.debug("Error decoding character_id '%s': %s", encoded_character_id, e) return DecodedCharacterId( custom_llm_provider=None, model_id=None, diff --git a/litellm/utils.py b/litellm/utils.py index eb3e578b7e8..5dbff5070aa 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -513,7 +513,9 @@ def _add_custom_logger_callback_to_specific_event(callback: str, logging_event: if callback not in litellm._known_custom_logger_compatible_callbacks: verbose_logger.debug( - f"Callback {callback} is not a valid custom logger compatible callback. Known list - {litellm._known_custom_logger_compatible_callbacks}" + "Callback %s is not a valid custom logger compatible callback. Known list - %s", + callback, + litellm._known_custom_logger_compatible_callbacks, ) return @@ -947,7 +949,7 @@ def function_setup( except Exception as e: # Log the error but don't fail the request - verbose_logger.warning(f"Error removing thought signatures from tool call IDs: {e}") + verbose_logger.warning("Error removing thought signatures from tool call IDs: %s", e) elif call_type == CallTypes.embedding.value or call_type == CallTypes.aembedding.value: messages = args[1] if len(args) > 1 else kwargs.get("input", None) elif call_type == CallTypes.image_generation.value or call_type == CallTypes.aimage_generation.value: @@ -1004,7 +1006,7 @@ def function_setup( else: messages = "default-message-value" except Exception as e: - verbose_logger.debug(f"Error extracting messages from Google contents: {e}") + verbose_logger.debug("Error extracting messages from Google contents: %s", e) messages = "default-message-value" else: messages = "default-message-value" @@ -1951,7 +1953,7 @@ def _select_tokenizer_helper(model: str) -> SelectTokenizerResponse: if result is not None: return result except Exception as e: - verbose_logger.debug(f"Error selecting tokenizer: {e}") + verbose_logger.debug("Error selecting tokenizer: %s", e) # default - tiktoken return _return_openai_tokenizer(model) @@ -2064,7 +2066,7 @@ def create_pretrained_tokenizer(identifier: str, revision="main", auth_token: st auth_token=auth_token, # type: ignore ) except Exception as e: - verbose_logger.error(f"Error creating pretrained tokenizer: {e}. Defaulting to version without 'auth_token'.") + verbose_logger.error("Error creating pretrained tokenizer: %s. Defaulting to version without 'auth_token'.", e) tokenizer = Tokenizer.from_pretrained(identifier, revision=revision) return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} @@ -2224,7 +2226,10 @@ def supports_native_streaming(model: str, custom_llm_provider: str | None) -> bo return supports_native_streaming except Exception as e: verbose_logger.debug( - f"Model not found or error in checking supports_native_streaming support. You passed model={model}, custom_llm_provider={custom_llm_provider}. Error: {e}" + "Model not found or error in checking supports_native_streaming support. You passed model=%s, custom_llm_provider=%s. Error: %s", + model, + custom_llm_provider, + e, ) return False @@ -2248,7 +2253,10 @@ def supports_response_schema(model: str, custom_llm_provider: str | None = None) model, custom_llm_provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) except Exception as e: verbose_logger.debug( - f"Model not found or error in checking response schema support. You passed model={model}, custom_llm_provider={custom_llm_provider}. Error: {e}" + "Model not found or error in checking response schema support. You passed model=%s, custom_llm_provider=%s. Error: %s", + model, + custom_llm_provider, + e, ) return False @@ -2362,7 +2370,11 @@ def _supports_factory(model: str, custom_llm_provider: str | None, key: str) -> return False except Exception as e: verbose_logger.debug( - f"Model not found or error in checking {key} support. You passed model={model}, custom_llm_provider={custom_llm_provider}. Error: {e}" + "Model not found or error in checking %s support. You passed model=%s, custom_llm_provider=%s. Error: %s", + key, + model, + custom_llm_provider, + e, ) supported_by_provider = _supports_provider_info_factory(model, custom_llm_provider, key) @@ -2402,9 +2414,11 @@ def _is_explicitly_disabled_factory(model: str, custom_llm_provider: str | None, return False except Exception as e: verbose_logger.debug( - f"Model not found or error in checking {key} disabled state. " - f"You passed model={model}, custom_llm_provider={custom_llm_provider}. " - f"Error: {e}" + "Model not found or error in checking %s disabled state. You passed model=%s, custom_llm_provider=%s. Error: %s", + key, + model, + custom_llm_provider, + e, ) return False @@ -2537,7 +2551,10 @@ def get_supported_regions(model: str, custom_llm_provider: str | None = None) -> return None except Exception as e: verbose_logger.debug( - f"Model not found or error in checking supported_regions support. You passed model={model}, custom_llm_provider={custom_llm_provider}. Error: {e}" + "Model not found or error in checking supported_regions support. You passed model=%s, custom_llm_provider=%s. Error: %s", + model, + custom_llm_provider, + e, ) return None @@ -2720,10 +2737,8 @@ def register_model(model_cost: str | dict): and value.get("cache_read_input_token_cost") is None ): verbose_logger.warning( - f"register_model: model={key} not in built-in cost map and no " - "prefix/region variant matched; cache cost fields will default " - "to 0. To track cache cost, add cache_creation_input_token_cost " - "and cache_read_input_token_cost to model_info" + "register_model: model=%s not in built-in cost map and no prefix/region variant matched; cache cost fields will default to 0. To track cache cost, add cache_creation_input_token_cost and cache_read_input_token_cost to model_info", + key, ) # ``get_model_info`` returns ``litellm_provider: None`` when the # provider is unknown (e.g. custom deployments registered via @@ -2754,7 +2769,7 @@ def register_model(model_cost: str | dict): # Invalidate case-insensitive lookup map since model_cost was modified _invalidate_model_cost_lowercase_map() - verbose_logger.debug(f"added/updated model={model_cost_key} in litellm.model_cost: {model_cost_key}") + verbose_logger.debug("added/updated model=%s in litellm.model_cost: %s", model_cost_key, model_cost_key) # add new model names to provider lists if value.get("litellm_provider") == "openai": if key not in litellm.open_ai_chat_completion_models: @@ -3828,9 +3843,9 @@ def get_optional_params( Args: supported_params: List[str] - supported params from the litellm config """ - verbose_logger.info(f"\nLiteLLM completion() model= {model}; provider = {custom_llm_provider}") - verbose_logger.debug(f"\nLiteLLM: Params passed to completion() {passed_params}") - verbose_logger.debug(f"\nLiteLLM: Non-Default params passed to completion() {non_default_params}") + verbose_logger.info("\nLiteLLM completion() model= %s; provider = %s", model, custom_llm_provider) + verbose_logger.debug("\nLiteLLM: Params passed to completion() %s", passed_params) + verbose_logger.debug("\nLiteLLM: Non-Default params passed to completion() %s", non_default_params) unsupported_params = {} for k in non_default_params.keys(): if k not in supported_params: @@ -4571,7 +4586,7 @@ def _infer_model_region(litellm_params: LiteLLM_Params) -> AllowedModelRegion | model_region = _get_model_region(custom_llm_provider=custom_llm_provider, litellm_params=litellm_params) if model_region is None: - verbose_logger.debug(f"Cannot infer model region for model: {litellm_params.model}") + verbose_logger.debug("Cannot infer model region for model: %s", litellm_params.model) return None if custom_llm_provider == "azure": @@ -5238,7 +5253,7 @@ def _get_model_info_helper( ########################## potential_model_names = _get_potential_model_names(model=model, custom_llm_provider=custom_llm_provider) - verbose_logger.debug(f"checking potential_model_names in litellm.model_cost: {potential_model_names}") + verbose_logger.debug("checking potential_model_names in litellm.model_cost: %s", potential_model_names) combined_model_name = potential_model_names["combined_model_name"] stripped_model_name = potential_model_names["stripped_model_name"] @@ -5373,7 +5388,9 @@ def _get_model_info_helper( if _input_cost_per_token is None: # default value to 0, be noisy about this verbose_logger.debug( - f"model={model}, custom_llm_provider={custom_llm_provider} has no input_cost_per_token in model_cost_map. Defaulting to 0." + "model=%s, custom_llm_provider=%s has no input_cost_per_token in model_cost_map. Defaulting to 0.", + model, + custom_llm_provider, ) _input_cost_per_token = 0 @@ -5381,7 +5398,9 @@ def _get_model_info_helper( if _output_cost_per_token is None: # default value to 0, be noisy about this verbose_logger.debug( - f"model={model}, custom_llm_provider={custom_llm_provider} has no output_cost_per_token in model_cost_map. Defaulting to 0." + "model=%s, custom_llm_provider=%s has no output_cost_per_token in model_cost_map. Defaulting to 0.", + model, + custom_llm_provider, ) _output_cost_per_token = 0 @@ -5548,7 +5567,7 @@ def _get_model_info_helper( returned_model_info[cost_key] = cost_value # type: ignore[literal-required] return returned_model_info except Exception as e: - verbose_logger.debug(f"Error getting model info: {e}") + verbose_logger.debug("Error getting model info: %s", e) raise Exception( f"This model isn't mapped yet. model={model}, custom_llm_provider={custom_llm_provider}. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json." ) @@ -6663,13 +6682,13 @@ def process_messages(messages, max_tokens, model): messages = messages[::-1] final_messages = [] verbose_logger.debug( - f"calling process_messages with messages: {messages}, max_tokens: {max_tokens}, model: {model}" + "calling process_messages with messages: %s, max_tokens: %s, model: %s", messages, max_tokens, model ) for message in messages: - verbose_logger.debug(f"processing final_messages: {final_messages}") + verbose_logger.debug("processing final_messages: %s", final_messages) used_tokens = get_token_count(final_messages, model) available_tokens = max_tokens - used_tokens - verbose_logger.debug(f"used_tokens: {used_tokens}, available_tokens: {available_tokens}") + verbose_logger.debug("used_tokens: %s, available_tokens: %s", used_tokens, available_tokens) if available_tokens <= 3: break @@ -6680,15 +6699,15 @@ def process_messages(messages, max_tokens, model): max_tokens=max_tokens, model=model, ) - verbose_logger.debug(f"final_messages after attempt_message_addition: {final_messages}") - verbose_logger.debug(f"Final messages: {final_messages}") + verbose_logger.debug("final_messages after attempt_message_addition: %s", final_messages) + verbose_logger.debug("Final messages: %s", final_messages) return final_messages def attempt_message_addition(final_messages, message, available_tokens, max_tokens, model): temp_messages = [message] + final_messages temp_message_tokens = get_token_count(messages=temp_messages, model=model) - verbose_logger.debug(f"temp_message_tokens: {temp_message_tokens}, max_tokens: {max_tokens}") + verbose_logger.debug("temp_message_tokens: %s, max_tokens: %s", temp_message_tokens, max_tokens) if temp_message_tokens <= max_tokens: return temp_messages @@ -6735,12 +6754,12 @@ def shorten_message_to_fit_limit(message, tokens_needed, model: str | None, rais content = message["content"] attempts = 0 - verbose_logger.debug(f"content: {content}") + verbose_logger.debug("content: %s", content) while attempts < MAX_TOKEN_TRIMMING_ATTEMPTS: - verbose_logger.debug(f"getting token count for message: {message}") + verbose_logger.debug("getting token count for message: %s", message) total_tokens = get_token_count([message], model) - verbose_logger.debug(f"total_tokens: {total_tokens}, tokens_needed: {tokens_needed}") + verbose_logger.debug("total_tokens: %s, tokens_needed: %s", total_tokens, tokens_needed) if total_tokens <= tokens_needed: break @@ -6756,7 +6775,7 @@ def shorten_message_to_fit_limit(message, tokens_needed, model: str | None, rais trimmed_content = left_half + ".." + right_half message["content"] = trimmed_content - verbose_logger.debug(f"trimmed_content: {trimmed_content}") + verbose_logger.debug("trimmed_content: %s", trimmed_content) content = trimmed_content attempts += 1 @@ -6851,9 +6870,9 @@ def trim_messages( # we remove all system messages from the messages list messages = [message for message in messages if message["role"] != "system"] - verbose_logger.debug(f"Processed system message: {system_message_event}") + verbose_logger.debug("Processed system message: %s", system_message_event) final_messages = process_messages(messages=messages, max_tokens=max_tokens, model=model) - verbose_logger.debug(f"Processed messages: {final_messages}") + verbose_logger.debug("Processed messages: %s", final_messages) # Add system message to the beginning of the final messages if system_message_event: @@ -6862,13 +6881,13 @@ def trim_messages( if len(tool_messages) > 0: final_messages.extend(tool_messages) - verbose_logger.debug(f"Final messages: {final_messages}, return_response_tokens: {return_response_tokens}") + verbose_logger.debug("Final messages: %s, return_response_tokens: %s", final_messages, return_response_tokens) if return_response_tokens: # if user wants token count with new trimmed messages response_tokens = max_tokens - get_token_count(final_messages, model) return final_messages, response_tokens return final_messages except Exception as e: # [NON-Blocking, if error occurs just return final_messages - verbose_logger.exception(f"Got exception while token trimming - {e}") + verbose_logger.exception("Got exception while token trimming - %s", e) return original_messages @@ -6982,7 +7001,7 @@ def _get_valid_models_from_provider_api( _model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) return models except Exception as e: - verbose_logger.warning(f"Error getting valid models: {e}") + verbose_logger.warning("Error getting valid models: %s", e) return [] @@ -7056,7 +7075,7 @@ def get_valid_models( return valid_models except Exception as e: - verbose_logger.warning(f"Error getting valid models: {e}") + verbose_logger.warning("Error getting valid models: %s", e) return [] # NON-Blocking @@ -9112,7 +9131,7 @@ def is_prompt_caching_valid_prompt( min_token_count = get_prompt_cache_min_tokens(model=model) return token_count >= min_token_count except Exception as e: - verbose_logger.error(f"Error in is_prompt_caching_valid_prompt: {e}") + verbose_logger.error("Error in is_prompt_caching_valid_prompt: %s", e) return False diff --git a/litellm/vector_stores/vector_store_registry.py b/litellm/vector_stores/vector_store_registry.py index 1350e2b187e..37866a76797 100644 --- a/litellm/vector_stores/vector_store_registry.py +++ b/litellm/vector_stores/vector_store_registry.py @@ -235,7 +235,7 @@ class VectorStoreRegistry: self.add_vector_store_to_registry(vector_store=db_vector_store) return db_vector_store except Exception as e: - verbose_logger.debug(f"Error fetching vector store from database: {e}") + verbose_logger.debug("Error fetching vector store from database: %s", e) return None @@ -341,12 +341,13 @@ class VectorStoreRegistry: if db_vector_store is None: # Vector store was deleted from database, remove from cache verbose_logger.debug( - f"Vector store {vector_store_id} found in memory but deleted from database, removing from cache" + "Vector store %s found in memory but deleted from database, removing from cache", + vector_store_id, ) self.delete_vector_store_from_registry(vector_store_id=vector_store_id) vector_store = None except Exception as e: - verbose_logger.debug(f"Error verifying vector store {vector_store_id} in database: {e}") + verbose_logger.debug("Error verifying vector store %s in database: %s", vector_store_id, e) # Fall back to database if not found in memory (or was deleted) if vector_store is None and prisma_client is not None: @@ -355,7 +356,7 @@ class VectorStoreRegistry: vector_store_id=vector_store_id, prisma_client=prisma_client ) except Exception as e: - verbose_logger.debug(f"Error fetching vector store {vector_store_id} from database: {e}") + verbose_logger.debug("Error fetching vector store %s from database: %s", vector_store_id, e) if vector_store is not None: # Create a copy to avoid modifying the registry diff --git a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py index c97856598f1..8e6fa35b452 100644 --- a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py +++ b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py @@ -552,9 +552,15 @@ class TestMCPClientResolvedAuth: await http_client.aclose() +def _rendered_log_message(call): + message = str(call.args[0]) + values = call.args[1:] + return message % values if values else message + + def _all_logged_messages(mock_logger): return " ".join( - str(call.args[0]) + _rendered_log_message(call) for level in ("info", "debug", "warning", "error", "exception") for call in getattr(mock_logger, level).call_args_list if call.args diff --git a/tests/test_litellm/integrations/newrelic/test_newrelic.py b/tests/test_litellm/integrations/newrelic/test_newrelic.py index 541b271cb77..00a115dd8f2 100644 --- a/tests/test_litellm/integrations/newrelic/test_newrelic.py +++ b/tests/test_litellm/integrations/newrelic/test_newrelic.py @@ -22,6 +22,13 @@ import litellm import litellm.integrations.newrelic.newrelic as nr_module from litellm.integrations.newrelic.newrelic import NewRelicLogger + +def _rendered_log_message(call): + message = str(call.args[0]) + values = call.args[1:] + return message % values if values else message + + # The module may have been imported before sys.modules was patched (e.g. via # litellm's own startup imports), leaving _newrelic_agent=None. Point it at # the mock agent so all tests see a non-None agent. @@ -268,8 +275,9 @@ class TestParseBoolEnv: assert mock_warn.call_count == 2 # Warning should mention the variable name and the raw value for call in mock_warn.call_args_list: - assert "MY_VAR" in call.args[0] - assert repr(raw) in call.args[0] + rendered = _rendered_log_message(call) + assert "MY_VAR" in rendered + assert repr(raw) in rendered # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index d94f0d5f47e..47baacd61d7 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -20,6 +20,12 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import StandardCallbackDynamicParams +def _rendered_log_message(call): + message = str(call.args[0]) + values = call.args[1:] + return message % values if values else message + + @pytest.mark.asyncio async def test_anthropic_cache_control_hook_system_message(): # Use patch.dict to mock environment variables instead of setting them directly @@ -339,7 +345,7 @@ async def test_anthropic_cache_control_hook_out_of_bounds_logging(): # Verify that warning was called with the expected message mock_logger.warning.assert_called_once() - warning_call = mock_logger.warning.call_args[0][0] + warning_call = _rendered_log_message(mock_logger.warning.call_args) # Check that the warning message contains the expected information assert "AnthropicCacheControlHook: Provided index 10 is out of bounds" in warning_call @@ -405,7 +411,7 @@ async def test_anthropic_cache_control_hook_negative_out_of_bounds_logging(): # Verify that warning was called with the expected message mock_logger.warning.assert_called_once() - warning_call = mock_logger.warning.call_args[0][0] + warning_call = _rendered_log_message(mock_logger.warning.call_args) # Check that the warning message contains the original negative index assert "AnthropicCacheControlHook: Provided index -5 is out of bounds" in warning_call diff --git a/tests/test_litellm/llms/azure/test_azure_common_utils.py b/tests/test_litellm/llms/azure/test_azure_common_utils.py index c5d4bd044cc..85db11fdb24 100644 --- a/tests/test_litellm/llms/azure/test_azure_common_utils.py +++ b/tests/test_litellm/llms/azure/test_azure_common_utils.py @@ -1430,7 +1430,7 @@ def test_token_provider_returns_non_string(setup_mocks): # Verify the error was logged setup_mocks["logger"].error.assert_any_call( - "Azure AD token provider returned non-string value: " + "Azure AD token provider returned non-string value: %s", int ) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index 3a84428add1..850d01c6e34 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -24,6 +24,12 @@ from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer +def _rendered_log_message(call): + message = str(call.args[0]) + values = call.args[1:] + return message % values if values else message + + @pytest.fixture(autouse=True) def cleanup_mcp_global_state(): """Clean up MCP global state before and after each test. @@ -1106,12 +1112,12 @@ async def test_get_tools_from_mcp_servers_continues_when_one_server_fails(): assert result.outcomes["failing"].tag == "internal" # Verify failure logging - mock_logger.exception.assert_any_call( - "Error getting tools from server failing_server: Server connection failed" - ) + assert "Error getting tools from server failing_server: Server connection failed" in [ + _rendered_log_message(c) for c in mock_logger.exception.call_args_list if c.args + ] # Verify success logging - mock_logger.info.assert_any_call("Successfully fetched 1 tools total from all MCP servers") + mock_logger.info.assert_any_call("Successfully fetched %s tools total from all MCP servers", 1) @pytest.mark.asyncio @@ -1201,15 +1207,20 @@ async def test_get_tools_from_mcp_servers_handles_all_servers_failing(): assert result.outcomes["failing2"].tag == "internal" # Verify failure logging for both servers - mock_logger.exception.assert_any_call( + rendered_exceptions = [ + _rendered_log_message(c) for c in mock_logger.exception.call_args_list if c.args + ] + assert ( "Error getting tools from server failing_server1: Server failing_server1 connection failed" + in rendered_exceptions ) - mock_logger.exception.assert_any_call( + assert ( "Error getting tools from server failing_server2: Server failing_server2 connection failed" + in rendered_exceptions ) # Verify total logging - mock_logger.info.assert_any_call("Successfully fetched 0 tools total from all MCP servers") + mock_logger.info.assert_any_call("Successfully fetched %s tools total from all MCP servers", 0) @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index 329b6d5c45d..7ba9f463197 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -22,6 +22,12 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.types.mcp import MCPAuth +def _rendered_log_message(call): + message = str(call.args[0]) + values = call.args[1:] + return message % values if values else message + + def _build_request( headers: Optional[Dict[str, str]] = None, *, @@ -2048,9 +2054,9 @@ class TestCallToolRestAPI: assert exc_info.value.headers.get("www-authenticate") == challenge # The expected caller-must-reauth signal is logged once, at info, and never at error, so # error-rate alerts do not fire on normal pass-through re-authentication. - error_messages = [str(c.args[0]) for c in mock_logger.error.call_args_list if c.args] + error_messages = [_rendered_log_message(c) for c in mock_logger.error.call_args_list if c.args] assert not any("MCP tool call" in m for m in error_messages) - info_messages = [str(c.args[0]) for c in mock_logger.info.call_args_list if c.args] + info_messages = [_rendered_log_message(c) for c in mock_logger.info.call_args_list if c.args] assert sum(str(upstream_status) in m for m in info_messages) == 1 async def test_local_permission_denial_keeps_error_level_logging(self, monkeypatch): @@ -2112,9 +2118,9 @@ class TestCallToolRestAPI: await rest_endpoints.call_tool_rest_api(request, user_api_key_dict=UserAPIKeyAuth()) assert exc_info.value.status_code == 403 - error_messages = [str(c.args[0]) for c in mock_logger.error.call_args_list if c.args] + error_messages = [_rendered_log_message(c) for c in mock_logger.error.call_args_list if c.args] assert any("HTTPException in MCP tool call" in m for m in error_messages) - info_messages = [str(c.args[0]) for c in mock_logger.info.call_args_list if c.args] + info_messages = [_rendered_log_message(c) for c in mock_logger.info.call_args_list if c.args] assert not any("relaying upstream" in m for m in info_messages) async def test_success_logging_cancellation_propagates(self, monkeypatch): diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index f5aa695cb78..d1b5395c73d 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -57,6 +57,12 @@ from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.utils import get_utc_datetime +def _rendered_log_message(call): + message = str(call.args[0]) + values = call.args[1:] + return message % values if values else message + + @pytest.fixture(autouse=True) def set_salt_key(monkeypatch): """Automatically set LITELLM_SALT_KEY for all tests""" @@ -1167,7 +1173,7 @@ def test_log_budget_lookup_failure_dry_run(): err = Exception("column 'policies' does not exist in prisma schema") _log_budget_lookup_failure("user", err) mock_logger.error.assert_called_once() - call_msg = mock_logger.error.call_args[0][0] + call_msg = _rendered_log_message(mock_logger.error.call_args) assert "user" in call_msg assert "cache will not be populated" in call_msg assert "policies" in call_msg or "prisma" in call_msg diff --git a/tests/test_litellm/proxy/auth/test_model_checks_fallbacks.py b/tests/test_litellm/proxy/auth/test_model_checks_fallbacks.py index f3bd1742827..c65f36d1ef0 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks_fallbacks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks_fallbacks.py @@ -2,6 +2,12 @@ import pytest from unittest.mock import Mock, patch +def _rendered_log_message(call): + message = str(call.args[0]) + values = call.args[1:] + return message % values if values else message + + def create_mock_router( fallbacks=None, context_window_fallbacks=None, content_policy_fallbacks=None ): @@ -150,7 +156,7 @@ def test_invalid_fallback_type_returns_empty_list(): ) assert result == [] - mock_logger.warning.assert_called_once_with("Unknown fallback_type: invalid") + mock_logger.warning.assert_called_once_with("Unknown fallback_type: %s", "invalid") def test_exception_handling_returns_empty_list(): @@ -171,7 +177,7 @@ def test_exception_handling_returns_empty_list(): assert result == [] mock_logger.error.assert_called_once() - error_call_args = mock_logger.error.call_args[0][0] + error_call_args = _rendered_log_message(mock_logger.error.call_args) assert ( "Error getting fallbacks for model claude-4-sonnet" in error_call_args ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index fc018a5333e..eedffa1ea5f 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -984,7 +984,8 @@ async def test_add_team_member_budget_table_exception_handling(): # Verify the error was logged mock_logger.info.assert_called_once_with( - "Team member budget table not found, passed team_member_budget_id=nonexistent-budget-456" + "Team member budget table not found, passed team_member_budget_id=%s", + "nonexistent-budget-456", ) # Verify database call was attempted diff --git a/tests/test_litellm/test_logging.py b/tests/test_litellm/test_logging.py index fbed044445b..beba5794444 100644 --- a/tests/test_litellm/test_logging.py +++ b/tests/test_litellm/test_logging.py @@ -1,7 +1,9 @@ +import ast import asyncio import json import os import sys +from pathlib import Path from typing import List import pytest @@ -328,3 +330,66 @@ async def test_cache_hit_includes_custom_llm_provider(): # Clean up litellm.callbacks = original_callbacks litellm.cache = None + + +LITELLM_LOGGER_NAMES = frozenset( + {"verbose_logger", "verbose_proxy_logger", "verbose_router_logger", "logger", "logging"} +) +LOG_LEVEL_METHODS = frozenset({"debug", "info", "warning", "error", "exception", "critical"}) +LITELLM_PACKAGE_ROOT = Path(__file__).resolve().parents[2] / "litellm" + + +def _receiver_name(node: ast.expr) -> str: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + return node.attr + return "" + + +def _is_logging_call(node: ast.AST) -> bool: + return ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr in LOG_LEVEL_METHODS + and _receiver_name(node.func.value) in LITELLM_LOGGER_NAMES + ) + + +def _has_format_spec(message: ast.JoinedStr) -> bool: + return any(isinstance(value, ast.FormattedValue) and value.format_spec is not None for value in message.values) + + +def _eager_logging_calls(source: str, path: Path) -> tuple[str, ...]: + return tuple( + f"{path}:{node.lineno}" + for node in ast.walk(ast.parse(source)) + if _is_logging_call(node) + and node.args + and isinstance(node.args[0], ast.JoinedStr) + and not _has_format_spec(node.args[0]) + ) + + +def test_logging_calls_do_not_build_their_message_eagerly(): + """A discarded log record must not have cost anything to build. + + `log.debug(f"payload: {body}")` interpolates before the call runs, so the message is + built and thrown away on every request the level filters out; `log.debug("payload: %s", body)` + defers that to `record.getMessage()`, which only runs once the record passes the level check. + + f-strings carrying a format spec are exempt: `%`-style has no faithful equivalent for + specs like `{ratio:.1%}`, and those sites interpolate scalars rather than payloads. + """ + offenders = tuple( + offender + for path in sorted(LITELLM_PACKAGE_ROOT.rglob("*.py")) + for offender in _eager_logging_calls( + path.read_text(encoding="utf-8"), path.relative_to(LITELLM_PACKAGE_ROOT.parent) + ) + ) + + assert offenders == (), ( + "these logging calls build their message eagerly; pass the values as %-style arguments instead:\n" + + "\n".join(offenders) + ) From abe3289398bba544193366d9777b6953e2b3ed9e Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 3 Aug 2026 22:08:58 -0700 Subject: [PATCH 116/123] fix(proxy): retry model cost map fetch with Retry-After-aware backoff and keep current map on reload failure (#35739) * fix(proxy): retry model cost map fetch with Retry-After-aware backoff and stop downgrading to the packaged backup on reload failure A 429 or transient network error during a manual or scheduled model cost map reload used to silently replace litellm.model_cost with the stale backup JSON bundled in the installed wheel, stamp the reload as successful, and clear the force_reload flag, so a fleet could serve months-old pricing until the next interval. Runtime reloads now go through refetch_model_cost_map, which retries 429/5xx/transport errors up to 3 times honoring Retry-After (capped at 30s, exponential backoff with jitter otherwise) and returns a failure value instead of the backup when the fetch or integrity validation fails. On failure the pod keeps its currently loaded map, the periodic job leaves last_run and force_reload untouched so it retries on the next config poll, and the manual endpoint returns 502 with the reason instead of reporting a fake success. Startup behavior is unchanged: boot still falls back to the packaged backup since there is no previously loaded map to keep. * fix(proxy): use shared async httpx client for cost map reload and make retry tests CI-env-proof The reload fetch now goes through get_async_httpx_client with a dedicated httpxSpecialProvider.ModelCostMap pool instead of constructing a raw httpx.AsyncClient, so it inherits deployment-level TLS and transport settings and passes the ensure_async_clients gate. Tests inject a MockTransport-backed client through the same seam. An autouse fixture clears LITELLM_LOCAL_MODEL_COST_MAP, which CI exports and which short-circuited the retry tests; the two TestPriceDataReloadAPI tests and the config sync pubsub reload test that still patched get_model_cost_map now patch refetch_model_cost_map instead. --- .../litellm_core_utils/get_model_cost_map.py | 157 +++++++++++++++ litellm/proxy/proxy_server.py | 26 ++- litellm/types/llms/custom_http.py | 1 + .../test_get_model_cost_map.py | 183 ++++++++++++++++++ .../common_utils/test_config_sync_pubsub.py | 10 +- .../test_routes_model_cost_map.py | 45 ++++- tests/test_litellm/proxy/test_proxy_server.py | 116 ++++++++--- 7 files changed, 502 insertions(+), 36 deletions(-) diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index e87e3d8aca2..5d36e391fbc 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -8,9 +8,14 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True ``` """ +import asyncio import json import os +import random +from collections.abc import Awaitable, Callable +from dataclasses import dataclass from importlib.resources import files +from typing import Protocol import httpx @@ -161,6 +166,158 @@ class GetModelCostMap: return response.json() +RETRYABLE_FETCH_STATUS_CODES = frozenset({429, 500, 502, 503, 504}) +MODEL_COST_MAP_FETCH_MAX_ATTEMPTS = 3 +MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS = 30.0 + + +@dataclass(frozen=True, slots=True) +class ModelCostMapReloaded: + model_cost_map: dict # mutable-ok: adopted as litellm.model_cost, whose consumer contract is a plain mutable dict + + +@dataclass(frozen=True, slots=True) +class ModelCostMapReloadUnavailable: + reason: str + + +ModelCostMapReloadResult = ModelCostMapReloaded | ModelCostMapReloadUnavailable + + +@dataclass(frozen=True, slots=True) +class _FetchAttemptRetryable: + reason: str + retry_after_seconds: float | None + + +def _parse_retry_after_seconds(response: httpx.Response) -> float | None: + header = response.headers.get("Retry-After") + if header is None: + return None + try: + seconds = float(header) + except ValueError: + return None + return seconds if seconds >= 0 else None + + +def _retry_wait_seconds(outcome: _FetchAttemptRetryable, attempt: int, rng: random.Random) -> float: + if outcome.retry_after_seconds is not None: + return min(outcome.retry_after_seconds, MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS) + return min(float(2**attempt) + rng.uniform(0.0, 1.0), MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS) + + +class _AsyncGetClient(Protocol): + def get(self, url: str, *, timeout: float | None = None) -> Awaitable[httpx.Response]: ... + + +def _default_reload_client() -> _AsyncGetClient: + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.llms.custom_http import httpxSpecialProvider + + return get_async_httpx_client(llm_provider=httpxSpecialProvider.ModelCostMap) + + +async def _attempt_fetch( + client: _AsyncGetClient, url: str, timeout: int +) -> ModelCostMapReloaded | ModelCostMapReloadUnavailable | _FetchAttemptRetryable: + try: + response = await client.get(url, timeout=timeout) + except httpx.HTTPError as e: + return _FetchAttemptRetryable(reason=f"{type(e).__name__} fetching {url}: {e}", retry_after_seconds=None) + if response.status_code in RETRYABLE_FETCH_STATUS_CODES: + return _FetchAttemptRetryable( + reason=f"HTTP {response.status_code} from {url}", + retry_after_seconds=_parse_retry_after_seconds(response), + ) + if response.is_error: + return ModelCostMapReloadUnavailable(reason=f"HTTP {response.status_code} from {url}") + try: + parsed = response.json() + except ValueError as e: + return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}") + if not isinstance(parsed, dict): + return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}") + return ModelCostMapReloaded(model_cost_map=parsed) + + +async def _fetch_remote_model_cost_map_with_retry( + url: str, + timeout: int, + max_attempts: int, + sleep: Callable[[float], Awaitable[None]], + rng: random.Random, + client: _AsyncGetClient, +) -> ModelCostMapReloadResult: + for attempt in range(1, max_attempts + 1): + outcome = await _attempt_fetch(client=client, url=url, timeout=timeout) + if not isinstance(outcome, _FetchAttemptRetryable): + return outcome + if attempt == max_attempts: + return ModelCostMapReloadUnavailable(reason=f"{outcome.reason} (after {max_attempts} attempts)") + wait_seconds = _retry_wait_seconds(outcome=outcome, attempt=attempt, rng=rng) + verbose_logger.warning( + "LiteLLM: model cost map fetch attempt %d/%d failed (%s); retrying in %.1fs", + attempt, + max_attempts, + outcome.reason, + wait_seconds, + ) + await sleep(wait_seconds) + return ModelCostMapReloadUnavailable(reason="model cost map fetch failed") + + +async def refetch_model_cost_map( + url: str, + timeout: int = 5, + max_attempts: int = MODEL_COST_MAP_FETCH_MAX_ATTEMPTS, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + rng: random.Random | None = None, + client: "_AsyncGetClient | None" = None, +) -> ModelCostMapReloadResult: + """ + Re-fetch the model cost map for a runtime reload, retrying transient HTTP + errors (429/5xx/transport) with Retry-After-aware backoff. + + Unlike ``get_model_cost_map`` this never falls back to the packaged backup: + on failure it returns ``ModelCostMapReloadUnavailable`` so callers keep the + map they already have. + """ + if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true": + _cost_map_source_info.source = "local" + _cost_map_source_info.url = None + _cost_map_source_info.is_env_forced = True + _cost_map_source_info.fallback_reason = None + return ModelCostMapReloaded( + model_cost_map=_finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()) + ) + + result = await _fetch_remote_model_cost_map_with_retry( + url=url, + timeout=timeout, + max_attempts=max_attempts, + sleep=sleep, + rng=rng if rng is not None else random.Random(), + client=client if client is not None else _default_reload_client(), + ) + if isinstance(result, ModelCostMapReloadUnavailable): + verbose_logger.warning( + "LiteLLM: model cost map reload failed: %s. Keeping the currently loaded map.", + result.reason, + ) + return result + if not GetModelCostMap.validate_model_cost_map( + fetched_map=result.model_cost_map, + backup_model_count=GetModelCostMap._get_backup_model_count(), + ): + return ModelCostMapReloadUnavailable(reason=f"model cost map from {url} failed integrity validation") + _cost_map_source_info.source = "remote" + _cost_map_source_info.url = url + _cost_map_source_info.is_env_forced = False + _cost_map_source_info.fallback_reason = None + return ModelCostMapReloaded(model_cost_map=_finalize_model_cost_map(result.model_cost_map)) + + class ModelCostMapSourceInfo: """Tracks the source of the currently loaded model cost map.""" diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 16a63540d4d..e8299be8f07 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6538,11 +6538,18 @@ class ProxyConfig: if should_reload: # Perform the reload from litellm.litellm_core_utils.get_model_cost_map import ( - get_model_cost_map, + ModelCostMapReloadUnavailable, + refetch_model_cost_map, ) model_cost_map_url = litellm.model_cost_map_url - new_model_cost_map = get_model_cost_map(url=model_cost_map_url) + reload_result = await refetch_model_cost_map(url=model_cost_map_url) + if isinstance(reload_result, ModelCostMapReloadUnavailable): + verbose_proxy_logger.warning( + f"Model cost map reload failed ({reload_result.reason}); keeping current pricing data, will retry on the next config poll" + ) + return + new_model_cost_map = reload_result.model_cost_map litellm.model_cost = new_model_cost_map # Invalidate case-insensitive lookup map since model_cost was replaced _invalidate_model_cost_lowercase_map() @@ -15836,10 +15843,19 @@ async def reload_model_cost_map( raise HTTPException(status_code=500, detail="Database connection not available") # Immediately reload the model cost map in the current pod - from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map + from litellm.litellm_core_utils.get_model_cost_map import ( + ModelCostMapReloadUnavailable, + refetch_model_cost_map, + ) model_cost_map_url = litellm.model_cost_map_url - new_model_cost_map = get_model_cost_map(url=model_cost_map_url) + reload_result = await refetch_model_cost_map(url=model_cost_map_url) + if isinstance(reload_result, ModelCostMapReloadUnavailable): + raise HTTPException( + status_code=502, + detail=f"Failed to reload model cost map: {reload_result.reason}. Current pricing data was kept.", + ) + new_model_cost_map = reload_result.model_cost_map litellm.model_cost = new_model_cost_map # Invalidate case-insensitive lookup map since model_cost was replaced _invalidate_model_cost_lowercase_map() @@ -15881,6 +15897,8 @@ async def reload_model_cost_map( "models_count": models_count, "timestamp": current_time.isoformat(), } + except HTTPException: + raise except Exception as e: verbose_proxy_logger.exception("Failed to reload model cost map: %s", e) raise HTTPException(status_code=500, detail=f"Failed to reload model cost map: {e}") diff --git a/litellm/types/llms/custom_http.py b/litellm/types/llms/custom_http.py index d7dfc0e486b..548dde4c04d 100644 --- a/litellm/types/llms/custom_http.py +++ b/litellm/types/llms/custom_http.py @@ -30,6 +30,7 @@ class httpxSpecialProvider(str, Enum): PromptManagement = "prompt_management" UI = "ui" Sandbox = "sandbox" + ModelCostMap = "model_cost_map" VerifyTypes = Union[str, bool, ssl.SSLContext] diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index f21c667276e..463bdc6161f 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -248,3 +248,186 @@ def test_azure_ai_claude_1m_context_entries(cost_map: dict): "azure_ai/claude-haiku-4-5", ]: assert cost_map[model]["max_input_tokens"] == 200000, model + + +# --------------------------------------------------------------------------- +# refetch_model_cost_map: retry/backoff behavior for runtime reloads +# --------------------------------------------------------------------------- + +import functools +import random + +import httpx + +from litellm.litellm_core_utils.get_model_cost_map import ( + ModelCostMapReloaded, + ModelCostMapReloadUnavailable, + refetch_model_cost_map, +) + +_URL = "https://example.invalid/model_prices.json" + + +@functools.lru_cache(maxsize=1) +def _real_map_bytes() -> bytes: + return json.dumps(_load_root_cost_map()).encode() + + +class _SleepRecorder: + """Injected in place of asyncio.sleep so tests assert waits without real delay.""" + + def __init__(self): + self.waits = [] + + async def __call__(self, seconds: float) -> None: + self.waits.append(seconds) + + +@pytest.fixture(autouse=True) +def _unset_local_cost_map_env(monkeypatch): + """CI exports LITELLM_LOCAL_MODEL_COST_MAP=True; clear it so fetch behavior is deterministic.""" + monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) + + +def _mock_client(outcomes): + """httpx client over a MockTransport serving one outcome per request; an exception instance is raised.""" + calls = {"count": 0} + + def handler(request): + idx = min(calls["count"], len(outcomes) - 1) + calls["count"] += 1 + outcome = outcomes[idx] + if isinstance(outcome, Exception): + raise outcome + return outcome + + return httpx.AsyncClient(transport=httpx.MockTransport(handler)), calls + + +@pytest.mark.asyncio +async def test_refetch_retries_429_honoring_retry_after(): + """Two 429s with Retry-After then success: waits follow the header, not backoff.""" + client, calls = _mock_client( + [ + httpx.Response(429, headers={"Retry-After": "7"}), + httpx.Response(429, headers={"Retry-After": "7"}), + httpx.Response(200, content=_real_map_bytes()), + ] + ) + sleeper = _SleepRecorder() + result = await refetch_model_cost_map( + url=_URL, sleep=sleeper, rng=random.Random(0), client=client + ) + assert isinstance(result, ModelCostMapReloaded) + assert len(result.model_cost_map) > 100 + assert calls["count"] == 3 + assert sleeper.waits == [7.0, 7.0] + + +@pytest.mark.asyncio +async def test_refetch_gives_up_after_max_attempts_with_exponential_backoff(): + """All 429 without Retry-After: exponential backoff waits, then a failure value.""" + client, calls = _mock_client([httpx.Response(429)]) + sleeper = _SleepRecorder() + result = await refetch_model_cost_map( + url=_URL, sleep=sleeper, rng=random.Random(0), client=client + ) + assert isinstance(result, ModelCostMapReloadUnavailable) + assert "429" in result.reason + assert "after 3 attempts" in result.reason + assert calls["count"] == 3 + assert len(sleeper.waits) == 2 + assert 2.0 <= sleeper.waits[0] < 3.0 + assert 4.0 <= sleeper.waits[1] < 5.0 + + +@pytest.mark.asyncio +async def test_refetch_caps_retry_after_wait(): + """A hostile/huge Retry-After is capped so reloads never sleep unbounded.""" + client, _calls = _mock_client( + [ + httpx.Response(429, headers={"Retry-After": "9999"}), + httpx.Response(200, content=_real_map_bytes()), + ] + ) + sleeper = _SleepRecorder() + result = await refetch_model_cost_map( + url=_URL, sleep=sleeper, rng=random.Random(0), client=client + ) + assert isinstance(result, ModelCostMapReloaded) + assert sleeper.waits == [30.0] + + +@pytest.mark.asyncio +async def test_refetch_retries_transport_errors(): + """Connection failures are transient: retried like 5xx, succeeding when the network heals.""" + client, calls = _mock_client( + [ + httpx.ConnectError("connection refused"), + httpx.Response(200, content=_real_map_bytes()), + ] + ) + sleeper = _SleepRecorder() + result = await refetch_model_cost_map( + url=_URL, sleep=sleeper, rng=random.Random(0), client=client + ) + assert isinstance(result, ModelCostMapReloaded) + assert calls["count"] == 2 + assert len(sleeper.waits) == 1 + + +@pytest.mark.asyncio +async def test_refetch_non_retryable_status_fails_immediately(): + """A 404 is permanent: one attempt, no sleeps, failure value.""" + client, calls = _mock_client([httpx.Response(404)]) + sleeper = _SleepRecorder() + result = await refetch_model_cost_map( + url=_URL, sleep=sleeper, rng=random.Random(0), client=client + ) + assert isinstance(result, ModelCostMapReloadUnavailable) + assert "404" in result.reason + assert calls["count"] == 1 + assert sleeper.waits == [] + + +@pytest.mark.asyncio +async def test_refetch_invalid_json_fails_immediately(): + client, calls = _mock_client([httpx.Response(200, content=b"not json")]) + sleeper = _SleepRecorder() + result = await refetch_model_cost_map( + url=_URL, sleep=sleeper, rng=random.Random(0), client=client + ) + assert isinstance(result, ModelCostMapReloadUnavailable) + assert "invalid JSON" in result.reason + assert calls["count"] == 1 + assert sleeper.waits == [] + + +@pytest.mark.asyncio +async def test_refetch_shrunk_map_fails_integrity_not_swapped_in(): + """A drastically shrunk upstream file is rejected instead of being adopted.""" + tiny = json.dumps(_make_models(60)).encode() + client, _calls = _mock_client([httpx.Response(200, content=tiny)]) + result = await refetch_model_cost_map( + url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client + ) + assert isinstance(result, ModelCostMapReloadUnavailable) + assert "integrity validation" in result.reason + + +@pytest.mark.asyncio +async def test_refetch_respects_local_env_override(monkeypatch): + """LITELLM_LOCAL_MODEL_COST_MAP=True short-circuits to the bundled backup, zero HTTP.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + + def _fail(request): + raise AssertionError("no HTTP request should be made when local map is forced") + + result = await refetch_model_cost_map( + url=_URL, + sleep=_SleepRecorder(), + rng=random.Random(0), + client=httpx.AsyncClient(transport=httpx.MockTransport(_fail)), + ) + assert isinstance(result, ModelCostMapReloaded) + assert len(result.model_cost_map) > 100 diff --git a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py index 6872407808c..e357f25123b 100644 --- a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py +++ b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py @@ -776,8 +776,14 @@ async def test_model_cost_map_reload_does_not_publish_config_change() -> None: original_model_cost = litellm.model_cost.copy() _set_redis_usage_cache(_FakeRedisCache(client)) try: - with patch("litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map") as mock_get_map: - mock_get_map.return_value = {"gpt-5.2": {"input_cost_per_token": 0.001}} + from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded + + with patch( + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + new=AsyncMock( + return_value=ModelCostMapReloaded(model_cost_map={"gpt-5.2": {"input_cost_per_token": 0.001}}) + ), + ): await ProxyConfig()._check_and_reload_model_cost_map(prisma_client=prisma_client) finally: litellm.model_cost = original_model_cost diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py b/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py index 16e410f1b1e..df1d096b3e2 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py @@ -48,6 +48,7 @@ def _attach_litellm_config(mock_prisma): def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma): """Admin can trigger a manual reload; handler returns model count + status.""" + from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -56,8 +57,8 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma): fake_cost_map = {"gpt-4": {"input_cost": 0.03}, "gpt-3.5": {"input_cost": 0.002}} monkeypatch.setattr( - "litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map", - lambda url=None: fake_cost_map, + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + AsyncMock(return_value=ModelCostMapReloaded(model_cost_map=fake_cost_map)), ) monkeypatch.setattr("litellm.add_known_models", lambda model_cost_map=None: None) monkeypatch.setattr("litellm.model_cost", {}, raising=False) @@ -85,6 +86,46 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma): assert table.upsert.await_count == 1 +def test_reload_model_cost_map_fetch_failure_502_keeps_map( + client, auth_as, monkeypatch, mock_prisma +): + """Fetch failure returns 502 with the reason; the pod's map and DB flag are untouched. + + Regression: the endpoint used to report success after silently swapping in + the stale packaged backup. + """ + from litellm.litellm_core_utils.get_model_cost_map import ( + ModelCostMapReloadUnavailable, + ) + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _attach_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + sentinel_map = {"existing-model": {"input_cost": 0.01}} + monkeypatch.setattr("litellm.model_cost", sentinel_map, raising=False) + monkeypatch.setattr( + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + AsyncMock( + return_value=ModelCostMapReloadUnavailable( + reason="HTTP 429 from upstream (after 3 attempts)" + ) + ), + ) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post("/reload/model_cost_map") + assert response.status_code == 502 + detail = response.json().get("detail", "") + assert "HTTP 429 from upstream" in detail + assert "Current pricing data was kept" in detail + import litellm as litellm_module + + assert litellm_module.model_cost is sentinel_map + assert table.upsert.await_count == 0 + + def test_reload_model_cost_map_not_admin_forbidden(client, auth_as): """Non-admin caller gets 403 with a role-specific detail.""" from litellm.proxy._types import LitellmUserRoles diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index ab64224fdc3..67e76585485 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -3622,14 +3622,18 @@ class TestPriceDataReloadAPI: # Save the original model_cost so the endpoint's direct assignment # (litellm.model_cost = new_model_cost_map) does not contaminate # subsequent tests running in the same worker process. + from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded + original_model_cost = litellm.model_cost.copy() try: with patch( - "litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map" - ) as mock_get_map: - mock_get_map.return_value = { - "gpt-3.5-turbo": {"input_cost_per_token": 0.001} - } + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + new=AsyncMock( + return_value=ModelCostMapReloaded( + model_cost_map={"gpt-3.5-turbo": {"input_cost_per_token": 0.001}} + ) + ), + ): # Mock the database connection with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: mock_prisma.db.litellm_config.find_unique = AsyncMock( @@ -3684,10 +3688,9 @@ class TestPriceDataReloadAPI: def test_reload_model_cost_map_error_handling(self, client_with_auth): """Test error handling in the reload endpoint""" with patch( - "litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map" - ) as mock_get_map: - mock_get_map.side_effect = Exception("Network error") - + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + new=AsyncMock(side_effect=Exception("Network error")), + ): # Mock the database connection with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) @@ -3697,7 +3700,7 @@ class TestPriceDataReloadAPI: assert ( response.status_code == 500 - ) # The new implementation immediately reloads and fails on error + ) # An unexpected exception still maps to 500 data = response.json() assert "Failed to reload model cost map" in data["detail"] @@ -3885,13 +3888,16 @@ class TestPriceDataReloadIntegration: "gpt-4": {"input_cost_per_token": 0.03, "output_cost_per_token": 0.06}, } + from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded + original_model_cost = litellm.model_cost.copy() try: with patch( - "litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map" - ) as mock_get_map: - mock_get_map.return_value = mock_cost_map - + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + new=AsyncMock( + return_value=ModelCostMapReloaded(model_cost_map=mock_cost_map) + ), + ): # Mock the database connection with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: mock_prisma.db.litellm_config.find_unique = AsyncMock( @@ -3956,15 +3962,18 @@ class TestPriceDataReloadIntegration: mock_prisma.get_generic_data = AsyncMock(return_value=mock_config) mock_prisma.db.litellm_config.upsert = AsyncMock(return_value=None) + from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded + original_model_cost = litellm.model_cost.copy() try: with patch( - "litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map" - ) as mock_get_map: - mock_get_map.return_value = { - "gpt-3.5-turbo": {"input_cost_per_token": 0.001} - } - + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + new=AsyncMock( + return_value=ModelCostMapReloaded( + model_cost_map={"gpt-3.5-turbo": {"input_cost_per_token": 0.001}} + ) + ), + ): # Should reload due to force flag asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma)) @@ -3999,13 +4008,18 @@ class TestPriceDataReloadIntegration: mock_prisma.get_generic_data = AsyncMock(return_value=mock_config) mock_prisma.db.litellm_config.upsert = AsyncMock(return_value=None) + from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded + original_model_cost = litellm.model_cost.copy() try: with patch( - "litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map" - ) as mock_get_map: - mock_get_map.return_value = {"gpt-4": {"input_cost_per_token": 0.001}} - + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + new=AsyncMock( + return_value=ModelCostMapReloaded( + model_cost_map={"gpt-4": {"input_cost_per_token": 0.001}} + ) + ), + ): asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma)) # Verify the upsert update branch preserves interval_hours @@ -4022,6 +4036,47 @@ class TestPriceDataReloadIntegration: litellm.model_cost = original_model_cost _invalidate_model_cost_lowercase_map() + def test_distributed_reload_keeps_current_map_when_fetch_fails(self): + """Fetch failure during a periodic/forced reload must not downgrade the pod. + + Regression: a 429/network failure used to silently replace litellm.model_cost + with the stale packaged backup, stamp last_run, and clear force_reload. + """ + from litellm.litellm_core_utils.get_model_cost_map import ( + ModelCostMapReloadUnavailable, + ) + from litellm.proxy import proxy_server as ps + from litellm.proxy.proxy_server import ProxyConfig + + proxy_config = ProxyConfig() + mock_prisma = MagicMock() + + mock_config = MagicMock() + mock_config.param_value = {"interval_hours": 6, "force_reload": True} + mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=mock_config) + mock_prisma.get_generic_data = AsyncMock(return_value=mock_config) + mock_prisma.db.litellm_config.upsert = AsyncMock(return_value=None) + + original_model_cost = litellm.model_cost + with patch( + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + new=AsyncMock( + return_value=ModelCostMapReloadUnavailable(reason="HTTP 429 from upstream") + ), + ): + with patch("litellm.proxy.proxy_server.last_model_cost_map_reload", None): + asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma)) + assert ps.last_model_cost_map_reload is None, ( + "a failed reload must not stamp the pod's last reload time, " + "otherwise the retry waits a full interval" + ) + + assert litellm.model_cost is original_model_cost, ( + "a failed reload must keep the currently loaded cost map, " + "not swap in the packaged backup" + ) + mock_prisma.db.litellm_config.upsert.assert_not_called() + def test_manual_reload_preserves_interval_hours(self): """Test that manual reload via /reload/model_cost_map preserves existing interval_hours. @@ -4041,13 +4096,18 @@ class TestPriceDataReloadIntegration: app.dependency_overrides[user_api_key_auth] = lambda: mock_auth client = TestClient(app) + from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded + original_model_cost = litellm.model_cost.copy() try: with patch( - "litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map" - ) as mock_get_map: - mock_get_map.return_value = {"gpt-4": {"input_cost_per_token": 0.001}} - + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + new=AsyncMock( + return_value=ModelCostMapReloaded( + model_cost_map={"gpt-4": {"input_cost_per_token": 0.001}} + ) + ), + ): with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: # Simulate existing config with a schedule mock_existing = MagicMock() From a625d1e1ca2ef9dff286ff309a9097fc877bd75a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 23:10:01 -0700 Subject: [PATCH 117/123] feat(otel): stamp service tier attributes on inference spans (#35679) * feat(otel): stamp service tier attributes on inference spans Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): bound requested service tier to known values The requested tier is caller-controlled and reaches the span verbatim, so an arbitrary string lands on every litellm_request span on success and on failure. A 100k character value was stamped uncapped; safe_set_attribute does not truncate and no span limits are configured. Apply KNOWN_REQUEST_SERVICE_TIERS in get_requested_service_tier so both the span attribute and the Prometheus label bound the value the same way. The served tier stays unrestricted since it comes from the provider, so a tier a provider adds later is still reported. Prometheus label behavior is unchanged. * fix: derive known service tiers from the ServiceTier enum The allowlist omitted "fast", which litellm models as a real tier and prices through the priority cost key, so a request naming it resolved to no tier on the span and no Prometheus label. Deriving the set from ServiceTier keeps the two in sync, so a tier added there for cost calculation cannot go missing here. Behavior change: a request with service_tier "fast" now carries the tier on the span and on the Prometheus service_tier label, where it previously resolved to none. Every other value resolves as before. * refactor: build the known service tiers without a mutable intermediate The set comprehension and set literal tripped LIT002, which bounds mutable collections. Concatenating tuples keeps the derivation from ServiceTier while every intermediate stays immutable; the resulting frozenset is unchanged. --------- Co-authored-by: milan Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Yucheng Zhu --- litellm/integrations/opentelemetry.py | 28 ++++ litellm/integrations/prometheus.py | 51 +------- .../litellm_core_utils/service_tier_utils.py | 68 ++++++++++ .../integrations/test_opentelemetry.py | 121 ++++++++++++++++++ .../test_prometheus_service_tier_label.py | 13 +- 5 files changed, 231 insertions(+), 50 deletions(-) create mode 100644 litellm/litellm_core_utils/service_tier_utils.py diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 94902460c81..e6147409ed8 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -24,6 +24,10 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import ( from litellm.integrations.otel.model.semconv import Metric from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.secret_redaction import redact_string +from litellm.litellm_core_utils.service_tier_utils import ( + get_requested_service_tier, + get_served_service_tier, +) from litellm.secret_managers.main import get_secret_bool, str_to_bool from litellm.types.services import ServiceLoggerPayload from litellm.types.utils import ( @@ -74,6 +78,11 @@ PREPROCESSING_DURATION_MS_ATTRIBUTE = "litellm.preprocessing.duration_ms" TEAM_METADATA_ATTRIBUTE = "litellm.team.metadata" MODEL_GROUP_ATTRIBUTE = "litellm.model_group" PROVIDER_MODEL_ATTRIBUTE = "litellm.provider.model" +# semconv names the service tier attributes under the openai namespace, but every +# provider that reports a tier (OpenAI, Anthropic, Bedrock, Groq, Vertex) uses the +# same request param and response field, so both keys carry all of them. +REQUEST_SERVICE_TIER_ATTRIBUTE = "gen_ai.openai.request.service_tier" +RESPONSE_SERVICE_TIER_ATTRIBUTE = "gen_ai.openai.response.service_tier" # Remove the hardcoded LITELLM_RESOURCE dictionary - we'll create it properly later RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request" LITELLM_REQUEST_SPAN_NAME = "litellm_request" @@ -1411,6 +1420,23 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): if provider_model: self.safe_set_attribute(span=span, key=PROVIDER_MODEL_ATTRIBUTE, value=provider_model) + def _set_service_tier_attributes( + self, + span: Span, + standard_logging_payload: StandardLoggingPayload, + ) -> None: + """Stamp the tier the caller asked for and the tier the provider reports it + served, so tier usage is segmentable in traces. Both are optional: a caller + may not name a tier, and streaming responses carry no served tier. + """ + requested_tier = get_requested_service_tier(standard_logging_payload) + if requested_tier is not None: + self.safe_set_attribute(span=span, key=REQUEST_SERVICE_TIER_ATTRIBUTE, value=requested_tier) + + served_tier = get_served_service_tier(standard_logging_payload) + if served_tier is not None: + self.safe_set_attribute(span=span, key=RESPONSE_SERVICE_TIER_ATTRIBUTE, value=served_tier) + @staticmethod def _team_metadata_json(value: Any, allowed_keys: list[str]) -> str | None: """JSON-serialize only the allowlisted sub-keys of a team's metadata. @@ -2310,6 +2336,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): value=response_obj.get("model"), ) + self._set_service_tier_attributes(span=span, standard_logging_payload=standard_logging_payload) + usage = response_obj and response_obj.get("usage") if usage: self.safe_set_attribute( diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 78f4e213f50..b92e4f7e723 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -34,6 +34,9 @@ from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, get_metadata_variable_name_from_kwargs, ) +from litellm.litellm_core_utils.service_tier_utils import ( + get_service_tier_from_standard_logging_payload, +) from litellm.proxy._types import ( LiteLLM_DeletedVerificationToken, LiteLLM_TeamTable, @@ -98,16 +101,6 @@ class _ExcludedLabelMetric: return self._metric.labels(*kept_values) if kept_values else self._metric -# Tiers a caller may name in a request, across the providers that accept the -# parameter: OpenAI ("auto", "default", "flex", "priority", "scale"), Bedrock and -# Groq (subsets of those), Anthropic ("auto", "standard_only") and Vertex, which -# maps "default" to "standard". Used to bound the caller-controlled fallback in -# ``get_service_tier_from_standard_logging_payload``. -KNOWN_REQUEST_SERVICE_TIERS = frozenset( - {"auto", "batch", "default", "flex", "priority", "scale", "standard", "standard_only"} -) - - def _get_budget_metrics_per_request_timeout() -> float: raw = os.getenv("PROMETHEUS_BUDGET_METRICS_PER_REQUEST_TIMEOUT") if raw is None: @@ -4181,44 +4174,6 @@ def get_custom_labels_from_metadata(metadata: dict) -> dict[str, str]: return result -def get_service_tier_from_standard_logging_payload( - standard_logging_payload: StandardLoggingPayload, -) -> str | None: - """ - Resolve the service tier a request ran on, for the ``service_tier`` label. - - The tier the provider actually served wins over the tier the caller asked for, - so latency and spend stay segmentable when the request said ``auto`` and the - provider picked the concrete tier. Providers report the served tier either at - the top level of the response (OpenAI, Bedrock, Groq) or on the usage object - (Anthropic). - - Streaming responses carry no served tier, so the requested tier is the - fallback. That value is caller-controlled and survives param mapping even - where the provider then ignores it (Bedrock and Groq accept the request and - drop an unrecognized tier), so it is only labelled when it names a known - tier; otherwise one caller could mint a Prometheus series per string. Values - the provider itself reports are not caller-controlled and stay unrestricted, - so a tier a provider adds later is still labelled correctly. - """ - response = standard_logging_payload.get("response") - usage_object = standard_logging_payload.get("metadata", {}).get("usage_object") - - served_candidates: tuple[object, ...] = ( - response.get("service_tier") if isinstance(response, dict) else None, - usage_object.get("service_tier") if isinstance(usage_object, dict) else None, - ) - served_tier = next((tier for tier in served_candidates if isinstance(tier, str) and tier), None) - if served_tier is not None: - return served_tier - - model_parameters = standard_logging_payload.get("model_parameters") - requested_tier = model_parameters.get("service_tier") if isinstance(model_parameters, dict) else None - if isinstance(requested_tier, str) and requested_tier in KNOWN_REQUEST_SERVICE_TIERS: - return requested_tier - return None - - def _get_combined_custom_metadata_from_standard_logging_payload( standard_logging_payload: dict | None, ) -> dict[str, Any]: diff --git a/litellm/litellm_core_utils/service_tier_utils.py b/litellm/litellm_core_utils/service_tier_utils.py new file mode 100644 index 00000000000..9e3f5da42c4 --- /dev/null +++ b/litellm/litellm_core_utils/service_tier_utils.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from litellm.types.utils import ServiceTier, StandardLoggingPayload + +# Tiers a caller may name in a request, across the providers that accept the +# parameter: OpenAI ("auto", "default", "flex", "priority", "scale"), Bedrock and +# Groq (subsets of those), Anthropic ("auto", "standard_only") and Vertex, which +# maps "default" to "standard". Bounds the caller-controlled requested tier +# wherever it is recorded. Derived from ``ServiceTier`` so a tier added there for +# cost calculation cannot go missing here. +KNOWN_REQUEST_SERVICE_TIERS = frozenset( + tuple(tier.value for tier in ServiceTier) + ("batch", "default", "scale", "standard", "standard_only") +) + + +def get_served_service_tier(standard_logging_payload: StandardLoggingPayload) -> str | None: + """ + The tier the provider reports it actually served the request on. + + Providers report it either at the top level of the response (OpenAI, Bedrock, + Groq) or on the usage object (Anthropic). Streaming responses carry no served + tier. + """ + response = standard_logging_payload.get("response") + usage_object = standard_logging_payload.get("metadata", {}).get("usage_object") + + served_candidates: tuple[object, ...] = ( + response.get("service_tier") if isinstance(response, dict) else None, + usage_object.get("service_tier") if isinstance(usage_object, dict) else None, + ) + return next((tier for tier in served_candidates if isinstance(tier, str) and tier), None) + + +def get_requested_service_tier(standard_logging_payload: StandardLoggingPayload) -> str | None: + """ + The tier the caller asked for, as sent to the provider. + + The value is caller-controlled and survives param mapping even where the + provider then ignores it (Bedrock and Groq accept the request and drop an + unrecognized tier), so it is only reported when it names a known tier. + """ + model_parameters = standard_logging_payload.get("model_parameters") + requested_tier = model_parameters.get("service_tier") if isinstance(model_parameters, dict) else None + if isinstance(requested_tier, str) and requested_tier in KNOWN_REQUEST_SERVICE_TIERS: + return requested_tier + return None + + +def get_service_tier_from_standard_logging_payload( + standard_logging_payload: StandardLoggingPayload, +) -> str | None: + """ + Resolve the service tier a request ran on, for the Prometheus ``service_tier`` label. + + The tier the provider actually served wins over the tier the caller asked for, + so latency and spend stay segmentable when the request said ``auto`` and the + provider picked the concrete tier. + + Streaming responses carry no served tier, so the requested tier is the + fallback. Values the provider itself reports are not caller-controlled and + stay unrestricted, so a tier a provider adds later is still labelled + correctly. + """ + served_tier = get_served_service_tier(standard_logging_payload) + if served_tier is not None: + return served_tier + + return get_requested_service_tier(standard_logging_payload) diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 05205cb76f2..b300c386326 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -5886,3 +5886,124 @@ class TestOpenTelemetryMetricAttributeFiltering(unittest.TestCase): exporter="console", attributes=attributes ) ) + + +class TestOTELServiceTierAttributes(unittest.TestCase): + """The tier a request asked for and the tier the provider served must land on + the litellm_request span, so tier usage is segmentable in traces.""" + + REQUEST_KEY = "gen_ai.openai.request.service_tier" + RESPONSE_KEY = "gen_ai.openai.response.service_tier" + + def _span_attributes(self, standard_logging_object, response_obj): + otel = OpenTelemetry() + mock_span = MagicMock() + kwargs = { + "model": "gpt-5-mini", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": standard_logging_object.get("model_parameters") or {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": standard_logging_object, + } + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + return {call[0][0]: call[0][1] for call in mock_span.set_attribute.call_args_list} + + def test_served_tier_from_response_and_requested_tier_are_stamped(self): + response_obj = { + "id": "chatcmpl-1", + "model": "gpt-5-mini", + "service_tier": "scale", + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + } + attributes = self._span_attributes( + { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + "model_parameters": {"service_tier": "auto"}, + "response": response_obj, + }, + response_obj, + ) + self.assertEqual(attributes[self.RESPONSE_KEY], "scale") + self.assertEqual(attributes[self.REQUEST_KEY], "auto") + + def test_served_tier_read_from_usage_object(self): + """Anthropic reports the served tier on the usage object, not the top level.""" + response_obj = { + "id": "chatcmpl-2", + "model": "claude-sonnet-4-5", + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + } + attributes = self._span_attributes( + { + "id": "test-id", + "call_type": "completion", + "metadata": {"usage_object": {"service_tier": "priority"}}, + "model_parameters": {}, + "response": response_obj, + }, + response_obj, + ) + self.assertEqual(attributes[self.RESPONSE_KEY], "priority") + self.assertNotIn(self.REQUEST_KEY, attributes) + + def test_no_tier_anywhere_stamps_nothing(self): + response_obj = { + "id": "chatcmpl-3", + "model": "gpt-5-mini", + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + } + attributes = self._span_attributes( + { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + "model_parameters": {}, + "response": response_obj, + }, + response_obj, + ) + self.assertNotIn(self.RESPONSE_KEY, attributes) + self.assertNotIn(self.REQUEST_KEY, attributes) + + def test_unknown_requested_tier_is_not_stamped(self): + """The requested tier is caller-controlled, so an unrecognized value is + dropped rather than written verbatim onto the span.""" + response_obj = { + "id": "chatcmpl-4", + "model": "gpt-5-mini", + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + } + attributes = self._span_attributes( + { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + "model_parameters": {"service_tier": "Z" * 5000}, + "response": response_obj, + }, + response_obj, + ) + self.assertNotIn(self.REQUEST_KEY, attributes) + + def test_served_tier_is_stamped_even_when_unrecognized(self): + """The served tier comes from the provider, not the caller, so a tier a + provider adds later is still stamped.""" + response_obj = { + "id": "chatcmpl-5", + "model": "gpt-5-mini", + "service_tier": "tier-added-by-provider-later", + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + } + attributes = self._span_attributes( + { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + "model_parameters": {}, + "response": response_obj, + }, + response_obj, + ) + self.assertEqual(attributes[self.RESPONSE_KEY], "tier-added-by-provider-later") diff --git a/tests/test_litellm/integrations/test_prometheus_service_tier_label.py b/tests/test_litellm/integrations/test_prometheus_service_tier_label.py index 9d702b19c6c..b2212c4ff41 100644 --- a/tests/test_litellm/integrations/test_prometheus_service_tier_label.py +++ b/tests/test_litellm/integrations/test_prometheus_service_tier_label.py @@ -13,9 +13,9 @@ import datetime import pytest -from litellm.integrations.prometheus import ( +from litellm.integrations.prometheus import PrometheusLogger +from litellm.litellm_core_utils.service_tier_utils import ( KNOWN_REQUEST_SERVICE_TIERS, - PrometheusLogger, get_service_tier_from_standard_logging_payload, ) from litellm.types.integrations.prometheus import ( @@ -230,3 +230,12 @@ async def test_success_event_emits_service_tier_on_latency_and_spend_metrics(): ) finally: _clear_prometheus_registry() + + +def test_allowlist_covers_every_modeled_service_tier(): + """A tier modeled for cost calculation is real traffic, so it must resolve + rather than being dropped as an unknown caller value.""" + from litellm.types.utils import ServiceTier + + missing = {tier.value for tier in ServiceTier} - KNOWN_REQUEST_SERVICE_TIERS + assert not missing, f"ServiceTier values missing from the allowlist: {sorted(missing)}" From 956d5177d1d915adc8084c142d9d2babad1ff7af Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 3 Aug 2026 23:41:21 -0700 Subject: [PATCH 118/123] fix(proxy): log the model cost map reload failure lazily (#35750) The reload-failure warning built its message with an f-string, so the interpolation ran on every failed reload whether or not the warning level was enabled. `test_logging_calls_do_not_build_their_message_eagerly` scans the whole litellm package and asserts zero offenders, so this one call has been reddening `misc / Run tests` on litellm_internal_staging for every branch cut from it Passing the reason as a %-style argument defers the interpolation to `record.getMessage()`, which only runs once the record passes the level check --- litellm/proxy/proxy_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index e8299be8f07..1aeb8814585 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6546,7 +6546,8 @@ class ProxyConfig: reload_result = await refetch_model_cost_map(url=model_cost_map_url) if isinstance(reload_result, ModelCostMapReloadUnavailable): verbose_proxy_logger.warning( - f"Model cost map reload failed ({reload_result.reason}); keeping current pricing data, will retry on the next config poll" + "Model cost map reload failed (%s); keeping current pricing data, will retry on the next config poll", + reload_result.reason, ) return new_model_cost_map = reload_result.model_cost_map From 368dd0be5b9f2c8218e3c1d8626f1a82fc32d4e3 Mon Sep 17 00:00:00 2001 From: Ahmed N <34286755+hMED22@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:08:11 +0100 Subject: [PATCH 119/123] fix(groq): translate web_search_options to the browser_search tool (#34971) --- litellm/constants.py | 1 + litellm/llms/__init__.py | 6 + litellm/llms/groq/chat/transformation.py | 66 ++++- litellm/llms/groq/cost_calculator.py | 27 ++ ...odel_prices_and_context_window_backup.json | 15 ++ litellm/types/utils.py | 1 + model_prices_and_context_window.json | 15 ++ tests/test_litellm/llms/groq/__init__.py | 0 tests/test_litellm/llms/groq/chat/__init__.py | 0 .../chat/test_groq_chat_transformation.py | 247 ++++++++++++++++++ .../llms/groq/test_groq_cost_calculator.py | 56 ++++ 11 files changed, 432 insertions(+), 2 deletions(-) create mode 100644 litellm/llms/groq/cost_calculator.py create mode 100644 tests/test_litellm/llms/groq/__init__.py create mode 100644 tests/test_litellm/llms/groq/chat/__init__.py create mode 100644 tests/test_litellm/llms/groq/chat/test_groq_chat_transformation.py create mode 100644 tests/test_litellm/llms/groq/test_groq_cost_calculator.py diff --git a/litellm/constants.py b/litellm/constants.py index 06421e6ed6a..164f5a77a76 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -313,6 +313,7 @@ MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES = int(os.getenv("MAX_LONG_SIDE_FOR_IMAGE_HIGH_R MAX_TILE_WIDTH = int(os.getenv("MAX_TILE_WIDTH", 512)) MAX_TILE_HEIGHT = int(os.getenv("MAX_TILE_HEIGHT", 512)) OPENAI_FILE_SEARCH_COST_PER_1K_CALLS = float(os.getenv("OPENAI_FILE_SEARCH_COST_PER_1K_CALLS", 2.5 / 1000)) +GROQ_BROWSER_VISIT_WEBSITE_COST_PER_CALL = 1.0 / 1000 # Azure OpenAI Assistants feature costs # Source: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/ AZURE_FILE_SEARCH_COST_PER_GB_PER_DAY = float( diff --git a/litellm/llms/__init__.py b/litellm/llms/__init__.py index a35fe5b2093..72c8bf15b47 100644 --- a/litellm/llms/__init__.py +++ b/litellm/llms/__init__.py @@ -55,6 +55,12 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo from .xai.cost_calculator import cost_per_web_search_request return cost_per_web_search_request(usage=usage, model_info=model_info) + elif custom_llm_provider == "groq": + from .groq.cost_calculator import ( + cost_per_web_search_request as groq_cost_per_web_search_request, + ) + + return groq_cost_per_web_search_request(usage=usage, model_info=model_info) else: return None diff --git a/litellm/llms/groq/chat/transformation.py b/litellm/llms/groq/chat/transformation.py index 35a4a14057f..b319d6067aa 100644 --- a/litellm/llms/groq/chat/transformation.py +++ b/litellm/llms/groq/chat/transformation.py @@ -11,7 +11,7 @@ from typing import ( ) import httpx -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_logger @@ -27,10 +27,20 @@ from litellm.types.llms.openai import ( ChatCompletionToolParam, ChatCompletionToolParamFunctionChunk, ) -from litellm.types.utils import ModelResponse, ModelResponseStream +from litellm.types.utils import ModelResponse, ModelResponseStream, ServerToolUse from ...openai_like.chat.transformation import OpenAILikeChatConfig +GROQ_COMPOUND_MODELS = frozenset({"compound", "compound-mini"}) + + +class GroqExecutedToolIdentity(BaseModel): + name: str | None = None + type: str | None = None + + +_EXECUTED_TOOLS_ADAPTER = TypeAdapter(tuple[GroqExecutedToolIdentity, ...]) + class GroqChatConfig(OpenAILikeChatConfig): frequency_penalty: int | None = None @@ -95,6 +105,12 @@ class GroqChatConfig(OpenAILikeChatConfig): except ValueError: pass + if not ( + self._is_compound_model(model) + or litellm.supports_web_search(model=model, custom_llm_provider=self.custom_llm_provider) + ): + base_params.remove("web_search_options") + try: if litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider): base_params.append("reasoning_effort") @@ -103,6 +119,10 @@ class GroqChatConfig(OpenAILikeChatConfig): return base_params + @staticmethod + def _is_compound_model(model: str) -> bool: + return model.removeprefix("groq/") in GROQ_COMPOUND_MODELS + @overload def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: Literal[True] @@ -238,7 +258,23 @@ class GroqChatConfig(OpenAILikeChatConfig): "response_format", None ) # only remove if it's a json_schema - handled via using groq's tool calling params. # else: model supports native json_schema, let response_format pass through + web_search_options = non_default_params.pop("web_search_options", None) optional_params = super().map_openai_params(non_default_params, optional_params, model, drop_params) + if web_search_options is None: + return optional_params + + if web_search_options: + verbose_logger.info( + "Groq web search enabled; ignoring unsupported web_search_options fields: %s", + sorted(web_search_options), + ) + if self._is_compound_model(model): + return optional_params + if not any(tool.get("type") == "browser_search" for tool in optional_params.get("tools") or ()): + optional_params = self._add_tools_to_optional_params( + optional_params=optional_params, + tools=[{"type": "browser_search"}], # mutable-ok: request tools must be json dicts in a list + ) return optional_params @@ -274,8 +310,34 @@ class GroqChatConfig(OpenAILikeChatConfig): original_service_tier=getattr(model_response, "service_tier") ) setattr(model_response, "service_tier", mapped_service_tier) + self._add_web_search_usage(model_response=model_response) return model_response + def _add_web_search_usage(self, model_response: ModelResponse) -> None: + usage = getattr(model_response, "usage", None) + if usage is None: + return + actions = self._executed_tool_actions(model_response) + searches = actions.count("browser.search") + actions.count("browser_search") + opens = actions.count("browser.open") + if searches == 0 and opens == 0: + return + usage.server_tool_use = ServerToolUse(web_search_requests=searches, browser_open_requests=opens) + + @staticmethod + def _executed_tool_actions(model_response: ModelResponse) -> tuple[str | None, ...]: + try: + return tuple( + identity.name or identity.type + for choice in model_response.choices + for identity in _EXECUTED_TOOLS_ADAPTER.validate_python( + getattr(getattr(choice, "message", None), "executed_tools", None) or () + ) + ) + except ValidationError as e: + verbose_logger.info("Groq executed_tools entries did not match the expected shape; not billed: %s", e) + return () + def _map_groq_service_tier(self, original_service_tier: str | None) -> Literal["auto", "default", "flex"]: """ Ensure groq service tier is OpenAI compatible. diff --git a/litellm/llms/groq/cost_calculator.py b/litellm/llms/groq/cost_calculator.py new file mode 100644 index 00000000000..c31dea0fd4d --- /dev/null +++ b/litellm/llms/groq/cost_calculator.py @@ -0,0 +1,27 @@ +""" +Groq-specific cost helpers. + +Groq bills the built-in browser tool per executed action +(https://groq.com/pricing): `browser.search` at $5 per 1k and +`browser.open` at $1 per 1k. The Groq chat transformation counts both +action kinds off `executed_tools` into `usage.server_tool_use`. +""" + +from typing import TYPE_CHECKING + +from litellm.constants import GROQ_BROWSER_VISIT_WEBSITE_COST_PER_CALL +from litellm.types.utils import Usage + +if TYPE_CHECKING: + from litellm.types.utils import ModelInfo + + +def cost_per_web_search_request(usage: Usage, model_info: "ModelInfo") -> float: + search_costs = model_info.get("search_context_cost_per_query") + cost_per_search = search_costs.get("search_context_size_medium", 0.0) if search_costs else 0.0 + server_tool_use = getattr(usage, "server_tool_use", None) + if server_tool_use is None: + return 0.0 + searches = server_tool_use.web_search_requests or 0 + opens = server_tool_use.browser_open_requests or 0 + return searches * cost_per_search + opens * GROQ_BROWSER_VISIT_WEBSITE_COST_PER_CALL diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 848af54cbcf..749663775ea 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -25681,6 +25681,11 @@ "max_tokens": 32766, "mode": "chat", "output_cost_per_token": 6e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.005, + "search_context_size_low": 0.005, + "search_context_size_medium": 0.005 + }, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, @@ -25697,6 +25702,11 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.005, + "search_context_size_low": 0.005, + "search_context_size_medium": 0.005 + }, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, @@ -25713,6 +25723,11 @@ "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.005, + "search_context_size_low": 0.005, + "search_context_size_medium": 0.005 + }, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index c63f4d53572..902253ca3a4 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1587,6 +1587,7 @@ class PromptTokensDetailsWrapper( class ServerToolUse(BaseModel): web_search_requests: Optional[int] = None tool_search_requests: Optional[int] = None + browser_open_requests: Optional[int] = None def __getitem__(self, key: str) -> Optional[int]: if key not in self.__class__.model_fields: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 56e0391f419..8a8bbdad8f1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25756,6 +25756,11 @@ "max_tokens": 32766, "mode": "chat", "output_cost_per_token": 6e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.005, + "search_context_size_low": 0.005, + "search_context_size_medium": 0.005 + }, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, @@ -25772,6 +25777,11 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.005, + "search_context_size_low": 0.005, + "search_context_size_medium": 0.005 + }, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, @@ -25788,6 +25798,11 @@ "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.005, + "search_context_size_low": 0.005, + "search_context_size_medium": 0.005 + }, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, diff --git a/tests/test_litellm/llms/groq/__init__.py b/tests/test_litellm/llms/groq/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/groq/chat/__init__.py b/tests/test_litellm/llms/groq/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/groq/chat/test_groq_chat_transformation.py b/tests/test_litellm/llms/groq/chat/test_groq_chat_transformation.py new file mode 100644 index 00000000000..b2ba919ac7f --- /dev/null +++ b/tests/test_litellm/llms/groq/chat/test_groq_chat_transformation.py @@ -0,0 +1,247 @@ +import logging +from unittest.mock import patch + +import httpx +import pytest + +import litellm +from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( + StandardBuiltInToolCostTracking, +) +from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.llms.groq.chat.transformation import GroqChatConfig +from litellm.utils import get_optional_params + +WEB_SEARCH_MODELS = ( + "openai/gpt-oss-120b", + "openai/gpt-oss-20b", + "openai/gpt-oss-safeguard-20b", +) + +COMPOUND_MODELS = ("compound", "compound-mini", "groq/compound", "groq/compound-mini") + + +@pytest.fixture +def local_model_cost_map(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +class TestGroqWebSearchOptions: + @pytest.mark.parametrize("model", WEB_SEARCH_MODELS + COMPOUND_MODELS) + def test_supported_on_search_capable_models(self, model: str): + assert "web_search_options" in GroqChatConfig().get_supported_openai_params(model) + + def test_not_supported_on_other_models(self): + assert "web_search_options" not in GroqChatConfig().get_supported_openai_params("llama-3.3-70b-versatile") + + @pytest.mark.parametrize("web_search_options", [{"search_context_size": "high"}, {}]) + def test_translates_to_browser_search_tool(self, web_search_options: dict): + optional_params = get_optional_params( + model="openai/gpt-oss-20b", + custom_llm_provider="groq", + web_search_options=web_search_options, + ) + assert optional_params["tools"] == [{"type": "browser_search"}] + assert "web_search_options" not in optional_params + + def test_no_duplicate_browser_search_tool(self): + optional_params = get_optional_params( + model="openai/gpt-oss-20b", + custom_llm_provider="groq", + web_search_options={"search_context_size": "high"}, + tools=[{"type": "browser_search"}], + ) + assert optional_params["tools"] == [{"type": "browser_search"}] + + def test_caller_function_tools_preserved(self): + function_tool = { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, + }, + } + optional_params = get_optional_params( + model="openai/gpt-oss-20b", + custom_llm_provider="groq", + web_search_options={}, + tools=[function_tool], + ) + assert optional_params["tools"] == [function_tool, {"type": "browser_search"}] + + def test_unsupported_model_drops_param_with_drop_params(self): + optional_params = get_optional_params( + model="llama-3.3-70b-versatile", + custom_llm_provider="groq", + web_search_options={"search_context_size": "high"}, + drop_params=True, + ) + assert "web_search_options" not in optional_params + assert "tools" not in optional_params + + def test_unsupported_model_raises_without_drop_params(self): + with pytest.raises(litellm.UnsupportedParamsError): + get_optional_params( + model="llama-3.3-70b-versatile", + custom_llm_provider="groq", + web_search_options={"search_context_size": "high"}, + drop_params=False, + ) + + @pytest.mark.parametrize("model", COMPOUND_MODELS) + def test_compound_injects_no_tool(self, model: str): + optional_params = get_optional_params( + model=model, + custom_llm_provider="groq", + web_search_options={"search_context_size": "high"}, + ) + assert "web_search_options" not in optional_params + assert "tools" not in optional_params + + def test_ignored_fields_logged_as_info(self, caplog: pytest.LogCaptureFixture): + with caplog.at_level(logging.INFO, logger="LiteLLM"): + get_optional_params( + model="openai/gpt-oss-20b", + custom_llm_provider="groq", + web_search_options={"search_context_size": "high", "user_location": {"type": "approximate"}}, + ) + ignored_fields_records = tuple( + record + for record in caplog.records + if "search_context_size" in record.message and "user_location" in record.message + ) + assert len(ignored_fields_records) == 1 + assert ignored_fields_records[0].levelno == logging.INFO + assert "enabled" in ignored_fields_records[0].message + + def test_empty_options_log_nothing(self, caplog: pytest.LogCaptureFixture): + with caplog.at_level(logging.INFO, logger="LiteLLM"): + get_optional_params( + model="openai/gpt-oss-20b", + custom_llm_provider="groq", + web_search_options={}, + ) + assert not [record for record in caplog.records if "web_search_options" in record.message] + + +def _searched_groq_response(executed_tools: list | None) -> dict: + return { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "openai/gpt-oss-20b", + "service_tier": "auto", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Top headline: example", + **({"executed_tools": executed_tools} if executed_tools is not None else {}), + }, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110}, + } + + +EXECUTED_TOOLS_THREE_SEARCHES_TWO_OPENS = [ + {"name": "browser.search", "type": "browser_search"}, + {"name": "browser.open", "type": "function"}, + {"name": "browser.search", "type": "browser_search"}, + {"type": "browser_search"}, + {"name": "browser.open", "type": "browser_search"}, + {"name": "browser.find", "type": "browser.find"}, +] + +EXECUTED_TOOLS_OPENS_ONLY = [ + {"name": "browser.open", "type": "browser.open"}, + {"name": "browser.open", "type": "function"}, + {"name": "browser.find", "type": "browser.find"}, +] + + +def _groq_completion_with_mocked_response(response_json: dict) -> litellm.ModelResponse: + client = HTTPHandler() + fake_response = httpx.Response( + status_code=200, + json=response_json, + request=httpx.Request("POST", "https://api.groq.com/openai/v1/chat/completions"), + ) + with patch.object(client, "post", return_value=fake_response): + return litellm.completion( + model="groq/openai/gpt-oss-20b", + messages=[{"role": "user", "content": "hi"}], + web_search_options={"search_context_size": "high"}, + api_key="fake-key", + client=client, + ) + + +class TestGroqWebSearchUsageSignal: + @pytest.mark.parametrize( + "executed_tools, expected_searches, expected_opens", + [ + (EXECUTED_TOOLS_THREE_SEARCHES_TWO_OPENS, 3, 2), + (EXECUTED_TOOLS_OPENS_ONLY, 0, 2), + ], + ) + def test_counts_actions_into_usage(self, executed_tools: list, expected_searches: int, expected_opens: int): + response = _groq_completion_with_mocked_response(_searched_groq_response(executed_tools)) + assert response.usage.server_tool_use.web_search_requests == expected_searches + assert response.usage.server_tool_use.browser_open_requests == expected_opens + + def test_no_signal_without_executed_tools(self): + response = _groq_completion_with_mocked_response(_searched_groq_response(None)) + assert getattr(response.usage, "server_tool_use", None) is None + + def test_malformed_executed_tools_skips_billing_without_breaking_response(self): + response = _groq_completion_with_mocked_response( + _searched_groq_response(["not-a-dict", {"name": {"nested": "junk"}}]) + ) + assert response.choices[0].message.content == "Top headline: example" + assert getattr(response.usage, "server_tool_use", None) is None + + def test_response_without_usage_is_left_untouched(self): + model_response = litellm.ModelResponse() + GroqChatConfig()._add_web_search_usage(model_response=model_response) + assert getattr(model_response, "usage", None) is None + + @pytest.mark.usefixtures("local_model_cost_map") + @pytest.mark.parametrize( + "executed_tools, expected_cost", + [ + (EXECUTED_TOOLS_THREE_SEARCHES_TWO_OPENS, 3 * 0.005 + 2 * 0.001), + (EXECUTED_TOOLS_OPENS_ONLY, 2 * 0.001), + ], + ) + def test_response_billed_per_action(self, executed_tools: list, expected_cost: float): + response = _groq_completion_with_mocked_response(_searched_groq_response(executed_tools)) + assert StandardBuiltInToolCostTracking.response_object_includes_web_search_call( + response_object=response, usage=response.usage + ) + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model="groq/openai/gpt-oss-20b", + response_object=response, + usage=response.usage, + custom_llm_provider="groq", + standard_built_in_tools_params={"web_search_options": {"search_context_size": "high"}}, + ) + assert cost == pytest.approx(expected_cost) + + +class TestGroqWebSearchCost: + @pytest.mark.usefixtures("local_model_cost_map") + @pytest.mark.parametrize("model", WEB_SEARCH_MODELS) + @pytest.mark.parametrize("search_context_size", ["low", "medium", "high"]) + def test_browser_search_priced_per_search(self, model: str, search_context_size: str): + cost = StandardBuiltInToolCostTracking.get_cost_for_web_search( + web_search_options={"search_context_size": search_context_size}, + model_info=litellm.get_model_info(model=model, custom_llm_provider="groq"), + ) + assert cost == 0.005 diff --git a/tests/test_litellm/llms/groq/test_groq_cost_calculator.py b/tests/test_litellm/llms/groq/test_groq_cost_calculator.py new file mode 100644 index 00000000000..bc3b8058b7f --- /dev/null +++ b/tests/test_litellm/llms/groq/test_groq_cost_calculator.py @@ -0,0 +1,56 @@ +import pytest + +from litellm.llms.groq.cost_calculator import cost_per_web_search_request +from litellm.types.utils import ModelInfo, ServerToolUse, Usage + +PRICED_MODEL_INFO = ModelInfo( + key="groq/openai/gpt-oss-20b", + litellm_provider="groq", + mode="chat", + search_context_cost_per_query={ + "search_context_size_low": 0.005, + "search_context_size_medium": 0.005, + "search_context_size_high": 0.005, + }, +) + + +def _usage_with_actions(searches: int | None, opens: int | None = None) -> Usage: + return Usage( + prompt_tokens=100, + completion_tokens=10, + total_tokens=110, + server_tool_use=ServerToolUse(web_search_requests=searches, browser_open_requests=opens), + ) + + +def test_bills_per_executed_search(): + cost = cost_per_web_search_request(usage=_usage_with_actions(7), model_info=PRICED_MODEL_INFO) + assert cost == pytest.approx(7 * 0.005) + + +def test_no_searches_costs_nothing(): + cost = cost_per_web_search_request(usage=_usage_with_actions(None), model_info=PRICED_MODEL_INFO) + assert cost == 0.0 + + +def test_missing_usage_signal_costs_nothing(): + usage = Usage(prompt_tokens=100, completion_tokens=10, total_tokens=110) + cost = cost_per_web_search_request(usage=usage, model_info=PRICED_MODEL_INFO) + assert cost == 0.0 + + +def test_missing_pricing_costs_nothing(): + unpriced = ModelInfo(key="groq/openai/gpt-oss-20b", litellm_provider="groq", mode="chat") + cost = cost_per_web_search_request(usage=_usage_with_actions(3), model_info=unpriced) + assert cost == 0.0 + + +def test_bills_visit_website_per_open(): + cost = cost_per_web_search_request(usage=_usage_with_actions(0, opens=15), model_info=PRICED_MODEL_INFO) + assert cost == pytest.approx(15 * 0.001) + + +def test_bills_searches_and_opens_together(): + cost = cost_per_web_search_request(usage=_usage_with_actions(2, opens=3), model_info=PRICED_MODEL_INFO) + assert cost == pytest.approx(2 * 0.005 + 3 * 0.001) From 6b3d4f2380543dd590f876388cf96d83c2e629eb Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 4 Aug 2026 09:24:29 -0700 Subject: [PATCH 120/123] feat(ui): add admin-configurable user banner (#35729) * feat(ui): add admin-configurable user banner Proxy admins can publish a markdown announcement that renders as a dismissible banner on every dashboard page for all authenticated users, editable from Admin Settings > UI Settings without a redeploy. Backed by new /get/user_banner and /update/user_banner endpoints persisting to the existing LiteLLM_UISettings table * fix(ui): re-surface dismissed banner on identical republish Stamp a server-side revision on every banner update and fold it into the client dismissal signature, so unpublishing and republishing the same message reaches users who dismissed the earlier run * fix(ui): stamp banner revision as an opaque uuid instead of a counter Two overlapping admin updates could read the same prior revision and both persist the same incremented value, letting an identical republish collide with a previously dismissed signature. A server-generated uuid per update makes every publication identity unique by construction with no read-modify-write * refactor(ui): drop the server-side banner cache Reads go straight to the single-row table; the dashboard already throttles fetches client-side, so the cache only added staleness windows under concurrent updates and multiple workers * refactor(ui): move banner storage behind a domain repository and drop the store_model_in_db gate UserBannerRepository owns the row shape instead of the endpoint reaching through the generic .table bridge, and publishing no longer depends on the unrelated STORE_MODEL_IN_DB flag; a connected database remains the only requirement --- litellm/proxy/_types.py | 1 + litellm/proxy/proxy_server.py | 4 + .../user_banner_endpoints.py | 130 +++++++++++++ .../repositories/user_banner_repository.py | 20 ++ .../proxy/auth/test_route_checks.py | 53 ++++++ .../test_user_banner_endpoints.py | 175 ++++++++++++++++++ .../admin-panel/_components/AdminPanel.tsx | 8 +- .../hooks/userBanner/useUpdateUserBanner.ts | 19 ++ .../hooks/userBanner/useUserBanner.ts | 21 +++ .../src/app/(dashboard)/layout.test.tsx | 4 + .../src/app/(dashboard)/layout.tsx | 3 + .../UserBannerSettings.test.tsx | 85 +++++++++ .../UserBannerSettings/UserBannerSettings.tsx | 152 +++++++++++++++ .../src/components/UserBanner.test.tsx | 102 ++++++++++ .../src/components/UserBanner.tsx | 71 +++++++ .../src/components/networking.tsx | 23 +++ .../src/components/shared/Alert.tsx | 4 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 151 +++++++++++++++ 18 files changed, 1025 insertions(+), 1 deletion(-) create mode 100644 litellm/proxy/ui_crud_endpoints/user_banner_endpoints.py create mode 100644 litellm/repositories/user_banner_repository.py create mode 100644 tests/test_litellm/proxy/ui_crud_endpoints/test_user_banner_endpoints.py create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/userBanner/useUpdateUserBanner.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/userBanner/useUserBanner.ts create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/UserBannerSettings/UserBannerSettings.test.tsx create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/UserBannerSettings/UserBannerSettings.tsx create mode 100644 ui/litellm-dashboard/src/components/UserBanner.test.tsx create mode 100644 ui/litellm-dashboard/src/components/UserBanner.tsx diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 5063bc790b5..e03d2a562e6 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -558,6 +558,7 @@ class LiteLLMRoutes(enum.Enum): "/models", "/v1/models", "/sso/get/ui_settings", + "/get/user_banner", ] # NOTE: ROUTES ONLY FOR MASTER KEY - only the Master Key should be able to Reset Spend diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1aeb8814585..6b2ef736c5b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -538,6 +538,9 @@ from litellm.proxy.types_utils.utils import get_instance_fn from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( router as ui_crud_endpoints_router, ) +from litellm.proxy.ui_crud_endpoints.user_banner_endpoints import ( + router as user_banner_endpoints_router, +) from litellm.proxy.utils import ( PrismaClient, ProxyLogging, @@ -16538,6 +16541,7 @@ app.include_router(callback_management_endpoints_router) app.include_router(debugging_endpoints_router) app.include_router(rust_control_plane_router) app.include_router(ui_crud_endpoints_router) +app.include_router(user_banner_endpoints_router) app.include_router(team_callback_router) app.include_router(budget_management_router) app.include_router(model_management_router) diff --git a/litellm/proxy/ui_crud_endpoints/user_banner_endpoints.py b/litellm/proxy/ui_crud_endpoints/user_banner_endpoints.py new file mode 100644 index 00000000000..cccd72ec7fd --- /dev/null +++ b/litellm/proxy/ui_crud_endpoints/user_banner_endpoints.py @@ -0,0 +1,130 @@ +import asyncio +import json +from typing import Annotated, Literal + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field, ValidationError, model_validator + +from litellm._uuid import uuid4 +from litellm.proxy._types import LitellmTableNames, LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.repositories.user_banner_repository import USER_BANNER_ROW_ID, UserBannerRepository + +router = APIRouter() + +USER_BANNER_MAX_MESSAGE_LENGTH = 4000 + +UserBannerSeverity = Literal["info", "warning", "error"] + + +class UserBannerUpdate(BaseModel): + enabled: bool = Field( + default=False, + description="If true, the banner is shown to all authenticated dashboard users.", + ) + message: str = Field( + default="", + max_length=USER_BANNER_MAX_MESSAGE_LENGTH, + description="Banner text shown to dashboard users. Markdown is supported.", + ) + severity: UserBannerSeverity = Field( + default="info", + description="Visual style of the banner.", + ) + + @model_validator(mode="after") + def _require_message_when_enabled(self) -> "UserBannerUpdate": + if self.enabled and not self.message.strip(): + raise ValueError("message must be non-empty when the banner is enabled") + return self + + +class UserBanner(UserBannerUpdate): + revision: str = Field( + default="", + description=( + "Server-stamped opaque publish identity; a fresh value is generated on every " + "update so clients re-surface dismissed banners on republish." + ), + ) + + +class UpdateUserBannerResponse(BaseModel): + message: str + banner: UserBanner + + +def parse_user_banner(raw_settings: object) -> UserBanner: + if raw_settings is None: + return UserBanner() + try: + parsed = json.loads(raw_settings) if isinstance(raw_settings, str) else raw_settings + return UserBanner.model_validate(parsed) + except (json.JSONDecodeError, ValidationError): + return UserBanner() + + +@router.get( + "/get/user_banner", + tags=["UI Settings"], # mutable-ok: FastAPI's route decorator only accepts a list + dependencies=[Depends(user_api_key_auth)], # mutable-ok: FastAPI's route decorator only accepts a list + response_model=UserBanner, +) +async def get_user_banner() -> UserBanner: + """ + Get the admin-published dashboard banner. + Readable by any authenticated user; rendered on every dashboard page. + """ + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + return UserBanner() + + raw_settings = await UserBannerRepository(prisma_client).get_raw_settings() + return parse_user_banner(raw_settings) + + +@router.patch( + "/update/user_banner", + tags=["UI Settings"], # mutable-ok: FastAPI's route decorator only accepts a list + response_model=UpdateUserBannerResponse, +) +async def update_user_banner( + banner_update: UserBannerUpdate, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> UpdateUserBannerResponse: + """ + Publish, edit, or unpublish the dashboard banner. + Only proxy admins are allowed to modify it. + """ + from litellm.proxy.proxy_server import create_config_audit_log, prisma_client + + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException(status_code=403, detail="Only proxy admins can update the user banner.") + + if prisma_client is None: + raise HTTPException(status_code=500, detail="Database not connected. Please connect a database.") + + repository = UserBannerRepository(prisma_client) + before = parse_user_banner(await repository.get_raw_settings()) + banner = UserBanner( + enabled=banner_update.enabled, + message=banner_update.message, + severity=banner_update.severity, + revision=uuid4().hex, + ) + + await repository.upsert_settings(json.dumps(banner.model_dump())) + + asyncio.create_task( + create_config_audit_log( + param_name=USER_BANNER_ROW_ID, + action="updated", + before_value=before.model_dump(), + after_value=banner.model_dump(), + user_api_key_dict=user_api_key_dict, + table_name=LitellmTableNames.UI_SETTINGS_TABLE_NAME, + ) + ) + + return UpdateUserBannerResponse(message="User banner updated successfully", banner=banner) diff --git a/litellm/repositories/user_banner_repository.py b/litellm/repositories/user_banner_repository.py new file mode 100644 index 00000000000..3ca7e7ceb1c --- /dev/null +++ b/litellm/repositories/user_banner_repository.py @@ -0,0 +1,20 @@ +from litellm.repositories.table_repositories import PrismaTableRepository + +USER_BANNER_ROW_ID = "user_banner" + + +class UserBannerRepository(PrismaTableRepository): + table_name = "litellm_uisettings" + + async def get_raw_settings(self) -> object: + db_record = await self.table.find_unique( + where={"id": USER_BANNER_ROW_ID} # mutable-ok: prisma filters are plain dicts + ) + return db_record.ui_settings if db_record is not None else None + + async def upsert_settings(self, payload: str) -> None: + row = {"id": USER_BANNER_ROW_ID, "ui_settings": payload} # mutable-ok: prisma rows are plain dicts + await self.table.upsert( + where={"id": USER_BANNER_ROW_ID}, # mutable-ok: prisma filters are plain dicts + data={"create": row, "update": {"ui_settings": payload}}, # mutable-ok: prisma payloads are plain dicts + ) diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 1426876783b..87f5187b5a1 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -86,6 +86,59 @@ def test_compliance_routes_open_to_non_admin_roles(role, route): ) +@pytest.mark.parametrize( + "role", + [ + LitellmUserRoles.INTERNAL_USER.value, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value, + ], +) +def test_user_banner_read_open_to_non_admin_roles(role): + """The dashboard banner renders for every authenticated user, so the read + route must be reachable by non-admin roles.""" + user_obj = LiteLLM_UserTable( + user_id="test_user", + user_email="test@example.com", + user_role=role, + ) + valid_token = UserAPIKeyAuth(user_id="test_user", user_role=role) + request = MagicMock(spec=Request) + request.query_params = {} + + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=role, + route="/get/user_banner", + request=request, + valid_token=valid_token, + request_data={}, + ) + + +def test_user_banner_update_rejected_for_non_admin(): + """Publishing the banner stays admin-only at the route layer.""" + user_obj = LiteLLM_UserTable( + user_id="test_user", + user_email="test@example.com", + user_role=LitellmUserRoles.INTERNAL_USER.value, + ) + valid_token = UserAPIKeyAuth(user_id="test_user", user_role=LitellmUserRoles.INTERNAL_USER.value) + request = MagicMock(spec=Request) + request.query_params = {} + + with pytest.raises(Exception) as exc_info: + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=LitellmUserRoles.INTERNAL_USER.value, + route="/update/user_banner", + request=request, + valid_token=valid_token, + request_data={}, + ) + + assert "Route=/update/user_banner" in str(exc_info.value) + + def test_proxy_admin_viewer_config_update_route_rejected(): """Test that proxy admin viewer users are rejected when trying to call /config/update""" diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_user_banner_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_user_banner_endpoints.py new file mode 100644 index 00000000000..5d4875073fd --- /dev/null +++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_user_banner_endpoints.py @@ -0,0 +1,175 @@ +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi.testclient import TestClient + +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.proxy_server import app + +client = TestClient(app) + +PUBLISHED_BANNER = { + "enabled": True, + "message": "**Scheduled maintenance** tonight at 10 PM UTC. See [status](https://status.example.com).", + "severity": "warning", + "revision": "1f2e3d4c5b6a79881f2e3d4c5b6a7988", +} +PUBLISH_BODY = {k: v for k, v in PUBLISHED_BANNER.items() if k != "revision"} +DISABLED_BANNER = {"enabled": False, "message": "", "severity": "info", "revision": ""} + + +def _auth_override(role: LitellmUserRoles): + async def override() -> UserAPIKeyAuth: + return UserAPIKeyAuth(api_key="sk-test", user_id="test-user", user_role=role) + + return override + + +@pytest.fixture +def admin_auth(): + app.dependency_overrides[user_api_key_auth] = _auth_override(LitellmUserRoles.PROXY_ADMIN) + yield + app.dependency_overrides.pop(user_api_key_auth, None) + + +@pytest.fixture +def internal_user_auth(): + app.dependency_overrides[user_api_key_auth] = _auth_override(LitellmUserRoles.INTERNAL_USER) + yield + app.dependency_overrides.pop(user_api_key_auth, None) + + +@pytest.fixture +def mock_audit_log(monkeypatch): + audit_mock = AsyncMock(return_value=None) + monkeypatch.setattr("litellm.proxy.proxy_server.create_config_audit_log", audit_mock) + return audit_mock + + +def _mock_prisma(monkeypatch, record=None): + mock_prisma = MagicMock() + mock_prisma.db.litellm_uisettings.find_unique = AsyncMock(return_value=record) + mock_prisma.db.litellm_uisettings.upsert = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + return mock_prisma + + +class TestGetUserBanner: + def test_requires_auth(self, monkeypatch): + _mock_prisma(monkeypatch) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-1234") + response = client.get("/get/user_banner") + assert response.status_code in (401, 403) + + def test_defaults_when_no_record(self, admin_auth, monkeypatch): + _mock_prisma(monkeypatch, record=None) + response = client.get("/get/user_banner") + assert response.status_code == 200 + assert response.json() == DISABLED_BANNER + + def test_returns_persisted_record(self, internal_user_auth, monkeypatch): + record = SimpleNamespace(ui_settings=json.dumps(PUBLISHED_BANNER)) + _mock_prisma(monkeypatch, record=record) + response = client.get("/get/user_banner") + assert response.status_code == 200 + assert response.json() == PUBLISHED_BANNER + + @pytest.mark.parametrize( + "raw", + [ + "not valid json", + json.dumps({"enabled": True, "message": "hi", "severity": "bogus"}), + json.dumps({"enabled": True, "message": ""}), + ], + ) + def test_corrupt_record_falls_back_to_disabled(self, admin_auth, monkeypatch, raw): + record = SimpleNamespace(ui_settings=raw) + _mock_prisma(monkeypatch, record=record) + response = client.get("/get/user_banner") + assert response.status_code == 200 + assert response.json() == DISABLED_BANNER + + def test_no_database_returns_disabled_banner(self, admin_auth, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + response = client.get("/get/user_banner") + assert response.status_code == 200 + assert response.json() == DISABLED_BANNER + + +class TestUpdateUserBanner: + def test_rejects_non_admin(self, internal_user_auth, monkeypatch): + mock_prisma = _mock_prisma(monkeypatch) + response = client.patch("/update/user_banner", json=PUBLISH_BODY) + assert response.status_code == 403 + mock_prisma.db.litellm_uisettings.upsert.assert_not_awaited() + + def test_persists_and_round_trips(self, admin_auth, monkeypatch, mock_audit_log): + mock_prisma = _mock_prisma(monkeypatch, record=None) + + response = client.patch("/update/user_banner", json=PUBLISH_BODY) + assert response.status_code == 200 + saved = response.json()["banner"] + assert {k: saved[k] for k in PUBLISH_BODY} == PUBLISH_BODY + assert saved["revision"] != "" + + upsert_kwargs = mock_prisma.db.litellm_uisettings.upsert.await_args.kwargs + assert upsert_kwargs["where"] == {"id": "user_banner"} + persisted_payload = upsert_kwargs["data"]["create"]["ui_settings"] + assert json.loads(persisted_payload) == saved + assert json.loads(upsert_kwargs["data"]["update"]["ui_settings"]) == saved + + mock_prisma.db.litellm_uisettings.find_unique = AsyncMock( + return_value=SimpleNamespace(ui_settings=persisted_payload) + ) + read_back = client.get("/get/user_banner") + assert read_back.status_code == 200 + assert read_back.json() == saved + + def test_republish_same_content_gets_fresh_revision(self, admin_auth, monkeypatch, mock_audit_log): + _mock_prisma(monkeypatch, record=None) + + first = client.patch("/update/user_banner", json=PUBLISH_BODY).json()["banner"]["revision"] + second = client.patch("/update/user_banner", json=PUBLISH_BODY).json()["banner"]["revision"] + assert first != "" + assert second != "" + assert first != second + + def test_client_supplied_revision_is_ignored(self, admin_auth, monkeypatch, mock_audit_log): + _mock_prisma(monkeypatch, record=None) + response = client.patch("/update/user_banner", json={**PUBLISH_BODY, "revision": "spoofed"}) + assert response.status_code == 200 + saved_revision = response.json()["banner"]["revision"] + assert saved_revision != "spoofed" + assert saved_revision != "" + + def test_unpublish_with_empty_message_is_allowed(self, admin_auth, monkeypatch, mock_audit_log): + _mock_prisma(monkeypatch, record=SimpleNamespace(ui_settings=json.dumps(PUBLISHED_BANNER))) + response = client.patch( + "/update/user_banner", + json={"enabled": False, "message": "", "severity": "info"}, + ) + assert response.status_code == 200 + saved = response.json()["banner"] + assert {k: saved[k] for k in ("enabled", "message", "severity")} == { + "enabled": False, + "message": "", + "severity": "info", + } + assert saved["revision"] not in ("", PUBLISHED_BANNER["revision"]) + + @pytest.mark.parametrize( + "payload", + [ + {"enabled": True, "message": "hi", "severity": "critical"}, + {"enabled": True, "message": " ", "severity": "info"}, + {"enabled": True, "message": "x" * 4001, "severity": "info"}, + ], + ) + def test_rejects_invalid_payloads(self, admin_auth, monkeypatch, payload): + mock_prisma = _mock_prisma(monkeypatch) + response = client.patch("/update/user_banner", json=payload) + assert response.status_code == 422 + mock_prisma.db.litellm_uisettings.upsert.assert_not_awaited() diff --git a/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx index 611efd6a588..6af9d65b994 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx @@ -24,6 +24,7 @@ import SCIMConfig from "@/components/SCIM"; import LoggingSettings from "@/components/Settings/AdminSettings/LoggingSettings/LoggingSettings"; import SSOSettings from "@/components/Settings/AdminSettings/SSOSettings/SSOSettings"; import UISettings from "@/components/Settings/AdminSettings/UISettings/UISettings"; +import UserBannerSettings from "@/components/Settings/AdminSettings/UserBannerSettings/UserBannerSettings"; import HashicorpVault from "@/components/Settings/AdminSettings/HashicorpVault/HashicorpVault"; import PluginSettings from "@/components/Settings/AdminSettings/PluginSettings/PluginSettings"; import SSOModals from "@/components/SSOModals"; @@ -362,7 +363,12 @@ const AdminPanel: React.FC = ({ proxySettings }) => { ), - children: , + children: ( +
+ + +
+ ), }, { key: "logging-settings", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/userBanner/useUpdateUserBanner.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/userBanner/useUpdateUserBanner.ts new file mode 100644 index 00000000000..b4441f0d09d --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/userBanner/useUpdateUserBanner.ts @@ -0,0 +1,19 @@ +import { updateUserBanner, UserBannerUpdate } from "@/components/networking"; +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { userBannerKeys } from "./useUserBanner"; + +export const useUpdateUserBanner = (accessToken: string | null) => { + const queryClient = useQueryClient(); + + return useMutation({ + mutationFn: async (banner: UserBannerUpdate) => { + if (!accessToken) { + throw new Error("Access token is required"); + } + return await updateUserBanner(accessToken, banner); + }, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: userBannerKeys.all }); + }, + }); +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/userBanner/useUserBanner.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/userBanner/useUserBanner.ts new file mode 100644 index 00000000000..1e407790fe3 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/userBanner/useUserBanner.ts @@ -0,0 +1,21 @@ +import { getUserBanner, UserBanner } from "@/components/networking"; +import { useQuery, UseQueryOptions } from "@tanstack/react-query"; +import { createQueryKeys } from "../common/queryKeysFactory"; + +export const userBannerKeys = createQueryKeys("userBanner"); + +export const useUserBanner = (accessToken: string | null) => { + const queryOptions: UseQueryOptions = { + queryKey: userBannerKeys.list({}), + queryFn: async () => { + if (!accessToken) { + throw new Error("Access token is required"); + } + return await getUserBanner(accessToken); + }, + enabled: Boolean(accessToken), + staleTime: 60 * 1000, + gcTime: 5 * 60 * 1000, + }; + return useQuery(queryOptions); +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx index f08258900eb..7973855ebd4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx @@ -29,6 +29,10 @@ vi.mock("@/components/LicenseExpiryBanner", () => ({ LicenseExpiryBanner: () => null, })); +vi.mock("@/components/UserBanner", () => ({ + UserBanner: () => null, +})); + vi.mock("@/contexts/ThemeContext", () => ({ ThemeProvider: ({ children }: { children: React.ReactNode }) => <>{children}, })); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx index d92aae30c67..fb3a4db58f7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx @@ -10,6 +10,7 @@ import SidebarProvider from "@/app/(dashboard)/components/SidebarProvider"; import { useRouter, useSearchParams, usePathname } from "next/navigation"; import { DebugWarningBanner } from "@/components/DebugWarningBanner"; import { LicenseExpiryBanner } from "@/components/LicenseExpiryBanner"; +import { UserBanner } from "@/components/UserBanner"; import { MIGRATED_PAGES, migratedHref, legacyPageHref, legacyKeyForPathname } from "@/utils/migratedPages"; import { PluginModeProvider, usePluginMode } from "@/contexts/PluginModeContext"; import { createApiClient } from "@/lib/http/client"; @@ -120,6 +121,7 @@ function DashboardShell({ children }: { children: React.ReactNode }) { +
@@ -142,6 +144,7 @@ function DashboardShell({ children }: { children: React.ReactNode }) { +
{children}
diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/UserBannerSettings/UserBannerSettings.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UserBannerSettings/UserBannerSettings.test.tsx new file mode 100644 index 00000000000..9a0e2e72311 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UserBannerSettings/UserBannerSettings.test.tsx @@ -0,0 +1,85 @@ +import { renderWithProviders, screen } from "../../../../../tests/test-utils"; +import { fireEvent } from "@testing-library/react"; +import { vi } from "vitest"; +import UserBannerSettings from "./UserBannerSettings"; +import { UserBanner } from "@/components/networking"; + +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: vi.fn(() => ({ accessToken: "token" })), +})); + +vi.mock("@/app/(dashboard)/hooks/userBanner/useUserBanner", () => ({ + useUserBanner: vi.fn(), +})); + +vi.mock("@/app/(dashboard)/hooks/userBanner/useUpdateUserBanner", () => ({ + useUpdateUserBanner: vi.fn(), +})); + +vi.mock("@/components/molecules/notifications_manager", () => ({ + default: { success: vi.fn(), fromBackend: vi.fn() }, +})); + +import { useUserBanner } from "@/app/(dashboard)/hooks/userBanner/useUserBanner"; +import { useUpdateUserBanner } from "@/app/(dashboard)/hooks/userBanner/useUpdateUserBanner"; + +const publishedBanner: UserBanner = { + enabled: true, + message: "**Maintenance** tonight at 10 PM UTC.", + severity: "warning", + revision: "rev-a", +}; + +const mockHooks = (banner: UserBanner | undefined, mutate = vi.fn()) => { + vi.mocked(useUserBanner).mockReturnValue({ data: banner, isLoading: false } as any); + vi.mocked(useUpdateUserBanner).mockReturnValue({ mutate, isPending: false } as any); + return mutate; +}; + +describe("UserBannerSettings", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("seeds the form from the persisted banner", () => { + mockHooks(publishedBanner); + renderWithProviders(); + expect(screen.getByLabelText("Message")).toHaveValue(publishedBanner.message); + expect(screen.getByRole("switch", { name: "Publish user banner" })).toHaveAttribute("data-checked"); + }); + + it("shows a live markdown preview with the selected severity icon", () => { + mockHooks(publishedBanner); + const { container } = renderWithProviders(); + expect(screen.getByText("Maintenance")).toBeInTheDocument(); + expect(container.querySelector(".lucide-triangle-alert")).toBeInTheDocument(); + }); + + it("saves the edited draft", () => { + const mutate = mockHooks(publishedBanner); + renderWithProviders(); + fireEvent.change(screen.getByLabelText("Message"), { target: { value: "New announcement" } }); + fireEvent.click(screen.getByRole("button", { name: "Save banner" })); + expect(mutate).toHaveBeenCalledWith( + { enabled: true, message: "New announcement", severity: "warning" }, + expect.anything(), + ); + }); + + it("blocks saving a published banner with an empty message", () => { + const mutate = mockHooks(publishedBanner); + renderWithProviders(); + fireEvent.change(screen.getByLabelText("Message"), { target: { value: " " } }); + expect(screen.getByText("Add a message before publishing.")).toBeInTheDocument(); + const saveButton = screen.getByRole("button", { name: "Save banner" }); + fireEvent.click(saveButton); + expect(mutate).not.toHaveBeenCalled(); + }); + + it("allows unpublishing without a message", () => { + const mutate = mockHooks({ enabled: false, message: "", severity: "info", revision: "" }); + renderWithProviders(); + fireEvent.click(screen.getByRole("button", { name: "Save banner" })); + expect(mutate).toHaveBeenCalledWith({ enabled: false, message: "", severity: "info" }, expect.anything()); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/UserBannerSettings/UserBannerSettings.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UserBannerSettings/UserBannerSettings.tsx new file mode 100644 index 00000000000..83207f89160 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UserBannerSettings/UserBannerSettings.tsx @@ -0,0 +1,152 @@ +"use client"; + +import React, { useState } from "react"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { useUpdateUserBanner } from "@/app/(dashboard)/hooks/userBanner/useUpdateUserBanner"; +import { useUserBanner } from "@/app/(dashboard)/hooks/userBanner/useUserBanner"; +import NotificationManager from "@/components/molecules/notifications_manager"; +import { UserBanner, UserBannerSeverity, UserBannerUpdate } from "@/components/networking"; +import { Alert, AlertDescription } from "@/components/shared/Alert"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Label } from "@/components/ui/label"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Switch } from "@/components/ui/switch"; +import { Textarea } from "@/components/ui/textarea"; +import { SEVERITY_ICONS, UserBannerMarkdown } from "@/components/UserBanner"; +import { Skeleton } from "@/components/ui/skeleton"; + +const SEVERITY_LABELS: Record = { + info: "Info", + warning: "Warning", + error: "Error", +}; + +const EMPTY_BANNER: UserBanner = { enabled: false, message: "", severity: "info", revision: "" }; + +export default function UserBannerSettings() { + const { accessToken } = useAuthorized(); + const { data: banner, isLoading } = useUserBanner(accessToken); + const { mutate: saveBanner, isPending } = useUpdateUserBanner(accessToken); + const persisted = banner ?? EMPTY_BANNER; + + return ( + + ); +} + +interface UserBannerSettingsFormProps { + persisted: UserBanner; + isLoading: boolean; + isPending: boolean; + saveBanner: ReturnType["mutate"]; +} + +function UserBannerSettingsForm({ persisted, isLoading, isPending, saveBanner }: UserBannerSettingsFormProps) { + const [draft, setDraft] = useState({ + enabled: persisted.enabled, + message: persisted.message, + severity: persisted.severity, + }); + + const messageMissing = draft.enabled && draft.message.trim() === ""; + + const handleSave = () => { + saveBanner(draft, { + onSuccess: () => { + NotificationManager.success("User banner updated successfully"); + }, + onError: (error) => { + NotificationManager.fromBackend(error); + }, + }); + }; + + return ( + + + User Banner + + Publish an announcement to all dashboard users. Markdown is supported; the banner appears below the header on + every page until you unpublish it. Users can dismiss it, and it reappears whenever the content changes. + + + + {isLoading ? ( + + ) : ( +
+
+ setDraft({ ...draft, enabled: checked })} + aria-label="Publish user banner" + /> + +
+ +
+ +