From ed5dc6a0776ce96c6a6cfe990395f3aca44106d5 Mon Sep 17 00:00:00 2001 From: Alexandros Solanos Date: Tue, 16 Dec 2025 15:28:35 +0100 Subject: [PATCH 01/66] Improve model repetition detection performance --- .../litellm_core_utils/streaming_handler.py | 52 ++++---- .../test_streaming_handler.py | 117 ++++++++++++++++++ 2 files changed, 147 insertions(+), 22 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index d92af417175..b8240839d75 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -145,6 +145,7 @@ class CustomStreamWrapper: self.chunks: List = ( [] ) # keep track of the returned chunks - used for calculating the input/output tokens for stream options + self._repeated_messages_count = 1 self.is_function_call = self.check_is_function_call(logging_obj=logging_obj) self.created: Optional[int] = None @@ -190,7 +191,7 @@ class CustomStreamWrapper: except Exception as e: raise e - def safety_checker(self) -> None: + def raise_on_model_repetition(self) -> None: """ Fixes - https://github.com/BerriAI/litellm/issues/5158 @@ -198,29 +199,36 @@ class CustomStreamWrapper: Raises - InternalServerError, if LLM enters infinite loop while streaming """ - if len(self.chunks) >= litellm.REPEATED_STREAMING_CHUNK_LIMIT: - # Get the last n chunks - last_chunks = self.chunks[-litellm.REPEATED_STREAMING_CHUNK_LIMIT :] + if len(self.chunks) < 2: + return - # Extract the relevant content from the chunks - last_contents = [chunk.choices[0].delta.content for chunk in last_chunks] + last_content = self.chunks[-1].choices[0].delta.content - # Check if all extracted contents are identical - if all(content == last_contents[0] for content in last_contents): - if ( - last_contents[0] is not None - and isinstance(last_contents[0], str) - and len(last_contents[0]) > 2 - ): # ignore empty content - https://github.com/BerriAI/litellm/issues/5158#issuecomment-2287156946 - # All last n chunks are identical - raise litellm.InternalServerError( - message="The model is repeating the same chunk = {}.".format( - last_contents[0] - ), - model="", - llm_provider="", - ) + if ( + last_content is None + or not isinstance(last_content, str) + or len(last_content) <= 2 + ): # ignore empty content - https://github.com/BerriAI/litellm/issues/5158#issuecomment-2287156946 + self._repeated_messages_count = 1 + return + second_to_last_content = self.chunks[-2].choices[0].delta.content + + if last_content == second_to_last_content: + self._repeated_messages_count += 1 + else: + self._repeated_messages_count = 1 + + if self._repeated_messages_count >= litellm.REPEATED_STREAMING_CHUNK_LIMIT: + # All last n chunks are identical + raise litellm.InternalServerError( + message="The model is repeating the same chunk = {}.".format( + last_content + ), + model="", + llm_provider="", + ) + def check_special_tokens(self, chunk: str, finish_reason: Optional[str]): """ Output parse / special tokens for sagemaker + hf streaming. @@ -879,7 +887,7 @@ class CustomStreamWrapper: if ( is_chunk_non_empty ): # cannot set content of an OpenAI Object to be an empty string - self.safety_checker() + self.raise_on_model_repetition() hold, model_response_str = self.check_special_tokens( chunk=completion_obj["content"], finish_reason=model_response.choices[0].finish_reason, diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 6a528fef8f0..fcce6ddc67f 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -1162,3 +1162,120 @@ def test_is_chunk_non_empty_with_valid_tool_calls( ) is True ) + + +def _make_chunk(content: Optional[str]) -> ModelResponseStream: + return ModelResponseStream( + id="test", + created=1741037890, + model="test-model", + choices=[StreamingChoices(index=0, delta=Delta(content=content))], + ) + + +def _build_chunks(pattern: list[str], N: int) -> list[ModelResponseStream]: + """ + Build a list of chunks based on a pattern specification. + """ + chunks = [] + for i, p in enumerate(pattern): + if p == "same": + chunks.append(_make_chunk("same_chunk")) + elif p == "diff": + chunks.append(_make_chunk(f"chunk_{i}")) + else: + chunks.append(_make_chunk(p)) + return chunks + +_REPETITION_TEST_CASES = [ + # Basic cases + pytest.param( + ["same"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + True, + id="all_identical_raises", + ), + pytest.param( + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT - 1), + False, + id="below_threshold_no_raise", + ), + pytest.param( + [None] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + False, + id="none_content_no_raise", + ), + pytest.param( + [""] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + False, + id="empty_content_no_raise", + ), + # Short content (len <= 2) should not raise + pytest.param( + ["##"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + False, + id="short_content_2chars_no_raise", + ), + pytest.param( + ["{"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + False, + id="short_content_1char_no_raise", + ), + pytest.param( + ["ab"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + False, + id="short_content_2chars_ab_no_raise", + ), + # All different chunks + pytest.param( + ["diff"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + False, + id="all_different_no_raise", + ), + # One chunk different at various positions + pytest.param( + ["different_first"] + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT - 1), + False, + id="first_chunk_different_no_raise", + ), + pytest.param( + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT - 1) + ["different_last"], + False, + id="last_chunk_different_no_raise", + ), + pytest.param( + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT // 2 + 1) + ["different_mid"] + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT - litellm.REPEATED_STREAMING_CHUNK_LIMIT // 2 + 1), + False, + id="middle_chunk_different_no_raise", + ), + pytest.param( + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT - 2) + ["diff", "diff"], + False, + id="last_two_different_no_raise", + ), + pytest.param( + ["diff"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT + ["same"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT + ["diff"], + True, + id="in_between_same_and_diff_raise", + ), +] + + +@pytest.mark.parametrize("chunks_pattern,should_raise", _REPETITION_TEST_CASES) +def test_raise_on_model_repetition( + initialized_custom_stream_wrapper: CustomStreamWrapper, + chunks_pattern: list, + should_raise: bool, +): + wrapper = initialized_custom_stream_wrapper + chunks = _build_chunks(chunks_pattern, len(chunks_pattern)) + + if should_raise: + with pytest.raises(litellm.InternalServerError) as exc_info: + for chunk in chunks: + wrapper.chunks.append(chunk) + wrapper.raise_on_model_repetition() + assert "repeating the same chunk" in str(exc_info.value) + else: + for chunk in chunks: + wrapper.chunks.append(chunk) + wrapper.raise_on_model_repetition() \ No newline at end of file From 19efe556cbd5e52f4ad68414400b9e88f86706de Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 18 Mar 2026 08:41:32 +0000 Subject: [PATCH 02/66] fix: /key/block and /key/unblock return 404 instead of misleading 401 for non-existent keys The block_key() and unblock_key() handlers previously returned a misleading 401 'Authentication Error' when the body 'key' didn't exist in the database, even though authentication (via Authorization header) succeeded correctly. Root cause: After auth passed, the handlers called get_key_object() for cache refresh. This function was designed for auth token lookup and raises ProxyException(code=401) when a token isn't found. Additionally, Prisma's update() silently returns None for non-existent records instead of raising an error, so the code reached get_key_object() without detecting the missing key. Fix: - Add an explicit existence check (find_unique) before the update - Return 404 ProxyException with 'Key not found' if the key doesn't exist - Replace get_key_object() + manual cache update with _delete_cache_key_object() to invalidate the cache (next read will re-fetch from DB) - Reuse the find_unique result for audit logs, eliminating duplicate queries Co-authored-by: yuneng-jiang --- dev_config.yaml | 9 +- .../key_management_endpoints.py | 90 +++++++------------ 2 files changed, 33 insertions(+), 66 deletions(-) diff --git a/dev_config.yaml b/dev_config.yaml index 64e3c14703e..142e0bf94e7 100644 --- a/dev_config.yaml +++ b/dev_config.yaml @@ -1,13 +1,8 @@ model_list: - - model_name: fake-openai-endpoint + - model_name: gpt-4 litellm_params: - model: openai/fake-model + model: gpt-4 api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ general_settings: master_key: sk-1234 - -litellm_settings: - drop_params: True - telemetry: False diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 1c0c212b60b..6dd0d4137be 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -4788,18 +4788,19 @@ async def block_key( else: hashed_token = data.key - if litellm.store_audit_logs is True: - # make an audit log for key update - record = await prisma_client.db.litellm_verificationtoken.find_unique( - where={"token": hashed_token} + # Check if the key exists before trying to block it + existing_record = await prisma_client.db.litellm_verificationtoken.find_unique( + where={"token": hashed_token} + ) + if existing_record is None: + raise ProxyException( + message=f"Key not found. Passed key={data.key}", + type=ProxyErrorTypes.not_found_error, + param="key", + code=status.HTTP_404_NOT_FOUND, ) - if record is None: - raise ProxyException( - message=f"Key {data.key} not found", - type=ProxyErrorTypes.bad_request_error, - param="key", - code=status.HTTP_404_NOT_FOUND, - ) + + if litellm.store_audit_logs is True: asyncio.create_task( create_audit_log_for_update( request_data=LiteLLM_AuditLogs( @@ -4813,7 +4814,7 @@ async def block_key( object_id=hashed_token, action="blocked", updated_values="{}", - before_value=record.model_dump_json(), + before_value=existing_record.model_dump_json(), ) ) ) @@ -4822,24 +4823,9 @@ async def block_key( where={"token": hashed_token}, data={"blocked": True} # type: ignore ) - ## UPDATE KEY CACHE - - ### get cached object ### - key_object = await get_key_object( + ## UPDATE KEY CACHE - invalidate so next read re-fetches from DB + await _delete_cache_key_object( hashed_token=hashed_token, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - parent_otel_span=None, - proxy_logging_obj=proxy_logging_obj, - ) - - ### update cached object ### - key_object.blocked = True - - ### store cached object ### - await _cache_key_object( - hashed_token=hashed_token, - user_api_key_obj=key_object, user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) @@ -4902,18 +4888,19 @@ async def unblock_key( else: hashed_token = data.key - if litellm.store_audit_logs is True: - # make an audit log for key update - record = await prisma_client.db.litellm_verificationtoken.find_unique( - where={"token": hashed_token} + # Check if the key exists before trying to unblock it + existing_record = await prisma_client.db.litellm_verificationtoken.find_unique( + where={"token": hashed_token} + ) + if existing_record is None: + raise ProxyException( + message=f"Key not found. Passed key={data.key}", + type=ProxyErrorTypes.not_found_error, + param="key", + code=status.HTTP_404_NOT_FOUND, ) - if record is None: - raise ProxyException( - message=f"Key {data.key} not found", - type=ProxyErrorTypes.bad_request_error, - param="key", - code=status.HTTP_404_NOT_FOUND, - ) + + if litellm.store_audit_logs is True: asyncio.create_task( create_audit_log_for_update( request_data=LiteLLM_AuditLogs( @@ -4925,9 +4912,9 @@ async def unblock_key( changed_by_api_key=user_api_key_dict.api_key, table_name=LitellmTableNames.KEY_TABLE_NAME, object_id=hashed_token, - action="blocked", + action="unblocked", updated_values="{}", - before_value=record.model_dump_json(), + before_value=existing_record.model_dump_json(), ) ) ) @@ -4936,24 +4923,9 @@ async def unblock_key( where={"token": hashed_token}, data={"blocked": False} # type: ignore ) - ## UPDATE KEY CACHE - - ### get cached object ### - key_object = await get_key_object( + ## UPDATE KEY CACHE - invalidate so next read re-fetches from DB + await _delete_cache_key_object( hashed_token=hashed_token, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - parent_otel_span=None, - proxy_logging_obj=proxy_logging_obj, - ) - - ### update cached object ### - key_object.blocked = False - - ### store cached object ### - await _cache_key_object( - hashed_token=hashed_token, - user_api_key_obj=key_object, user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) From b428cfb4a4b249768ef29c9ba6cf1c1134cef52e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 18 Mar 2026 08:44:22 +0000 Subject: [PATCH 03/66] test: add unit tests for block_key/unblock_key with non-existent keys - test_block_key_nonexistent_key_returns_404: verifies block_key returns 404 (not misleading 401) when the key doesn't exist in the DB - test_unblock_key_nonexistent_key_returns_404: same for unblock_key - test_block_key_existing_key_succeeds: verifies block_key succeeds and invalidates cache for existing keys - Update test_unblock_key_supports_both_sk_and_hashed_tokens to reflect the new cache invalidation pattern (_delete_cache_key_object instead of get_key_object + _cache_key_object) Co-authored-by: yuneng-jiang --- .../test_key_management_endpoints.py | 208 +++++++++++++++++- 1 file changed, 196 insertions(+), 12 deletions(-) diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index cfc16808afb..49be6ce6bb5 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -1338,19 +1338,12 @@ async def test_unblock_key_supports_both_sk_and_hashed_tokens(monkeypatch): ) # Disable audit logs for simpler test # Mock get_key_object and _cache_key_object - async def mock_get_key_object(**kwargs): - return mock_key_object - - async def mock_cache_key_object(**kwargs): + async def mock_delete_cache_key_object(**kwargs): pass monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_key_object", - mock_get_key_object, - ) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints._cache_key_object", - mock_cache_key_object, + "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object", + mock_delete_cache_key_object, ) # Create mock request and user auth @@ -1375,11 +1368,9 @@ async def test_unblock_key_supports_both_sk_and_hashed_tokens(monkeypatch): ) assert result == mock_key_record - assert mock_key_object.blocked == False # Should be updated to unblocked # Reset mocks for second test mock_prisma_client.db.litellm_verificationtoken.update.reset_mock() - mock_key_object.blocked = True # Reset to blocked state # Test Case 2: Using already hashed token hashed_token_request = BlockKeyRequest(key=test_hashed_token) @@ -1435,6 +1426,199 @@ async def test_unblock_key_invalid_key_format(monkeypatch): assert "Invalid key format" in str(exc_info.value.message) +@pytest.mark.asyncio +async def test_block_key_nonexistent_key_returns_404(monkeypatch): + """ + Test that block_key returns 404 (not misleading 401) when the key + doesn't exist in the database, even when the caller is authenticated + as a proxy admin. + + Previously, block_key would call get_key_object() for cache refresh, + which raised a 401 ProxyException with 'Authentication Error' — making + it look like an auth failure when it was really a missing-key error. + """ + from litellm.proxy._types import BlockKeyRequest + from litellm.proxy.management_endpoints.key_management_endpoints import block_key + + mock_prisma_client = AsyncMock() + mock_user_api_key_cache = MagicMock() + mock_proxy_logging_obj = MagicMock() + + # find_unique returns None → key does not exist + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=None + ) + + def mock_hash_token(token): + return "abcd1234" * 8 # 64-char hex + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr( + "litellm.proxy.proxy_server.user_api_key_cache", mock_user_api_key_cache + ) + monkeypatch.setattr( + "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging_obj + ) + monkeypatch.setattr("litellm.proxy.proxy_server.hash_token", mock_hash_token) + monkeypatch.setattr("litellm.store_audit_logs", False) + + mock_request = MagicMock() + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-admin", user_id="admin_user" + ) + + data = BlockKeyRequest(key="sk-does-not-exist-key") + + with pytest.raises(ProxyException) as exc_info: + await block_key( + data=data, + http_request=mock_request, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + + assert exc_info.value.code == "404" + assert "not found" in str(exc_info.value.message).lower() + # Must NOT contain "Authentication Error" + assert "Authentication Error" not in str(exc_info.value.message) + # update should never be called since the key doesn't exist + mock_prisma_client.db.litellm_verificationtoken.update.assert_not_called() + + +@pytest.mark.asyncio +async def test_unblock_key_nonexistent_key_returns_404(monkeypatch): + """ + Test that unblock_key returns 404 (not misleading 401) when the key + doesn't exist in the database. + """ + from litellm.proxy._types import BlockKeyRequest + from litellm.proxy.management_endpoints.key_management_endpoints import ( + unblock_key, + ) + + mock_prisma_client = AsyncMock() + mock_user_api_key_cache = MagicMock() + mock_proxy_logging_obj = MagicMock() + + # find_unique returns None → key does not exist + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=None + ) + + def mock_hash_token(token): + return "abcd1234" * 8 + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr( + "litellm.proxy.proxy_server.user_api_key_cache", mock_user_api_key_cache + ) + monkeypatch.setattr( + "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging_obj + ) + monkeypatch.setattr("litellm.proxy.proxy_server.hash_token", mock_hash_token) + monkeypatch.setattr("litellm.store_audit_logs", False) + + mock_request = MagicMock() + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-admin", user_id="admin_user" + ) + + data = BlockKeyRequest(key="sk-does-not-exist-key") + + with pytest.raises(ProxyException) as exc_info: + await unblock_key( + data=data, + http_request=mock_request, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + + assert exc_info.value.code == "404" + assert "not found" in str(exc_info.value.message).lower() + assert "Authentication Error" not in str(exc_info.value.message) + mock_prisma_client.db.litellm_verificationtoken.update.assert_not_called() + + +@pytest.mark.asyncio +async def test_block_key_existing_key_succeeds(monkeypatch): + """ + Test that block_key successfully blocks an existing key and + invalidates the cache entry. + """ + from litellm.proxy._types import BlockKeyRequest + from litellm.proxy.management_endpoints.key_management_endpoints import block_key + + mock_prisma_client = AsyncMock() + mock_user_api_key_cache = MagicMock() + mock_proxy_logging_obj = MagicMock() + + test_hashed_token = "a1b2c3d4e5f6789012345678901234567890123456789012345678901234abcd" + + mock_key_record = MagicMock() + mock_key_record.token = test_hashed_token + mock_key_record.blocked = False + mock_key_record.model_dump_json.return_value = ( + f'{{"token": "{test_hashed_token}", "blocked": false}}' + ) + + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=mock_key_record + ) + mock_updated_record = MagicMock() + mock_updated_record.token = test_hashed_token + mock_updated_record.blocked = True + mock_prisma_client.db.litellm_verificationtoken.update = AsyncMock( + return_value=mock_updated_record + ) + + def mock_hash_token(token): + if token.startswith("sk-"): + return test_hashed_token + return token + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr( + "litellm.proxy.proxy_server.user_api_key_cache", mock_user_api_key_cache + ) + monkeypatch.setattr( + "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging_obj + ) + monkeypatch.setattr("litellm.proxy.proxy_server.hash_token", mock_hash_token) + monkeypatch.setattr("litellm.store_audit_logs", False) + + # Mock _delete_cache_key_object + async def mock_delete_cache_key_object(**kwargs): + pass + + monkeypatch.setattr( + "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object", + mock_delete_cache_key_object, + ) + + mock_request = MagicMock() + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-admin", user_id="admin_user" + ) + + data = BlockKeyRequest(key="sk-test123456789") + + result = await block_key( + data=data, + http_request=mock_request, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + + # Verify the key was found and updated + mock_prisma_client.db.litellm_verificationtoken.find_unique.assert_called_once_with( + where={"token": test_hashed_token} + ) + mock_prisma_client.db.litellm_verificationtoken.update.assert_called_once_with( + where={"token": test_hashed_token}, data={"blocked": True} + ) + assert result == mock_updated_record + + @pytest.mark.asyncio async def test_validate_key_team_change_with_member_permissions(): """ From 5e7645a99b6c432cdafd7cc9a21be3851bd327c2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 18 Mar 2026 08:47:44 +0000 Subject: [PATCH 04/66] chore: remove unused imports (get_key_object, _cache_key_object) These were only used in block_key/unblock_key for cache refresh, which now uses _delete_cache_key_object instead. Co-authored-by: yuneng-jiang --- .../proxy/management_endpoints/key_management_endpoints.py | 2 -- .../management_endpoints/test_key_management_endpoints.py | 5 ----- 2 files changed, 7 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 6dd0d4137be..55def0008b7 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -41,10 +41,8 @@ from litellm.proxy._experimental.mcp_server.db import ( from litellm.proxy._types import * from litellm.proxy._types import LiteLLM_VerificationToken from litellm.proxy.auth.auth_checks import ( - _cache_key_object, _delete_cache_key_object, can_team_access_model, - get_key_object, get_org_object, get_project_object, get_team_object, diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 49be6ce6bb5..664a08989e9 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -1314,10 +1314,6 @@ async def test_unblock_key_supports_both_sk_and_hashed_tokens(monkeypatch): return_value=mock_key_record ) - # Mock get_key_object and _cache_key_object functions - mock_key_object = MagicMock() - mock_key_object.blocked = True # Initially blocked - # Mock hash_token function def mock_hash_token(token): if token == "sk-test123456789": @@ -1388,7 +1384,6 @@ async def test_unblock_key_supports_both_sk_and_hashed_tokens(monkeypatch): ) assert result == mock_key_record - assert mock_key_object.blocked == False # Should be updated to unblocked @pytest.mark.asyncio From 3f7f23cd3c0a26203b7de2fdf8b9a17640dfb79b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 18 Mar 2026 08:51:12 +0000 Subject: [PATCH 05/66] chore: restore original dev_config.yaml Co-authored-by: yuneng-jiang --- dev_config.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dev_config.yaml b/dev_config.yaml index 142e0bf94e7..64e3c14703e 100644 --- a/dev_config.yaml +++ b/dev_config.yaml @@ -1,8 +1,13 @@ model_list: - - model_name: gpt-4 + - model_name: fake-openai-endpoint litellm_params: - model: gpt-4 + model: openai/fake-model api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ general_settings: master_key: sk-1234 + +litellm_settings: + drop_params: True + telemetry: False From 92b89353ae6017bcc2cec821c4a9c6a71c6e0da5 Mon Sep 17 00:00:00 2001 From: Andrzej Pomirski Date: Mon, 16 Mar 2026 23:22:00 +0100 Subject: [PATCH 06/66] fix: surface Anthropic code execution results as code_interpreter_call in Responses API PR #18945 added support for capturing Anthropic server-side tool results (bash_code_execution_tool_result, etc.) in provider_specific_fields, but the data never reached the Responses API output because: 1. Non-streaming: provider_specific_fields wasn't copied into _hidden_params 2. Streaming: chunk delta's provider_specific_fields wasn't accumulated 3. Tool results weren't mapped to standard output items This fix: - Copies provider_specific_fields to _hidden_params in transform_response() - Accumulates provider_specific_fields from streaming chunk deltas - Maps bash_code_execution_tool_result to code_interpreter_call output items with code and outputs (matching OpenAI's native shape) - Removes redundant function_call items for server-side tools - Adds OutputCodeInterpreterCall type to the output union --- litellm/llms/anthropic/chat/handler.py | 86 +- litellm/llms/anthropic/chat/transformation.py | 79 +- .../streaming_iterator.py | 52 +- .../transformation.py | 57 +- litellm/types/llms/openai.py | 55 +- litellm/types/responses/main.py | 18 + .../chat/test_anthropic_chat_handler.py | 247 ++++- .../test_anthropic_chat_transformation.py | 889 +++++++++--------- 8 files changed, 971 insertions(+), 512 deletions(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 5eebebc2e23..51b9c9835a7 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -48,6 +48,10 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ) +from litellm.types.responses.main import ( + OutputCodeInterpreterCall, + OutputCodeInterpreterCallLog, +) from litellm.types.utils import ( Delta, GenericStreamingChunk, @@ -538,6 +542,11 @@ class ModelResponseIterator: # Accumulate compaction blocks for multi-turn reconstruction self.compaction_blocks: List[Dict[str, Any]] = [] + # Track server tool use inputs and results for code_interpreter_results + self._server_tool_inputs: Dict[str, Any] = {} + self.tool_results: List[Dict[str, Any]] = [] + self._last_code_interpreter_results_count: int = 0 + def check_empty_tool_call_args(self) -> bool: """ Check if the tool call block so far has been an empty string @@ -568,9 +577,7 @@ class ModelResponseIterator: speed=self.speed, ) - def _content_block_delta_helper( - self, chunk: dict - ) -> Tuple[ + def _content_block_delta_helper(self, chunk: dict) -> Tuple[ str, Optional[ChatCompletionToolCallChunk], List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]], @@ -682,6 +689,44 @@ class ModelResponseIterator: return content_block_start + def _build_code_interpreter_results(self) -> list: + """Convert accumulated tool_results to OutputCodeInterpreterCall objects. + + Called during streaming to produce provider-neutral code_interpreter_results + alongside the raw tool_results, so the Responses API layer doesn't need + Anthropic-specific knowledge. + """ + # Only convert tool_results added since the last call to avoid + # duplicates when _merge_provider_specific_fields extends the list. + new_results = self.tool_results[self._last_code_interpreter_results_count :] + self._last_code_interpreter_results_count = len(self.tool_results) + results = [] + for tr in new_results: + call_id = tr.get("tool_use_id", "") + content = tr.get("content", {}) + if isinstance(content, dict): + parts = [] + if content.get("stdout"): + parts.append(content["stdout"]) + if content.get("stderr"): + parts.append(f"STDERR: {content['stderr']}") + logs = "".join(parts) if parts else str(content) + else: + logs = str(content) + tool_input = self._server_tool_inputs.get(call_id, {}) + code = tool_input.get("command", "") if isinstance(tool_input, dict) else "" + results.append( + OutputCodeInterpreterCall( + type="code_interpreter_call", + id=call_id, + code=code, + container_id=None, + status="completed", + outputs=[OutputCodeInterpreterCallLog(type="logs", logs=logs)], + ) + ) + return results + def chunk_parser(self, chunk: dict) -> ModelResponseStream: # noqa: PLR0915 try: type_chunk = chunk.get("type", "") or "" @@ -748,6 +793,17 @@ class ModelResponseIterator: ), index=self.tool_index, ) + # Track server tool use inputs for code_interpreter_results + if ( + content_block_start["content_block"]["type"] + == "server_tool_use" + ): + tool_input = content_block_start["content_block"].get( + "input", {} + ) + self._server_tool_inputs[ + content_block_start["content_block"]["id"] + ] = tool_input # Include caller information if present (for programmatic tool calling) if "caller" in content_block_start["content_block"]: caller_data = content_block_start["content_block"]["caller"] @@ -768,9 +824,9 @@ class ModelResponseIterator: # Handle compaction blocks # The full content comes in content_block_start self.compaction_blocks.append(content_block_start["content_block"]) - provider_specific_fields[ - "compaction_blocks" - ] = self.compaction_blocks + provider_specific_fields["compaction_blocks"] = ( + self.compaction_blocks + ) provider_specific_fields["compaction_start"] = { "type": "compaction", "content": content_block_start["content_block"].get( @@ -792,9 +848,9 @@ class ModelResponseIterator: self.web_search_results.append( content_block_start["content_block"] ) - provider_specific_fields[ - "web_search_results" - ] = self.web_search_results + provider_specific_fields["web_search_results"] = ( + self.web_search_results + ) elif content_type == "web_fetch_tool_result": # Capture web_fetch_tool_result for multi-turn reconstruction # The full content comes in content_block_start, not in deltas @@ -802,16 +858,18 @@ class ModelResponseIterator: self.web_search_results.append( content_block_start["content_block"] ) - provider_specific_fields[ - "web_search_results" - ] = self.web_search_results + provider_specific_fields["web_search_results"] = ( + self.web_search_results + ) elif content_type != "tool_search_tool_result": # Handle other tool results (code execution, etc.) # Skip tool_search_tool_result as it's internal metadata - if not hasattr(self, "tool_results"): - self.tool_results = [] self.tool_results.append(content_block_start["content_block"]) provider_specific_fields["tool_results"] = self.tool_results + # Convert to provider-neutral code_interpreter_results + provider_specific_fields["code_interpreter_results"] = ( + self._build_code_interpreter_results() + ) elif type_chunk == "content_block_stop": ContentBlockStop(**chunk) # type: ignore diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 47cdd8287e0..033afea2ffa 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -59,6 +59,10 @@ from litellm.types.utils import ( PromptTokensDetailsWrapper, ServerToolUse, ) +from litellm.types.responses.main import ( + OutputCodeInterpreterCall, + OutputCodeInterpreterCallLog, +) from litellm.utils import ( ModelResponse, Usage, @@ -960,11 +964,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if mcp_servers: optional_params["mcp_servers"] = mcp_servers elif param == "tool_choice" or param == "parallel_tool_calls": - _tool_choice: Optional[ - AnthropicMessagesToolChoice - ] = self._map_tool_choice( - tool_choice=non_default_params.get("tool_choice"), - parallel_tool_use=non_default_params.get("parallel_tool_calls"), + _tool_choice: Optional[AnthropicMessagesToolChoice] = ( + self._map_tool_choice( + tool_choice=non_default_params.get("tool_choice"), + parallel_tool_use=non_default_params.get("parallel_tool_calls"), + ) ) if _tool_choice is not None: @@ -1062,9 +1066,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self.map_openai_context_management_to_anthropic(value) ) if anthropic_context_management is not None: - optional_params[ - "context_management" - ] = anthropic_context_management + optional_params["context_management"] = ( + anthropic_context_management + ) elif param == "speed" and isinstance(value, str): # Pass through Anthropic-specific speed parameter for fast mode optional_params["speed"] = value @@ -1138,9 +1142,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): text=system_message_block["content"], ) if "cache_control" in system_message_block: - anthropic_system_message_content[ - "cache_control" - ] = system_message_block["cache_control"] + anthropic_system_message_content["cache_control"] = ( + system_message_block["cache_control"] + ) anthropic_system_message_list.append( anthropic_system_message_content ) @@ -1164,9 +1168,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) ) if "cache_control" in _content: - anthropic_system_message_content[ - "cache_control" - ] = _content["cache_control"] + anthropic_system_message_content["cache_control"] = ( + _content["cache_control"] + ) anthropic_system_message_list.append( anthropic_system_message_content @@ -1463,9 +1467,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) return _message - def extract_response_content( - self, completion_response: dict - ) -> Tuple[ + def extract_response_content(self, completion_response: dict) -> Tuple[ str, Optional[List[Any]], Optional[ @@ -1749,6 +1751,48 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): provider_specific_fields["web_search_results"] = web_search_results if tool_results is not None: provider_specific_fields["tool_results"] = tool_results + # Convert to provider-neutral OutputCodeInterpreterCall objects + # so the Responses API layer can use them without Anthropic-specific knowledge. + container_id = ( + completion_response.get("container", {}).get("id") + if isinstance(completion_response.get("container"), dict) + else None + ) + code_by_id: Dict[str, str] = {} + for tc in tool_calls: + try: + args = json.loads(tc.get("function", {}).get("arguments", "{}")) + code_by_id[tc.get("id", "")] = args.get("command", "") + except Exception: + pass + code_interpreter_results = [] + for tr in tool_results: + call_id = tr.get("tool_use_id", "") + content = tr.get("content", {}) + if isinstance(content, dict): + parts = [] + if content.get("stdout"): + parts.append(content["stdout"]) + if content.get("stderr"): + parts.append(f"STDERR: {content['stderr']}") + logs = "".join(parts) if parts else str(content) + else: + logs = str(content) + code_interpreter_results.append( + OutputCodeInterpreterCall( + type="code_interpreter_call", + id=call_id, + code=code_by_id.get(call_id, ""), + container_id=container_id, + status="completed", + outputs=[ + OutputCodeInterpreterCallLog(type="logs", logs=logs) + ], + ) + ) + provider_specific_fields["code_interpreter_results"] = ( + code_interpreter_results + ) if container is not None: provider_specific_fields["container"] = container if compaction_blocks is not None: @@ -1794,6 +1838,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): model_response.created = int(time.time()) model_response.model = completion_response["model"] + _hidden_params["provider_specific_fields"] = provider_specific_fields model_response._hidden_params = _hidden_params return model_response diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index ce037850b86..0b7d6e8a7a4 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -107,6 +107,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._reasoning_done_emitted = False self._reasoning_item_id: Optional[str] = None self._accumulated_reasoning_content_parts: List[str] = [] + self._accumulated_provider_specific_fields: Dict[str, Any] = {} def _get_or_assign_tool_output_index(self, call_id: str) -> int: existing = self._tool_output_index_by_call_id.get(call_id) @@ -479,16 +480,37 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): event.__dict__["sequence_number"] = self._sequence_number return event - def create_litellm_model_response( - self, - ) -> Optional[ModelResponse]: - return cast( + def _merge_provider_specific_fields(self, src: dict) -> None: + """Merge provider_specific_fields, extending list values instead of replacing.""" + for key, val in src.items(): + existing = self._accumulated_provider_specific_fields.get(key) + if ( + existing is not None + and isinstance(val, list) + and isinstance(existing, list) + ): + existing.extend(val) + else: + self._accumulated_provider_specific_fields[key] = val + + def create_litellm_model_response(self) -> Optional[ModelResponse]: + response = cast( Optional[ModelResponse], stream_chunk_builder( chunks=self.collected_chat_completion_chunks, logging_obj=self.litellm_logging_obj, ), ) + if response is not None and self._accumulated_provider_specific_fields: + if ( + not hasattr(response, "_hidden_params") + or response._hidden_params is None + ): + response._hidden_params = {} + response._hidden_params.setdefault("provider_specific_fields", {}).update( + self._accumulated_provider_specific_fields + ) + return response @staticmethod def _snapshot_chunk_for_stream_chunk_builder( @@ -853,6 +875,17 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if chunk is not None: chunk = cast(ModelResponseStream, chunk) self._ensure_output_item_for_chunk(chunk) + # Accumulate provider_specific_fields from chunk and delta + for src in ( + getattr(chunk, "provider_specific_fields", None), + getattr( + chunk.choices[0].delta if chunk.choices else None, + "provider_specific_fields", + None, + ), + ): + if src and isinstance(src, dict): + self._merge_provider_specific_fields(src) # Proceed to transformation self.collected_chat_completion_chunks.append( self._snapshot_chunk_for_stream_chunk_builder(chunk) @@ -964,6 +997,17 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): try: chunk = self.litellm_custom_stream_wrapper.__next__() self._ensure_output_item_for_chunk(chunk) + # Accumulate provider_specific_fields from chunk and delta + for src in ( + getattr(chunk, "provider_specific_fields", None), + getattr( + chunk.choices[0].delta if chunk.choices else None, + "provider_specific_fields", + None, + ), + ): + if src and isinstance(src, dict): + self._merge_provider_specific_fields(src) # Emit any just-queued output_item event if self._pending_response_events: return self._pending_response_events.pop(0) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 71fa88fb751..b54d5930efc 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -42,6 +42,7 @@ from litellm.types.llms.openai import ( from litellm.types.responses.main import ( GenericResponseOutputItem, GenericResponseOutputItemContentAnnotation, + OutputCodeInterpreterCall, OutputFunctionToolCall, OutputImageGenerationCall, OutputText, @@ -1696,6 +1697,7 @@ class LiteLLMCompletionResponsesConfig: ) -> List[ Union[ GenericResponseOutputItem, + OutputCodeInterpreterCall, OutputFunctionToolCall, OutputImageGenerationCall, ResponseFunctionToolCall, @@ -1704,6 +1706,7 @@ class LiteLLMCompletionResponsesConfig: responses_output: List[ Union[ GenericResponseOutputItem, + OutputCodeInterpreterCall, OutputFunctionToolCall, OutputImageGenerationCall, ResponseFunctionToolCall, @@ -1725,8 +1728,56 @@ class LiteLLMCompletionResponsesConfig: chat_completion_response=chat_completion_response ) ) + + # Convert server-side tool results (e.g. Anthropic code execution) + # into code_interpreter_call output items, replacing the corresponding + # function_call items so the output matches OpenAI's native shape. + tool_result_items = ( + LiteLLMCompletionResponsesConfig._extract_tool_result_output_items( + chat_completion_response + ) + ) + if tool_result_items: + result_by_id = {item.id: item for item in tool_result_items} + replaced_ids = set(result_by_id.keys()) + responses_output = [ + ( + result_by_id[getattr(item, "call_id", None)] + if ( + getattr(item, "type", None) == "function_call" + and getattr(item, "call_id", None) in replaced_ids + ) + else item + ) + for item in responses_output + ] + return responses_output + @staticmethod + def _extract_tool_result_output_items( + chat_completion_response: ModelResponse, + ) -> list: + """Extract pre-built code_interpreter_call output items from provider_specific_fields. + + Provider transformers (e.g. Anthropic) convert their native tool results + into OutputCodeInterpreterCall objects and store them in + provider_specific_fields["code_interpreter_results"]. This method + simply retrieves them — no provider-specific parsing here. + """ + output_items: list = [] + for choice in chat_completion_response.choices or []: + message = getattr(choice, "message", None) + if not message: + continue + psf = getattr(message, "provider_specific_fields", None) + if not psf or not isinstance(psf, dict): + continue + results = psf.get("code_interpreter_results") + if results and isinstance(results, list): + output_items.extend(results) + return output_items + @staticmethod def _extract_reasoning_output_items( chat_completion_response: ModelResponse, @@ -2055,9 +2106,9 @@ class LiteLLMCompletionResponsesConfig: hasattr(completion_details, "reasoning_tokens") and completion_details.reasoning_tokens is not None ): - output_details_dict[ - "reasoning_tokens" - ] = completion_details.reasoning_tokens + output_details_dict["reasoning_tokens"] = ( + completion_details.reasoning_tokens + ) else: output_details_dict["reasoning_tokens"] = 0 diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index a2df3f2e0d6..a265198e6b8 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -84,6 +84,7 @@ from typing_extensions import Annotated, Dict, Required, TypedDict, override from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject from litellm.types.responses.main import ( GenericResponseOutputItem, + OutputCodeInterpreterCall, OutputFunctionToolCall, OutputImageGenerationCall, ) @@ -969,12 +970,12 @@ class OpenAIChatCompletionChunk(ChatCompletionChunk): class Hyperparameters(BaseModel): batch_size: Optional[Union[str, int]] = None # "Number of examples in each batch." - learning_rate_multiplier: Optional[ - Union[str, float] - ] = None # Scaling factor for the learning rate - n_epochs: Optional[ - Union[str, int] - ] = None # "The number of epochs to train the model for" + learning_rate_multiplier: Optional[Union[str, float]] = ( + None # Scaling factor for the learning rate + ) + n_epochs: Optional[Union[str, int]] = ( + None # "The number of epochs to train the model for" + ) model_config = {"extra": "allow"} @@ -1003,18 +1004,18 @@ class FineTuningJobCreate(BaseModel): model: str # "The name of the model to fine-tune." training_file: str # "The ID of an uploaded file that contains training data." - hyperparameters: Optional[ - Hyperparameters - ] = None # "The hyperparameters used for the fine-tuning job." - suffix: Optional[ - str - ] = None # "A string of up to 18 characters that will be added to your fine-tuned model name." - validation_file: Optional[ - str - ] = None # "The ID of an uploaded file that contains validation data." - integrations: Optional[ - List[str] - ] = None # "A list of integrations to enable for your fine-tuning job." + hyperparameters: Optional[Hyperparameters] = ( + None # "The hyperparameters used for the fine-tuning job." + ) + suffix: Optional[str] = ( + None # "A string of up to 18 characters that will be added to your fine-tuned model name." + ) + validation_file: Optional[str] = ( + None # "The ID of an uploaded file that contains validation data." + ) + integrations: Optional[List[str]] = ( + None # "A list of integrations to enable for your fine-tuning job." + ) seed: Optional[int] = None # "The seed controls the reproducibility of the job." @@ -1242,6 +1243,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): List[ Union[ GenericResponseOutputItem, + OutputCodeInterpreterCall, OutputFunctionToolCall, OutputImageGenerationCall, ResponseFunctionToolCall, @@ -1308,13 +1310,16 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): if not isinstance(serialized, list): return serialized return [ - { - k: v - for k, v in item.items() - if v is not None or k not in ("status", "content", "encrypted_content") - } - if isinstance(item, dict) and item.get("type") == "reasoning" - else item + ( + { + k: v + for k, v in item.items() + if v is not None + or k not in ("status", "content", "encrypted_content") + } + if isinstance(item, dict) and item.get("type") == "reasoning" + else item + ) for item in serialized ] diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index 7a666d5e65f..e46857565c7 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -49,6 +49,24 @@ class OutputImageGenerationCall(BaseLiteLLMOpenAIResponseObject): result: Optional[str] # Base64 encoded image data (without data:image prefix) +class OutputCodeInterpreterCallLog(BaseLiteLLMOpenAIResponseObject): + """Log output from a code interpreter call""" + + type: Literal["logs"] + logs: str + + +class OutputCodeInterpreterCall(BaseLiteLLMOpenAIResponseObject): + """A code interpreter / code execution call output""" + + type: Literal["code_interpreter_call"] + id: str + code: Optional[str] + container_id: Optional[str] + status: Literal["in_progress", "completed", "incomplete", "failed"] + outputs: Optional[List[OutputCodeInterpreterCallLog]] + + class GenericResponseOutputItem(BaseLiteLLMOpenAIResponseObject): """ Generic response API output item diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index d9f513d8d1d..35c7a62027b 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -6,6 +6,7 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ) +from litellm.types.responses.main import OutputCodeInterpreterCall def test_redacted_thinking_content_block_delta(): @@ -479,14 +480,22 @@ def test_partial_json_chunk_accumulation(): # First partial chunk should return None (still accumulating) result1 = iterator._parse_sse_data(f"data:{partial_chunk_1}") assert result1 is None, "First partial chunk should return None while accumulating" - assert iterator.chunk_type == "accumulated_json", "Should switch to accumulated_json mode" - assert iterator.accumulated_json == partial_chunk_1, "Should have accumulated first part" + assert ( + iterator.chunk_type == "accumulated_json" + ), "Should switch to accumulated_json mode" + assert ( + iterator.accumulated_json == partial_chunk_1 + ), "Should have accumulated first part" # Second partial chunk should complete the JSON and return a parsed result result2 = iterator._parse_sse_data(f"data:{partial_chunk_2}") assert result2 is not None, "Second chunk should return parsed result" - assert iterator.accumulated_json == "", "Buffer should be cleared after successful parse" - assert result2.choices[0].delta.content == "Hello", f"Expected 'Hello', got '{result2.choices[0].delta.content}'" + assert ( + iterator.accumulated_json == "" + ), "Buffer should be cleared after successful parse" + assert ( + result2.choices[0].delta.content == "Hello" + ), f"Expected 'Hello', got '{result2.choices[0].delta.content}'" def test_complete_json_chunk_no_accumulation(): @@ -503,7 +512,9 @@ def test_complete_json_chunk_no_accumulation(): assert result is not None, "Complete chunk should return parsed result immediately" assert iterator.chunk_type == "valid_json", "Should remain in valid_json mode" assert iterator.accumulated_json == "", "Buffer should remain empty" - assert result.choices[0].delta.content == "Hello", f"Expected 'Hello', got '{result.choices[0].delta.content}'" + assert ( + result.choices[0].delta.content == "Hello" + ), f"Expected 'Hello', got '{result.choices[0].delta.content}'" def test_multiple_partial_chunks_accumulation(): @@ -620,7 +631,9 @@ def test_web_search_tool_result_no_extra_tool_calls(): # Should have exactly 2 tool calls: # 1. From content_block_start (server_tool_use) with id and name # 2. From content_block_delta with the actual query - assert len(tool_calls_emitted) == 2, f"Expected 2 tool calls, got {len(tool_calls_emitted)}" + assert ( + len(tool_calls_emitted) == 2 + ), f"Expected 2 tool calls, got {len(tool_calls_emitted)}" # First tool call should have the id and name assert tool_calls_emitted[0]["id"] == "srvtoolu_01ABC123" @@ -722,7 +735,10 @@ def test_web_search_tool_result_captured_in_provider_specific_fields(): { "type": "content_block_delta", "index": 0, - "delta": {"type": "input_json_delta", "partial_json": '{"query": "otter facts"}'}, + "delta": { + "type": "input_json_delta", + "partial_json": '{"query": "otter facts"}', + }, }, # 4. content_block_stop for server_tool_use {"type": "content_block_stop", "index": 0}, @@ -822,7 +838,10 @@ def test_web_fetch_tool_result_captured_in_provider_specific_fields(): { "type": "content_block_delta", "index": 0, - "delta": {"type": "input_json_delta", "partial_json": '{"url": "https://example.com"}'}, + "delta": { + "type": "input_json_delta", + "partial_json": '{"url": "https://example.com"}', + }, }, # 4. content_block_stop for server_tool_use {"type": "content_block_stop", "index": 0}, @@ -946,7 +965,7 @@ def test_web_fetch_tool_result_no_extra_tool_calls(): def test_container_in_provider_specific_fields_streaming(): """ Test that container is captured in provider_specific_fields for streaming responses. - + When container with skills is used, the container field should be present in the provider_specific_fields of the message_delta chunk. """ @@ -1025,7 +1044,9 @@ def test_container_in_provider_specific_fields_streaming(): ] # Verify container was captured - assert container_field is not None, "container should be captured in provider_specific_fields" + assert ( + container_field is not None + ), "container should be captured in provider_specific_fields" assert ( container_field["id"] == "container_011CW9hA9zpZ8xD3bjjShy4p" ), "container id should match" @@ -1033,18 +1054,14 @@ def test_container_in_provider_specific_fields_streaming(): container_field["expires_at"] == "2025-12-16T04:57:16.913181Z" ), "expires_at should match" assert len(container_field["skills"]) == 1, "Should have 1 skill" - assert ( - container_field["skills"][0]["skill_id"] == "pptx" - ), "skill_id should be pptx" - assert ( - container_field["skills"][0]["version"] == "20251013" - ), "version should match" + assert container_field["skills"][0]["skill_id"] == "pptx", "skill_id should be pptx" + assert container_field["skills"][0]["version"] == "20251013", "version should match" def test_container_in_provider_specific_fields_non_streaming(): """ Test that container is captured in provider_specific_fields for non-streaming responses. - + When container with skills is used in non-streaming, the container field should be present in the provider_specific_fields of the response. """ @@ -1106,7 +1123,7 @@ def test_container_in_provider_specific_fields_non_streaming(): def test_container_absent_when_not_provided(): """ Test that container is not added to provider_specific_fields when not provided. - + This ensures we don't add empty or None container fields. """ iterator = ModelResponseIterator( @@ -1133,3 +1150,197 @@ def test_container_absent_when_not_provided(): assert ( "container" not in model_response.choices[0].delta.provider_specific_fields ), "container should not be present when not provided in delta" + + +def test_streaming_code_execution_produces_code_interpreter_results(): + """ + Test that bash_code_execution_tool_result content blocks in streaming + produce code_interpreter_results in provider_specific_fields, so the + Responses API layer can use them without Anthropic-specific knowledge. + """ + + chunks = [ + { + "type": "message_start", + "message": { + "id": "msg_01XYZ", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 100, "output_tokens": 1}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "text", + "text": "", + }, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "Running code..."}, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "server_tool_use", + "id": "srvtoolu_01ABC", + "name": "bash_code_execution", + "input": {"command": "echo hello"}, + }, + }, + {"type": "content_block_stop", "index": 1}, + { + "type": "content_block_start", + "index": 2, + "content_block": { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01ABC", + "content": { + "type": "bash_code_execution_result", + "stdout": "hello\n", + "stderr": "", + "return_code": 0, + }, + }, + }, + {"type": "content_block_stop", "index": 2}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 50}, + }, + ] + + iterator = ModelResponseIterator(None, sync_stream=True) + + found_code_interpreter_results = False + for chunk in chunks: + parsed = iterator.chunk_parser(chunk) + psf = None + if parsed.choices and parsed.choices[0].delta: + psf = getattr(parsed.choices[0].delta, "provider_specific_fields", None) + if psf and "code_interpreter_results" in psf: + found_code_interpreter_results = True + results = psf["code_interpreter_results"] + assert len(results) == 1 + assert isinstance(results[0], OutputCodeInterpreterCall) + assert results[0].type == "code_interpreter_call" + assert results[0].id == "srvtoolu_01ABC" + assert results[0].code == "echo hello" + assert results[0].outputs is not None + assert len(results[0].outputs) == 1 + assert results[0].outputs[0].logs == "hello\n" + + assert found_code_interpreter_results, ( + "code_interpreter_results should appear in provider_specific_fields " + "when bash_code_execution_tool_result is streamed" + ) + + +def test_streaming_multiple_code_executions_no_duplicates(): + """ + Test that multiple code executions in a single streaming response produce + exactly one code_interpreter_result per execution — no duplicates from + _build_code_interpreter_results rebuilding the full list. + """ + chunks = [ + { + "type": "message_start", + "message": { + "id": "msg_01XYZ", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 100, "output_tokens": 1}, + }, + }, + # First code execution + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "server_tool_use", + "id": "srvtoolu_01AAA", + "name": "bash_code_execution", + "input": {"command": "echo first"}, + }, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01AAA", + "content": { + "type": "bash_code_execution_result", + "stdout": "first\n", + "stderr": "", + "return_code": 0, + }, + }, + }, + {"type": "content_block_stop", "index": 1}, + # Second code execution + { + "type": "content_block_start", + "index": 2, + "content_block": { + "type": "server_tool_use", + "id": "srvtoolu_01BBB", + "name": "bash_code_execution", + "input": {"command": "echo second"}, + }, + }, + {"type": "content_block_stop", "index": 2}, + { + "type": "content_block_start", + "index": 3, + "content_block": { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01BBB", + "content": { + "type": "bash_code_execution_result", + "stdout": "second\n", + "stderr": "", + "return_code": 0, + }, + }, + }, + {"type": "content_block_stop", "index": 3}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 50}, + }, + ] + + iterator = ModelResponseIterator(None, sync_stream=True) + + # Collect ALL code_interpreter_results emitted across all chunks + all_results = [] + for chunk in chunks: + parsed = iterator.chunk_parser(chunk) + psf = None + if parsed.choices and parsed.choices[0].delta: + psf = getattr(parsed.choices[0].delta, "provider_specific_fields", None) + if psf and "code_interpreter_results" in psf: + all_results.extend(psf["code_interpreter_results"]) + + # Should have exactly 2 results, one per execution — no duplicates + assert len(all_results) == 2, ( + f"Expected 2 code_interpreter_results, got {len(all_results)}. " + f"IDs: {[r.id for r in all_results]}" + ) + assert all_results[0].id == "srvtoolu_01AAA" + assert all_results[0].code == "echo first" + assert all_results[0].outputs[0].logs == "first\n" + assert all_results[1].id == "srvtoolu_01BBB" + assert all_results[1].code == "echo second" + assert all_results[1].outputs[0].logs == "second\n" diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index a95b9413b9d..a3469964862 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -183,7 +183,9 @@ def test_extract_response_content_with_citations(): }, } - _, citations, _, _, _, _, _, _ = config.extract_response_content(completion_response) + _, citations, _, _, _, _, _, _ = config.extract_response_content( + completion_response + ) assert citations == [ [ { @@ -305,7 +307,7 @@ def test_web_search_tool_result_extraction(): "type": "server_tool_use", "id": "srvtoolu_01ABC123", "name": "web_search", - "input": {"query": "average weight african elephant kg"} + "input": {"query": "average weight african elephant kg"}, }, { "type": "web_search_tool_result", @@ -317,32 +319,39 @@ def test_web_search_tool_result_extraction(): "title": "African Elephant Facts", "encrypted_content": "encrypted_data_here", "page_age": "2024-01-15", - "snippet": "Adult African elephants weigh between 4,000-6,000 kg..." + "snippet": "Adult African elephants weigh between 4,000-6,000 kg...", } - ] + ], }, { "type": "text", - "text": "Based on my search, African elephants weigh around 5,000 kg." + "text": "Based on my search, African elephants weigh around 5,000 kg.", }, { "type": "tool_use", "id": "toolu_01XYZ789", "name": "add_numbers", - "input": {"a": 5000, "b": 100} - } + "input": {"a": 5000, "b": 100}, + }, ], "stop_reason": "tool_use", "usage": { "input_tokens": 100, "output_tokens": 50, - "server_tool_use": {"web_search_requests": 1} - } + "server_tool_use": {"web_search_requests": 1}, + }, } - text, citations, thinking_blocks, reasoning_content, tool_calls, web_search_results, tool_results, compaction_blocks = config.extract_response_content( - completion_response - ) + ( + text, + citations, + thinking_blocks, + reasoning_content, + tool_calls, + web_search_results, + tool_results, + compaction_blocks, + ) = config.extract_response_content(completion_response) # Verify text extraction assert "Based on my search" in text @@ -388,7 +397,7 @@ def test_web_search_tool_result_in_provider_specific_fields(): "type": "server_tool_use", "id": "srvtoolu_provider_test", "name": "web_search", - "input": {"query": "test query"} + "input": {"query": "test query"}, }, { "type": "web_search_tool_result", @@ -398,21 +407,18 @@ def test_web_search_tool_result_in_provider_specific_fields(): "type": "web_search_result", "url": "https://example.com/test", "title": "Test Result", - "snippet": "Test snippet content" + "snippet": "Test snippet content", } - ] + ], }, - { - "type": "text", - "text": "Here is the result." - } + {"type": "text", "text": "Here is the result."}, ], "stop_reason": "end_turn", "usage": { "input_tokens": 50, "output_tokens": 25, - "server_tool_use": {"web_search_requests": 1} - } + "server_tool_use": {"web_search_requests": 1}, + }, } raw_response = httpx.Response(status_code=200, headers={}) @@ -432,7 +438,10 @@ def test_web_search_tool_result_in_provider_specific_fields(): assert "web_search_results" in provider_fields assert len(provider_fields["web_search_results"]) == 1 assert provider_fields["web_search_results"][0]["type"] == "web_search_tool_result" - assert provider_fields["web_search_results"][0]["tool_use_id"] == "srvtoolu_provider_test" + assert ( + provider_fields["web_search_results"][0]["tool_use_id"] + == "srvtoolu_provider_test" + ) def test_multiple_web_search_tool_results(): @@ -447,34 +456,52 @@ def test_multiple_web_search_tool_results(): "type": "server_tool_use", "id": "srvtoolu_search1", "name": "web_search", - "input": {"query": "african elephant weight"} + "input": {"query": "african elephant weight"}, }, { "type": "web_search_tool_result", "tool_use_id": "srvtoolu_search1", - "content": [{"type": "web_search_result", "url": "https://example1.com", "title": "Result 1", "snippet": "First result"}] + "content": [ + { + "type": "web_search_result", + "url": "https://example1.com", + "title": "Result 1", + "snippet": "First result", + } + ], }, { "type": "server_tool_use", "id": "srvtoolu_search2", "name": "web_search", - "input": {"query": "asian elephant weight"} + "input": {"query": "asian elephant weight"}, }, { "type": "web_search_tool_result", "tool_use_id": "srvtoolu_search2", - "content": [{"type": "web_search_result", "url": "https://example2.com", "title": "Result 2", "snippet": "Second result"}] + "content": [ + { + "type": "web_search_result", + "url": "https://example2.com", + "title": "Result 2", + "snippet": "Second result", + } + ], }, - { - "type": "text", - "text": "Found information about both elephants." - } + {"type": "text", "text": "Found information about both elephants."}, ] } - text, citations, thinking_blocks, reasoning_content, tool_calls, web_search_results, tool_results, compaction_blocks = config.extract_response_content( - completion_response - ) + ( + text, + citations, + thinking_blocks, + reasoning_content, + tool_calls, + web_search_results, + tool_results, + compaction_blocks, + ) = config.extract_response_content(completion_response) # Verify both web_search_tool_results are extracted assert web_search_results is not None @@ -751,7 +778,7 @@ def test_anthropic_beta_header_merging_with_output_format(): optional_params = { "output_format": { "type": "json_schema", - "schema": {"type": "object", "properties": {}} + "schema": {"type": "object", "properties": {}}, } } @@ -761,10 +788,12 @@ def test_anthropic_beta_header_merging_with_output_format(): # Both beta headers should be present beta_value = result_headers["anthropic-beta"] - assert "context-1m-2025-08-07" in beta_value, \ - f"User's context-1m beta header missing from: {beta_value}" - assert "structured-outputs-2025-11-13" in beta_value, \ - f"Structured output beta header missing from: {beta_value}" + assert ( + "context-1m-2025-08-07" in beta_value + ), f"User's context-1m beta header missing from: {beta_value}" + assert ( + "structured-outputs-2025-11-13" in beta_value + ), f"Structured output beta header missing from: {beta_value}" def test_anthropic_beta_header_merging_with_multiple_features(): @@ -780,10 +809,10 @@ def test_anthropic_beta_header_merging_with_multiple_features(): optional_params = { "output_format": { "type": "json_schema", - "schema": {"type": "object", "properties": {}} + "schema": {"type": "object", "properties": {}}, }, "context_management": _sample_context_management_payload(), - "tools": [{"type": "web_fetch_20250910", "name": "web_fetch"}] + "tools": [{"type": "web_fetch_20250910", "name": "web_fetch"}], } result_headers = config.update_headers_with_optional_anthropic_beta( @@ -950,20 +979,12 @@ def test_tool_search_regex_detection(): # Test with tool search regex tool tools = [ - { - "type": "tool_search_tool_regex_20251119", - "name": "tool_search_tool_regex" - } + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"} ] assert config.is_tool_search_used(tools) is True # Test without tool search - tools = [ - { - "type": "function", - "function": {"name": "get_weather"} - } - ] + tools = [{"type": "function", "function": {"name": "get_weather"}}] assert config.is_tool_search_used(tools) is False @@ -975,10 +996,7 @@ def test_tool_search_bm25_detection(): # Test with tool search BM25 tool tools = [ - { - "type": "tool_search_tool_bm25_20251119", - "name": "tool_search_tool_bm25" - } + {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"} ] assert config.is_tool_search_used(tools) is True @@ -1002,10 +1020,7 @@ def test_tool_search_regex_mapping(): """Test that tool search regex tools are properly mapped""" config = AnthropicConfig() - tool = { - "type": "tool_search_tool_regex_20251119", - "name": "tool_search_tool_regex" - } + tool = {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"} mapped_tool, mcp_server = config._map_tool_helper(tool) @@ -1019,10 +1034,7 @@ def test_tool_search_bm25_mapping(): """Test that tool search BM25 tools are properly mapped""" config = AnthropicConfig() - tool = { - "type": "tool_search_tool_bm25_20251119", - "name": "tool_search_tool_bm25" - } + tool = {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"} mapped_tool, mcp_server = config._map_tool_helper(tool) @@ -1037,20 +1049,17 @@ def test_deferred_tools_separation(): config = AnthropicConfig() tools = [ - { - "type": "tool_search_tool_regex_20251119", - "name": "tool_search_tool_regex" - }, + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, { "type": "function", "function": {"name": "get_weather"}, - "defer_loading": True + "defer_loading": True, }, { "type": "function", "function": {"name": "search_files"}, - "defer_loading": False - } + "defer_loading": False, + }, ] non_deferred, deferred = config._separate_deferred_tools(tools) @@ -1069,14 +1078,21 @@ def test_server_tool_use_in_response(): "type": "server_tool_use", "id": "srvtoolu_01ABC123", "name": "tool_search_tool_regex", - "input": {"query": "weather"} + "input": {"query": "weather"}, } ] } - text, citations, thinking_blocks, reasoning_content, tool_calls, web_search_results, tool_results, compaction_blocks = config.extract_response_content( - completion_response - ) + ( + text, + citations, + thinking_blocks, + reasoning_content, + tool_calls, + web_search_results, + tool_results, + compaction_blocks, + ) = config.extract_response_content(completion_response) assert len(tool_calls) == 1 assert tool_calls[0]["id"] == "srvtoolu_01ABC123" @@ -1091,9 +1107,7 @@ def test_tool_search_usage_tracking(): usage_object = { "input_tokens": 100, "output_tokens": 50, - "server_tool_use": { - "tool_search_requests": 2 - } + "server_tool_use": {"tool_search_requests": 2}, } usage = config.calculate_usage(usage_object=usage_object, reasoning_content=None) @@ -1109,16 +1123,13 @@ def test_tool_reference_expansion(): deferred_tools = [ { "type": "function", - "function": { - "name": "get_weather", - "description": "Get weather" - } + "function": {"name": "get_weather", "description": "Get weather"}, } ] content = [ {"type": "text", "text": "I'll search for tools"}, - {"type": "tool_reference", "tool_name": "get_weather"} + {"type": "tool_reference", "tool_name": "get_weather"}, ] expanded = config._expand_tool_references(content, deferred_tools) @@ -1140,13 +1151,11 @@ def test_defer_loading_preserved_in_transformation(): "description": "Get weather information", "parameters": { "type": "object", - "properties": { - "location": {"type": "string"} - }, - "required": ["location"] - } + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, }, - "defer_loading": True + "defer_loading": True, } mapped_tool, mcp_server = config._map_tool_helper(tool) @@ -1166,45 +1175,51 @@ def test_tool_search_complete_response_parsing(): "content": [ { "type": "text", - "text": "I'll search for weather-related tools that can help you." + "text": "I'll search for weather-related tools that can help you.", }, { "type": "server_tool_use", "id": "srvtoolu_015i6aVA2niwzv4RG4DtnxDJ", "name": "tool_search_tool_regex", "input": {"pattern": "weather", "limit": 5}, - "caller": {"type": "direct"} + "caller": {"type": "direct"}, }, { "type": "tool_search_tool_result", "tool_use_id": "srvtoolu_015i6aVA2niwzv4RG4DtnxDJ", "content": { "type": "tool_search_tool_search_result", - "tool_references": [{"type": "tool_reference", "tool_name": "get_weather"}] - } - }, - { - "type": "text", - "text": "Great! I found a weather tool." + "tool_references": [ + {"type": "tool_reference", "tool_name": "get_weather"} + ], + }, }, + {"type": "text", "text": "Great! I found a weather tool."}, { "type": "tool_use", "id": "toolu_01CrCNx4ntSaeeV9iArT4JfQ", "name": "get_weather", - "input": {"location": "San Francisco"} - } + "input": {"location": "San Francisco"}, + }, ], "usage": { "input_tokens": 1639, "output_tokens": 170, - "server_tool_use": {"web_search_requests": 0} - } + "server_tool_use": {"web_search_requests": 0}, + }, } # Extract content - text, citations, thinking_blocks, reasoning_content, tool_calls, web_search_results, tool_results, compaction_blocks = config.extract_response_content( - completion_response - ) + ( + text, + citations, + thinking_blocks, + reasoning_content, + tool_calls, + web_search_results, + tool_results, + compaction_blocks, + ) = config.extract_response_content(completion_response) # Verify text extraction (should concatenate both text blocks) assert "I'll search for weather-related tools" in text @@ -1222,12 +1237,14 @@ def test_tool_search_complete_response_parsing(): usage = config.calculate_usage( usage_object=completion_response["usage"], reasoning_content=None, - completion_response=completion_response + completion_response=completion_response, ) assert usage.server_tool_use is not None assert usage.server_tool_use.web_search_requests == 0 - assert usage.server_tool_use.tool_search_requests == 1 # Counted from server_tool_use blocks + assert ( + usage.server_tool_use.tool_search_requests == 1 + ) # Counted from server_tool_use blocks def test_allowed_callers_field_preservation(): @@ -1242,13 +1259,11 @@ def test_allowed_callers_field_preservation(): "description": "Execute a SQL query", "parameters": { "type": "object", - "properties": { - "sql": {"type": "string"} - }, - "required": ["sql"] - } + "properties": {"sql": {"type": "string"}}, + "required": ["sql"], + }, }, - "allowed_callers": ["code_execution_20250825"] + "allowed_callers": ["code_execution_20250825"], } transformed_tool, _ = config._map_tool_helper(tool_with_allowed_callers) @@ -1265,19 +1280,16 @@ def test_programmatic_tool_calling_beta_header(): # Test detection with allowed_callers tools = [ - { - "type": "code_execution_20250825", - "name": "code_execution" - }, + {"type": "code_execution_20250825", "name": "code_execution"}, { "type": "function", "function": { "name": "query_database", "description": "Execute a SQL query", - "parameters": {"type": "object", "properties": {}} + "parameters": {"type": "object", "properties": {}}, }, - "allowed_callers": ["code_execution_20250825"] - } + "allowed_callers": ["code_execution_20250825"], + }, ] is_programmatic = model_info.is_programmatic_tool_calling_used(tools) @@ -1285,8 +1297,7 @@ def test_programmatic_tool_calling_beta_header(): # Test header generation headers = model_info.get_anthropic_headers( - api_key="test-key", - programmatic_tool_calling_used=True + api_key="test-key", programmatic_tool_calling_used=True ) assert "anthropic-beta" in headers @@ -1303,10 +1314,7 @@ def test_caller_field_in_response(): "type": "message", "role": "assistant", "content": [ - { - "type": "text", - "text": "I'll query the database." - }, + {"type": "text", "text": "I'll query the database."}, { "type": "tool_use", "id": "toolu_123", @@ -1314,15 +1322,24 @@ def test_caller_field_in_response(): "input": {"sql": "SELECT * FROM users"}, "caller": { "type": "code_execution_20250825", - "tool_id": "srvtoolu_abc" - } - } + "tool_id": "srvtoolu_abc", + }, + }, ], "stop_reason": "tool_use", - "usage": {"input_tokens": 100, "output_tokens": 50} + "usage": {"input_tokens": 100, "output_tokens": 50}, } - text, citations, thinking, reasoning, tool_calls, web_search_results, tool_results, compaction_blocks = config.extract_response_content(completion_response) + ( + text, + citations, + thinking, + reasoning, + tool_calls, + web_search_results, + tool_results, + compaction_blocks, + ) = config.extract_response_content(completion_response) assert len(tool_calls) == 1 assert tool_calls[0]["id"] == "toolu_123" @@ -1337,10 +1354,7 @@ def test_code_execution_20250825_tool_type(): """Test that code_execution_20250825 tool type is handled correctly.""" config = AnthropicConfig() - tool = { - "type": "code_execution_20250825", - "name": "code_execution" - } + tool = {"type": "code_execution_20250825", "name": "code_execution"} transformed_tool, _ = config._map_tool_helper(tool) assert transformed_tool is not None @@ -1360,13 +1374,11 @@ def test_allowed_callers_in_function_field(): "description": "Execute a SQL query", "parameters": { "type": "object", - "properties": { - "sql": {"type": "string"} - }, - "required": ["sql"] + "properties": {"sql": {"type": "string"}}, + "required": ["sql"], }, - "allowed_callers": ["code_execution_20250825"] - } + "allowed_callers": ["code_execution_20250825"], + }, } transformed_tool, _ = config._map_tool_helper(tool) @@ -1389,15 +1401,15 @@ def test_input_examples_field_preservation(): "type": "object", "properties": { "location": {"type": "string"}, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]} + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, }, - "required": ["location"] - } + "required": ["location"], + }, }, "input_examples": [ {"location": "San Francisco, CA", "unit": "fahrenheit"}, - {"location": "Tokyo, Japan", "unit": "celsius"} - ] + {"location": "Tokyo, Japan", "unit": "celsius"}, + ], } transformed_tool, _ = config._map_tool_helper(tool_with_examples) @@ -1420,11 +1432,9 @@ def test_input_examples_beta_header(): "function": { "name": "get_weather", "description": "Get weather information", - "parameters": {"type": "object", "properties": {}} + "parameters": {"type": "object", "properties": {}}, }, - "input_examples": [ - {"location": "San Francisco, CA"} - ] + "input_examples": [{"location": "San Francisco, CA"}], } ] @@ -1433,8 +1443,7 @@ def test_input_examples_beta_header(): # Test header generation headers = model_info.get_anthropic_headers( - api_key="test-key", - input_examples_used=True + api_key="test-key", input_examples_used=True ) assert "anthropic-beta" in headers @@ -1453,16 +1462,14 @@ def test_input_examples_in_function_field(): "description": "Get weather information", "parameters": { "type": "object", - "properties": { - "location": {"type": "string"} - }, - "required": ["location"] + "properties": {"location": {"type": "string"}}, + "required": ["location"], }, "input_examples": [ {"location": "Paris, France"}, - {"location": "London, UK"} - ] - } + {"location": "London, UK"}, + ], + }, } transformed_tool, _ = config._map_tool_helper(tool) @@ -1483,17 +1490,13 @@ def test_input_examples_with_other_features(): "description": "Execute a SQL query", "parameters": { "type": "object", - "properties": { - "sql": {"type": "string"} - }, - "required": ["sql"] - } + "properties": {"sql": {"type": "string"}}, + "required": ["sql"], + }, }, - "input_examples": [ - {"sql": "SELECT * FROM users WHERE id = 1"} - ], + "input_examples": [{"sql": "SELECT * FROM users WHERE id = 1"}], "defer_loading": True, - "allowed_callers": ["code_execution_20250825"] + "allowed_callers": ["code_execution_20250825"], } transformed_tool, _ = config._map_tool_helper(tool) @@ -1517,19 +1520,20 @@ def test_input_examples_empty_list_not_added(): "description": "Get weather information", "parameters": { "type": "object", - "properties": { - "location": {"type": "string"} - }, - "required": ["location"] - } + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, }, - "input_examples": [] + "input_examples": [], } transformed_tool, _ = config._map_tool_helper(tool) assert transformed_tool is not None # Empty list should not be added - assert "input_examples" not in transformed_tool or len(transformed_tool.get("input_examples", [])) == 0 + assert ( + "input_examples" not in transformed_tool + or len(transformed_tool.get("input_examples", [])) == 0 + ) # ============ Effort Parameter Tests ============ @@ -1540,18 +1544,14 @@ def test_effort_output_config_preservation(): config = AnthropicConfig() messages = [{"role": "user", "content": "Analyze this code"}] - optional_params = { - "output_config": { - "effort": "medium" - } - } + optional_params = {"output_config": {"effort": "medium"}} result = config.transform_request( model="claude-opus-4-5-20251101", messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) assert "output_config" in result @@ -1565,18 +1565,13 @@ def test_effort_beta_header_injection(): model_info = AnthropicModelInfo() # Test with effort parameter - optional_params = { - "output_config": { - "effort": "low" - } - } + optional_params = {"output_config": {"effort": "low"}} effort_used = model_info.is_effort_used(optional_params=optional_params) assert effort_used is True headers = model_info.get_anthropic_headers( - api_key="test-key", - effort_used=effort_used + api_key="test-key", effort_used=effort_used ) assert "anthropic-beta" in headers @@ -1597,7 +1592,7 @@ def test_effort_validation(): messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) assert result["output_config"]["effort"] == effort @@ -1609,7 +1604,7 @@ def test_effort_validation(): messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) @@ -1618,18 +1613,14 @@ def test_effort_with_claude_opus_45(): config = AnthropicConfig() messages = [{"role": "user", "content": "Complex analysis task"}] - optional_params = { - "output_config": { - "effort": "high" - } - } + optional_params = {"output_config": {"effort": "high"}} result = config.transform_request( model="claude-opus-4-5-20251101", messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) assert "output_config" in result @@ -1650,7 +1641,7 @@ def test_effort_validation_with_opus_46(): messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) assert result["output_config"]["effort"] == effort @@ -1661,14 +1652,16 @@ def test_max_effort_rejected_for_opus_45(): messages = [{"role": "user", "content": "Test"}] - with pytest.raises(ValueError, match="effort='max' is only supported by Claude Opus 4.6"): + with pytest.raises( + ValueError, match="effort='max' is only supported by Claude Opus 4.6" + ): optional_params = {"output_config": {"effort": "max"}} config.transform_request( model="claude-opus-4-5-20251101", messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) @@ -1685,23 +1678,16 @@ def test_effort_with_other_features(): "description": "Get data", "parameters": { "type": "object", - "properties": { - "query": {"type": "string"} - }, - "required": ["query"] - } - } + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + }, } ] optional_params = { - "output_config": { - "effort": "low" - }, + "output_config": {"effort": "low"}, "tools": tools, - "thinking": { - "type": "enabled", - "budget_tokens": 1000 - } + "thinking": {"type": "enabled", "budget_tokens": 1000}, } result = config.transform_request( @@ -1709,7 +1695,7 @@ def test_effort_with_other_features(): messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) # Verify all features are present @@ -1752,11 +1738,14 @@ def test_translate_system_message_skips_empty_list_content(): # Test list content with empty text block messages = [ - {"role": "system", "content": [ - {"type": "text", "text": ""}, - {"type": "text", "text": "Valid content"}, - {"type": "text", "text": ""}, - ]}, + { + "role": "system", + "content": [ + {"type": "text", "text": ""}, + {"type": "text", "text": "Valid content"}, + {"type": "text", "text": ""}, + ], + }, {"role": "user", "content": "Hello"}, ] @@ -1794,9 +1783,16 @@ def test_translate_system_message_preserves_cache_control(): # Test list content with cache_control messages = [ - {"role": "system", "content": [ - {"type": "text", "text": "Cached content", "cache_control": {"type": "ephemeral"}}, - ]}, + { + "role": "system", + "content": [ + { + "type": "text", + "text": "Cached content", + "cache_control": {"type": "ephemeral"}, + }, + ], + }, {"role": "user", "content": "Hello"}, ] @@ -1938,7 +1934,7 @@ def test_transform_request_uses_dynamic_max_tokens(): messages=messages, optional_params={}, # No max_tokens provided litellm_params={}, - headers={} + headers={}, ) assert result["max_tokens"] == 64000 @@ -1959,7 +1955,7 @@ def test_transform_request_respects_user_max_tokens(): messages=messages, optional_params={"max_tokens": 1000}, litellm_params={}, - headers={} + headers={}, ) assert result["max_tokens"] == 1000 @@ -2006,11 +2002,12 @@ def test_calculate_usage_completion_tokens_details_with_reasoning(): "output_tokens": 500, } # Simulating reasoning content that would count as ~50 tokens - reasoning_content = "Let me think about this step by step. " * 10 # Roughly 50 tokens + reasoning_content = ( + "Let me think about this step by step. " * 10 + ) # Roughly 50 tokens usage = config.calculate_usage( - usage_object=usage_object, - reasoning_content=reasoning_content + usage_object=usage_object, reasoning_content=reasoning_content ) # completion_tokens_details should be populated with both reasoning and text tokens @@ -2051,7 +2048,7 @@ def test_reasoning_effort_maps_to_adaptive_thinking_for_claude_4_6_models(): non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=False + drop_params=False, ) # Should map to adaptive thinking type @@ -2062,7 +2059,9 @@ def test_reasoning_effort_maps_to_adaptive_thinking_for_claude_4_6_models(): # reasoning_effort should not be in the result (it's transformed to thinking) assert "reasoning_effort" not in result # Should set output_config with the mapped effort value - assert "output_config" in result, f"output_config missing for {model} with effort={effort}" + assert ( + "output_config" in result + ), f"output_config missing for {model} with effort={effort}" assert result["output_config"]["effort"] == effort_map[effort] @@ -2123,10 +2122,10 @@ def test_reasoning_effort_maps_to_budget_thinking_for_non_opus_4_6(): # Test with Claude Sonnet 4.5 (non-Opus 4.6 model) test_cases = [ - ("low", 1024), # DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET - ("medium", 2048), # DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET - ("high", 4096), # DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET - ("minimal", 128), # DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET + ("low", 1024), # DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET + ("medium", 2048), # DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET + ("high", 4096), # DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET + ("minimal", 128), # DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET ] for effort, expected_budget in test_cases: @@ -2137,7 +2136,7 @@ def test_reasoning_effort_maps_to_budget_thinking_for_non_opus_4_6(): non_default_params=non_default_params, optional_params=optional_params, model="claude-sonnet-4-5-20250929", - drop_params=False + drop_params=False, ) # Should map to enabled thinking type with budget_tokens @@ -2166,9 +2165,9 @@ def test_reasoning_effort_sets_output_config_for_46_models(): drop_params=False, ) - assert "output_config" in result, ( - f"output_config missing for {model} with effort={effort}" - ) + assert ( + "output_config" in result + ), f"output_config missing for {model} with effort={effort}" assert result["output_config"]["effort"] == effort @@ -2207,9 +2206,9 @@ def test_reasoning_effort_does_not_set_output_config_for_older_models(): drop_params=False, ) - assert "output_config" not in result, ( - f"output_config should not be set for {model}" - ) + assert ( + "output_config" not in result + ), f"output_config should not be set for {model}" def test_max_effort_rejected_for_sonnet_46(): @@ -2217,7 +2216,9 @@ def test_max_effort_rejected_for_sonnet_46(): config = AnthropicConfig() messages = [{"role": "user", "content": "Test"}] - with pytest.raises(ValueError, match="effort='max' is only supported by Claude Opus 4.6"): + with pytest.raises( + ValueError, match="effort='max' is only supported by Claude Opus 4.6" + ): config.transform_request( model="claude-sonnet-4-6-20260219", messages=messages, @@ -2260,9 +2261,7 @@ def test_effort_beta_header_not_injected_for_46_models(): optional_params={"output_config": {"effort": "high"}}, model=model, ) - assert result is False, ( - f"is_effort_used should return False for {model}" - ) + assert result is False, f"is_effort_used should return False for {model}" def test_effort_beta_header_still_injected_for_older_models(): @@ -2302,17 +2301,12 @@ def test_code_execution_tool_results_extraction(): "role": "assistant", "model": "claude-sonnet-4-5-20250929", "content": [ - { - "type": "text", - "text": "I'll calculate that for you." - }, + {"type": "text", "text": "I'll calculate that for you."}, { "type": "server_tool_use", "id": "srvtoolu_01ABC", "name": "bash_code_execution", - "input": { - "command": "python3 << 'EOF'\nprint(2 + 2)\nEOF\n" - } + "input": {"command": "python3 << 'EOF'\nprint(2 + 2)\nEOF\n"}, }, { "type": "bash_code_execution_tool_result", @@ -2321,8 +2315,8 @@ def test_code_execution_tool_results_extraction(): "type": "bash_code_execution_result", "stdout": "4\n", "stderr": "", - "return_code": 0 - } + "return_code": 0, + }, }, { "type": "server_tool_use", @@ -2331,28 +2325,22 @@ def test_code_execution_tool_results_extraction(): "input": { "command": "create", "path": "test.txt", - "file_text": "Hello" - } + "file_text": "Hello", + }, }, { "type": "text_editor_code_execution_tool_result", "tool_use_id": "srvtoolu_01DEF", "content": { "type": "text_editor_code_execution_result", - "is_file_update": False - } + "is_file_update": False, + }, }, - { - "type": "text", - "text": "Done!" - } + {"type": "text", "text": "Done!"}, ], "stop_reason": "stop", "stop_sequence": None, - "usage": { - "input_tokens": 100, - "output_tokens": 50 - } + "usage": {"input_tokens": 100, "output_tokens": 50}, } # Create mock HTTP response @@ -2377,11 +2365,17 @@ def test_code_execution_tool_results_extraction(): # Verify first tool call assert transformed_response.choices[0].message.tool_calls[0].id == "srvtoolu_01ABC" - assert transformed_response.choices[0].message.tool_calls[0].function.name == "bash_code_execution" + assert ( + transformed_response.choices[0].message.tool_calls[0].function.name + == "bash_code_execution" + ) # Verify second tool call assert transformed_response.choices[0].message.tool_calls[1].id == "srvtoolu_01DEF" - assert transformed_response.choices[0].message.tool_calls[1].function.name == "text_editor_code_execution" + assert ( + transformed_response.choices[0].message.tool_calls[1].function.name + == "text_editor_code_execution" + ) # Verify tool results are in provider_specific_fields provider_fields = transformed_response.choices[0].message.provider_specific_fields @@ -2404,10 +2398,83 @@ def test_code_execution_tool_results_extraction(): assert editor_result["content"]["is_file_update"] is False # Verify text content is properly concatenated - assert "I'll calculate that for you." in transformed_response.choices[0].message.content + assert ( + "I'll calculate that for you." + in transformed_response.choices[0].message.content + ) assert "Done!" in transformed_response.choices[0].message.content +def test_code_execution_tool_results_in_hidden_params(): + """ + Test that tool_results reaches _hidden_params so the Responses API adapter + can surface them via provider_specific_fields. + + The Responses API adapter reads _hidden_params.get("provider_specific_fields") + to set provider_specific_fields on the response. Without this, server-side + code execution results (stdout/stderr) are lost when using responses.create(). + """ + import httpx + + from litellm.types.utils import ModelResponse + + config = AnthropicConfig() + + mock_anthropic_response = { + "id": "msg_01XYZ", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5-20250929", + "content": [ + {"type": "text", "text": "Here's the result."}, + { + "type": "server_tool_use", + "id": "srvtoolu_01ABC", + "name": "bash_code_execution", + "input": {"command": "echo hello"}, + }, + { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01ABC", + "content": { + "type": "bash_code_execution_result", + "stdout": "hello\n", + "stderr": "", + "return_code": 0, + }, + }, + ], + "stop_reason": "stop", + "stop_sequence": None, + "usage": {"input_tokens": 100, "output_tokens": 50}, + } + + mock_raw_response = MagicMock(spec=httpx.Response) + mock_raw_response.json.return_value = mock_anthropic_response + mock_raw_response.status_code = 200 + mock_raw_response.headers = {} + + model_response = ModelResponse() + + transformed_response = config.transform_parsed_response( + completion_response=mock_anthropic_response, + raw_response=mock_raw_response, + model_response=model_response, + json_mode=False, + prefix_prompt=None, + ) + + # Verify tool_results is in _hidden_params for the Responses API adapter + hidden = transformed_response._hidden_params + assert "provider_specific_fields" in hidden + assert "tool_results" in hidden["provider_specific_fields"] + assert len(hidden["provider_specific_fields"]["tool_results"]) == 1 + assert ( + hidden["provider_specific_fields"]["tool_results"][0]["content"]["stdout"] + == "hello\n" + ) + + def test_tool_search_tool_result_not_in_tool_results(): """ Test that tool_search_tool_result is NOT included in tool_results @@ -2425,21 +2492,12 @@ def test_tool_search_tool_result_not_in_tool_results(): "role": "assistant", "model": "claude-sonnet-4-5-20250929", "content": [ - { - "type": "text", - "text": "Found tools." - }, - { - "type": "tool_search_tool_result", - "tool_references": ["tool1", "tool2"] - } + {"type": "text", "text": "Found tools."}, + {"type": "tool_search_tool_result", "tool_references": ["tool1", "tool2"]}, ], "stop_reason": "stop", "stop_sequence": None, - "usage": { - "input_tokens": 100, - "output_tokens": 50 - } + "usage": {"input_tokens": 100, "output_tokens": 50}, } mock_raw_response = MagicMock(spec=httpx.Response) @@ -2479,22 +2537,16 @@ def test_web_search_tool_result_backwards_compatibility(): "role": "assistant", "model": "claude-sonnet-4-5-20250929", "content": [ - { - "type": "text", - "text": "Here are the results." - }, + {"type": "text", "text": "Here are the results."}, { "type": "web_search_tool_result", "search_query": "test query", - "results": [{"title": "Result 1", "url": "https://example.com"}] - } + "results": [{"title": "Result 1", "url": "https://example.com"}], + }, ], "stop_reason": "stop", "stop_sequence": None, - "usage": { - "input_tokens": 100, - "output_tokens": 50 - } + "usage": {"input_tokens": 100, "output_tokens": 50}, } mock_raw_response = MagicMock(spec=httpx.Response) @@ -2540,24 +2592,28 @@ def test_compaction_block_extraction(): "content": [ { "type": "compaction", - "content": "Summary of the conversation: The user requested help building a web scraper..." + "content": "Summary of the conversation: The user requested help building a web scraper...", }, { "type": "text", - "text": "I don't have access to real-time data, so I can't provide the current weather in San Francisco." - } + "text": "I don't have access to real-time data, so I can't provide the current weather in San Francisco.", + }, ], "stop_reason": "max_tokens", "stop_sequence": None, - "usage": { - "input_tokens": 86, - "output_tokens": 100 - } + "usage": {"input_tokens": 86, "output_tokens": 100}, } - text, citations, thinking_blocks, reasoning_content, tool_calls, web_search_results, tool_results, compaction_blocks = config.extract_response_content( - completion_response - ) + ( + text, + citations, + thinking_blocks, + reasoning_content, + tool_calls, + web_search_results, + tool_results, + compaction_blocks, + ) = config.extract_response_content(completion_response) # Verify compaction blocks are extracted assert compaction_blocks is not None @@ -2587,18 +2643,12 @@ def test_compaction_block_in_provider_specific_fields(): "content": [ { "type": "compaction", - "content": "Summary of the conversation: The user requested help building a web scraper..." + "content": "Summary of the conversation: The user requested help building a web scraper...", }, - { - "type": "text", - "text": "Here is the response." - } + {"type": "text", "text": "Here is the response."}, ], "stop_reason": "end_turn", - "usage": { - "input_tokens": 50, - "output_tokens": 25 - } + "usage": {"input_tokens": 50, "output_tokens": 25}, } raw_response = httpx.Response(status_code=200, headers={}) @@ -2618,7 +2668,10 @@ def test_compaction_block_in_provider_specific_fields(): assert "compaction_blocks" in provider_fields assert len(provider_fields["compaction_blocks"]) == 1 assert provider_fields["compaction_blocks"][0]["type"] == "compaction" - assert "Summary of the conversation" in provider_fields["compaction_blocks"][0]["content"] + assert ( + "Summary of the conversation" + in provider_fields["compaction_blocks"][0]["content"] + ) def test_multiple_compaction_blocks(): @@ -2629,24 +2682,22 @@ def test_multiple_compaction_blocks(): completion_response = { "content": [ - { - "type": "compaction", - "content": "First summary..." - }, - { - "type": "text", - "text": "Some text." - }, - { - "type": "compaction", - "content": "Second summary..." - } + {"type": "compaction", "content": "First summary..."}, + {"type": "text", "text": "Some text."}, + {"type": "compaction", "content": "Second summary..."}, ] } - text, citations, thinking_blocks, reasoning_content, tool_calls, web_search_results, tool_results, compaction_blocks = config.extract_response_content( - completion_response - ) + ( + text, + citations, + thinking_blocks, + reasoning_content, + tool_calls, + web_search_results, + tool_results, + compaction_blocks, + ) = config.extract_response_content(completion_response) # Verify both compaction blocks are extracted assert compaction_blocks is not None @@ -2665,37 +2716,26 @@ def test_compaction_block_request_transformation(): ) messages = [ - { - "role": "user", - "content": "What is the weather in San Francisco?" - }, + {"role": "user", "content": "What is the weather in San Francisco?"}, { "role": "assistant", "content": [ - { - "type": "text", - "text": "I don't have access to real-time data." - } + {"type": "text", "text": "I don't have access to real-time data."} ], "provider_specific_fields": { "compaction_blocks": [ { "type": "compaction", - "content": "Summary of the conversation: The user requested help building a web scraper..." + "content": "Summary of the conversation: The user requested help building a web scraper...", } ] - } + }, }, - { - "role": "user", - "content": "What about New York?" - } + {"role": "user", "content": "What about New York?"}, ] result = anthropic_messages_pt( - messages=messages, - model="claude-opus-4-6", - llm_provider="anthropic" + messages=messages, model="claude-opus-4-6", llm_provider="anthropic" ) # Find the assistant message @@ -2727,14 +2767,8 @@ def test_compaction_with_context_management(): messages = [{"role": "user", "content": "Hello"}] optional_params = { - "context_management": { - "edits": [ - { - "type": "compact_20260112" - } - ] - }, - "max_tokens": 100 + "context_management": {"edits": [{"type": "compact_20260112"}]}, + "max_tokens": 100, } result = config.transform_request( @@ -2742,7 +2776,7 @@ def test_compaction_with_context_management(): messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) # Verify context_management is included @@ -2758,30 +2792,28 @@ def test_compaction_block_with_other_content_types(): completion_response = { "content": [ - { - "type": "compaction", - "content": "Summary of previous conversation..." - }, - { - "type": "thinking", - "thinking": "Let me think about this..." - }, - { - "type": "text", - "text": "Based on my analysis..." - }, + {"type": "compaction", "content": "Summary of previous conversation..."}, + {"type": "thinking", "thinking": "Let me think about this..."}, + {"type": "text", "text": "Based on my analysis..."}, { "type": "tool_use", "id": "toolu_123", "name": "get_weather", - "input": {"location": "San Francisco"} - } + "input": {"location": "San Francisco"}, + }, ] } - text, citations, thinking_blocks, reasoning_content, tool_calls, web_search_results, tool_results, compaction_blocks = config.extract_response_content( - completion_response - ) + ( + text, + citations, + thinking_blocks, + reasoning_content, + tool_calls, + web_search_results, + tool_results, + compaction_blocks, + ) = config.extract_response_content(completion_response) # Verify all content types are extracted assert compaction_blocks is not None @@ -2798,11 +2830,11 @@ def test_map_openai_context_management_to_anthropic(): Test mapping OpenAI Responses API context_management format to Anthropic format. """ config = AnthropicConfig() - + # Test OpenAI list format with compaction openai_format = [{"type": "compaction", "compact_threshold": 200000}] result = config.map_openai_context_management_to_anthropic(openai_format) - + assert result is not None assert "edits" in result assert len(result["edits"]) == 1 @@ -2811,26 +2843,32 @@ def test_map_openai_context_management_to_anthropic(): assert result["edits"][0]["trigger"]["value"] == 200000 # Test OpenAI format with instructions - openai_format_with_instructions = [{ - "type": "compaction", - "compact_threshold": 150000, - "instructions": "Focus on preserving code snippets" - }] - result = config.map_openai_context_management_to_anthropic(openai_format_with_instructions) - + openai_format_with_instructions = [ + { + "type": "compaction", + "compact_threshold": 150000, + "instructions": "Focus on preserving code snippets", + } + ] + result = config.map_openai_context_management_to_anthropic( + openai_format_with_instructions + ) + assert result is not None assert result["edits"][0]["trigger"]["value"] == 150000 assert result["edits"][0]["instructions"] == "Focus on preserving code snippets" - + # Test Anthropic format (should pass through) anthropic_format = { - "edits": [{ - "type": "compact_20260112", - "trigger": {"type": "input_tokens", "value": 150000} - }] + "edits": [ + { + "type": "compact_20260112", + "trigger": {"type": "input_tokens", "value": 150000}, + } + ] } result = config.map_openai_context_management_to_anthropic(anthropic_format) - + assert result == anthropic_format @@ -2839,46 +2877,51 @@ def test_map_openai_params_with_context_management(): Test that map_openai_params correctly transforms context_management from OpenAI to Anthropic format. """ config = AnthropicConfig() - + # Test with OpenAI list format non_default_params = { "context_management": [{"type": "compaction", "compact_threshold": 200000}] } optional_params = {} - + result = config.map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model="claude-opus-4-6", - drop_params=False + drop_params=False, ) - + assert "context_management" in result assert "edits" in result["context_management"] assert result["context_management"]["edits"][0]["type"] == "compact_20260112" assert result["context_management"]["edits"][0]["trigger"]["value"] == 200000 - + # Test with Anthropic dict format (should pass through) non_default_params_anthropic = { "context_management": { - "edits": [{ - "type": "compact_20260112", - "trigger": {"type": "input_tokens", "value": 150000}, - "instructions": "Focus on preserving code" - }] + "edits": [ + { + "type": "compact_20260112", + "trigger": {"type": "input_tokens", "value": 150000}, + "instructions": "Focus on preserving code", + } + ] } } optional_params = {} - + result = config.map_openai_params( non_default_params=non_default_params_anthropic, optional_params=optional_params, model="claude-opus-4-6", - drop_params=False + drop_params=False, ) - + assert "context_management" in result - assert result["context_management"] == non_default_params_anthropic["context_management"] + assert ( + result["context_management"] + == non_default_params_anthropic["context_management"] + ) def test_cache_control_in_supported_params(): @@ -2897,9 +2940,7 @@ def test_map_openai_params_with_cache_control(): """ config = AnthropicConfig() - non_default_params = { - "cache_control": {"type": "ephemeral"} - } + non_default_params = {"cache_control": {"type": "ephemeral"}} optional_params = {} result = config.map_openai_params( @@ -2919,9 +2960,7 @@ def test_map_openai_params_cache_control_ignored_when_not_dict(): """ config = AnthropicConfig() - non_default_params = { - "cache_control": "ephemeral" - } + non_default_params = {"cache_control": "ephemeral"} optional_params = {} result = config.map_openai_params( @@ -2974,17 +3013,9 @@ def test_compaction_block_empty_list_not_added(): "type": "message", "role": "assistant", "model": "claude-opus-4-6", - "content": [ - { - "type": "text", - "text": "Just a regular response." - } - ], + "content": [{"type": "text", "text": "Just a regular response."}], "stop_reason": "end_turn", - "usage": { - "input_tokens": 10, - "output_tokens": 5 - } + "usage": {"input_tokens": 10, "output_tokens": 5}, } raw_response = httpx.Response(status_code=200, headers={}) @@ -3001,7 +3032,10 @@ def test_compaction_block_empty_list_not_added(): # Verify compaction_blocks is not in provider_specific_fields when there are none provider_fields = result.choices[0].message.provider_specific_fields if provider_fields: - assert "compaction_blocks" not in provider_fields or provider_fields.get("compaction_blocks") is None + assert ( + "compaction_blocks" not in provider_fields + or provider_fields.get("compaction_blocks") is None + ) def test_fast_mode_beta_header(): @@ -3014,8 +3048,7 @@ def test_fast_mode_beta_header(): optional_params = {"speed": "fast"} result_headers = config.update_headers_with_optional_anthropic_beta( - headers=headers, - optional_params=optional_params + headers=headers, optional_params=optional_params ) assert "anthropic-beta" in result_headers @@ -3029,14 +3062,10 @@ def test_fast_mode_with_other_beta_headers(): config = AnthropicConfig() headers = {} - optional_params = { - "speed": "fast", - "output_format": {"type": "json_object"} - } + optional_params = {"speed": "fast", "output_format": {"type": "json_object"}} result_headers = config.update_headers_with_optional_anthropic_beta( - headers=headers, - optional_params=optional_params + headers=headers, optional_params=optional_params ) assert "anthropic-beta" in result_headers @@ -3056,9 +3085,7 @@ def test_fast_mode_usage_calculation(): } usage = config.calculate_usage( - usage_object=usage_object, - reasoning_content=None, - speed="fast" + usage_object=usage_object, reasoning_content=None, speed="fast" ) assert usage.prompt_tokens == 1000 @@ -3171,7 +3198,7 @@ def test_fast_mode_parameter_mapping(): non_default_params=non_default_params, optional_params=optional_params, model="claude-opus-4-6", - drop_params=False + drop_params=False, ) assert "speed" in result @@ -3236,9 +3263,9 @@ def test_map_tool_helper_enforces_object_type_when_missing(): assert "properties" in result["input_schema"] assert "query" in result["input_schema"]["properties"] # Original parameters dict must not be modified in place - assert tool["function"]["parameters"] == original_params, ( - "parameters dict was mutated; _map_tool_helper should not modify caller data" - ) + assert ( + tool["function"]["parameters"] == original_params + ), "parameters dict was mutated; _map_tool_helper should not modify caller data" def test_map_tool_helper_enforces_object_type_when_wrong_type(): @@ -3264,13 +3291,13 @@ def test_map_tool_helper_enforces_object_type_when_wrong_type(): result, _ = config._map_tool_helper(tool) assert result is not None assert result["input_schema"]["type"] == "object" - assert result["input_schema"].get("properties") == {}, ( - "properties should be injected as {} when schema has non-object type and no properties key" - ) + assert ( + result["input_schema"].get("properties") == {} + ), "properties should be injected as {} when schema has non-object type and no properties key" # Original parameters dict must not be modified in place - assert tool["function"]["parameters"] == original_params, ( - "parameters dict was mutated; _map_tool_helper should not modify caller data" - ) + assert ( + tool["function"]["parameters"] == original_params + ), "parameters dict was mutated; _map_tool_helper should not modify caller data" def test_map_tool_helper_preserves_valid_object_schema(): From 2bf8751f6b0cfe30b51e4bb298c7e36ebc4ea46b Mon Sep 17 00:00:00 2001 From: Andrzej Pomirski Date: Tue, 17 Mar 2026 17:57:25 +0100 Subject: [PATCH 07/66] fix: streaming code_interpreter_results dropped for multiple code executions stream_chunk_builder uses "last value wins" for list-valued provider_specific_fields keys. _build_code_interpreter_results was emitting only new items (incremental), so earlier results were silently dropped when multiple sequential code executions occurred. - Emit cumulative list from _build_code_interpreter_results, matching web_search_results pattern - Assemble server_tool_use input from input_json_delta deltas at content_block_stop (Anthropic streams input: {} in start block) - Handle dict items in _extract_tool_result_output_items after model_dump() serialization in stream_chunk_builder - Simplify _merge_provider_specific_fields to last-value-wins for lists, matching stream_chunk_builder semantics --- litellm/llms/anthropic/chat/handler.py | 45 ++++-- .../streaming_iterator.py | 19 ++- .../transformation.py | 5 +- .../chat/test_anthropic_chat_handler.py | 133 +++++++++++++++--- 4 files changed, 165 insertions(+), 37 deletions(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 51b9c9835a7..7d5fa2a5591 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -545,7 +545,7 @@ class ModelResponseIterator: # Track server tool use inputs and results for code_interpreter_results self._server_tool_inputs: Dict[str, Any] = {} self.tool_results: List[Dict[str, Any]] = [] - self._last_code_interpreter_results_count: int = 0 + self._current_server_tool_id: Optional[str] = None def check_empty_tool_call_args(self) -> bool: """ @@ -695,13 +695,14 @@ class ModelResponseIterator: Called during streaming to produce provider-neutral code_interpreter_results alongside the raw tool_results, so the Responses API layer doesn't need Anthropic-specific knowledge. + + Returns the full cumulative list each time (not incremental), matching + how web_search_results works. stream_chunk_builder uses "last value + wins" for list-valued provider_specific_fields keys, so the last + emission must contain every result. """ - # Only convert tool_results added since the last call to avoid - # duplicates when _merge_provider_specific_fields extends the list. - new_results = self.tool_results[self._last_code_interpreter_results_count :] - self._last_code_interpreter_results_count = len(self.tool_results) results = [] - for tr in new_results: + for tr in self.tool_results: call_id = tr.get("tool_use_id", "") content = tr.get("content", {}) if isinstance(content, dict): @@ -793,17 +794,23 @@ class ModelResponseIterator: ), index=self.tool_index, ) - # Track server tool use inputs for code_interpreter_results + # Track server tool use inputs for code_interpreter_results. + # The initial input in content_block_start is typically {} + # for streaming; the full input arrives via input_json_delta + # and is assembled at content_block_stop. if ( content_block_start["content_block"]["type"] == "server_tool_use" ): + self._current_server_tool_id = content_block_start[ + "content_block" + ]["id"] tool_input = content_block_start["content_block"].get( "input", {} ) - self._server_tool_inputs[ - content_block_start["content_block"]["id"] - ] = tool_input + self._server_tool_inputs[self._current_server_tool_id] = ( + tool_input + ) # Include caller information if present (for programmatic tool calling) if "caller" in content_block_start["content_block"]: caller_data = content_block_start["content_block"]["caller"] @@ -886,6 +893,24 @@ class ModelResponseIterator: ), index=self.tool_index, ) + # Update server_tool_inputs with fully assembled input + # from input_json_delta chunks (content_block_start has {}) + if ( + self.current_content_block_type == "server_tool_use" + and self._current_server_tool_id + ): + args = "" + for block in self.content_blocks: + if block["delta"]["type"] == "input_json_delta": + args += block["delta"].get("partial_json", "") + if args: + try: + self._server_tool_inputs[ + self._current_server_tool_id + ] = json.loads(args) + except (json.JSONDecodeError, TypeError): + pass + self._current_server_tool_id = None # Reset response_format tool tracking when block stops self.is_response_format_tool = False # Reset current content block type diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 0b7d6e8a7a4..0672b03bcd7 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -481,17 +481,16 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return event def _merge_provider_specific_fields(self, src: dict) -> None: - """Merge provider_specific_fields, extending list values instead of replacing.""" + """Merge provider_specific_fields using last-value-wins for lists. + + List-valued keys (web_search_results, tool_results, + code_interpreter_results, etc.) are emitted cumulatively — each + emission contains the full list so far. Using "last value wins" + matches stream_chunk_builder's semantics and avoids quadratic + growth from repeated extend calls. + """ for key, val in src.items(): - existing = self._accumulated_provider_specific_fields.get(key) - if ( - existing is not None - and isinstance(val, list) - and isinstance(existing, list) - ): - existing.extend(val) - else: - self._accumulated_provider_specific_fields[key] = val + self._accumulated_provider_specific_fields[key] = val def create_litellm_model_response(self) -> Optional[ModelResponse]: response = cast( diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index b54d5930efc..b7f7e9adda2 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1738,7 +1738,10 @@ class LiteLLMCompletionResponsesConfig: ) ) if tool_result_items: - result_by_id = {item.id: item for item in tool_result_items} + result_by_id = { + (item.get("id") if isinstance(item, dict) else item.id): item + for item in tool_result_items + } replaced_ids = set(result_by_id.keys()) responses_output = [ ( diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index 35c7a62027b..d7a04a054ab 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -1245,9 +1245,9 @@ def test_streaming_code_execution_produces_code_interpreter_results(): def test_streaming_multiple_code_executions_no_duplicates(): """ - Test that multiple code executions in a single streaming response produce - exactly one code_interpreter_result per execution — no duplicates from - _build_code_interpreter_results rebuilding the full list. + Test that multiple code executions in a single streaming response emit + cumulative code_interpreter_results on each chunk (matching stream_chunk_builder's + "last value wins" contract). The final emission must contain ALL results. """ chunks = [ { @@ -1323,24 +1323,125 @@ def test_streaming_multiple_code_executions_no_duplicates(): iterator = ModelResponseIterator(None, sync_stream=True) - # Collect ALL code_interpreter_results emitted across all chunks - all_results = [] + # Collect each emission of code_interpreter_results + emissions = [] for chunk in chunks: parsed = iterator.chunk_parser(chunk) psf = None if parsed.choices and parsed.choices[0].delta: psf = getattr(parsed.choices[0].delta, "provider_specific_fields", None) if psf and "code_interpreter_results" in psf: - all_results.extend(psf["code_interpreter_results"]) + emissions.append(psf["code_interpreter_results"]) - # Should have exactly 2 results, one per execution — no duplicates - assert len(all_results) == 2, ( - f"Expected 2 code_interpreter_results, got {len(all_results)}. " - f"IDs: {[r.id for r in all_results]}" + # Should have 2 emissions (one per tool_result block) + assert len(emissions) == 2, f"Expected 2 emissions, got {len(emissions)}" + + # First emission: cumulative list with 1 result + assert len(emissions[0]) == 1 + assert emissions[0][0].id == "srvtoolu_01AAA" + assert emissions[0][0].code == "echo first" + assert emissions[0][0].outputs[0].logs == "first\n" + + # Second (final) emission: cumulative list with BOTH results + # This is what stream_chunk_builder will pick as "last value wins" + assert len(emissions[1]) == 2, ( + f"Expected final emission to have 2 results, got {len(emissions[1])}. " + f"IDs: {[r.id for r in emissions[1]]}" ) - assert all_results[0].id == "srvtoolu_01AAA" - assert all_results[0].code == "echo first" - assert all_results[0].outputs[0].logs == "first\n" - assert all_results[1].id == "srvtoolu_01BBB" - assert all_results[1].code == "echo second" - assert all_results[1].outputs[0].logs == "second\n" + assert emissions[1][0].id == "srvtoolu_01AAA" + assert emissions[1][0].code == "echo first" + assert emissions[1][0].outputs[0].logs == "first\n" + assert emissions[1][1].id == "srvtoolu_01BBB" + assert emissions[1][1].code == "echo second" + assert emissions[1][1].outputs[0].logs == "second\n" + + +def test_streaming_code_execution_input_assembled_from_deltas(): + """ + In real Anthropic streaming, content_block_start for server_tool_use has + input: {}. The actual input arrives via input_json_delta deltas and must + be assembled at content_block_stop so the code field is populated. + + This test uses realistic chunk shapes (empty input in start, partial JSON + in deltas) to exercise the input assembly path. + """ + chunks = [ + { + "type": "message_start", + "message": { + "id": "msg_01XYZ", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 100, "output_tokens": 1}, + }, + }, + # server_tool_use with empty input (real streaming behaviour) + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "server_tool_use", + "id": "srvtoolu_01AAA", + "name": "code_execution", + "input": {}, + }, + }, + # Input arrives via deltas, split across two chunks + { + "type": "content_block_delta", + "index": 0, + "delta": { + "type": "input_json_delta", + "partial_json": '{"comma', + }, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": { + "type": "input_json_delta", + "partial_json": 'nd": "echo hello"}', + }, + }, + {"type": "content_block_stop", "index": 0}, + # Tool result + { + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "code_execution_tool_result", + "tool_use_id": "srvtoolu_01AAA", + "content": { + "type": "code_execution_result", + "stdout": "hello\n", + "stderr": "", + "return_code": 0, + }, + }, + }, + {"type": "content_block_stop", "index": 1}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 50}, + }, + ] + + iterator = ModelResponseIterator(None, sync_stream=True) + + code_results = None + for chunk in chunks: + parsed = iterator.chunk_parser(chunk) + psf = None + if parsed.choices and parsed.choices[0].delta: + psf = getattr(parsed.choices[0].delta, "provider_specific_fields", None) + if psf and "code_interpreter_results" in psf: + code_results = psf["code_interpreter_results"] + + # The code field must contain the assembled input, not be empty + assert code_results is not None, "No code_interpreter_results emitted" + assert len(code_results) == 1 + assert code_results[0].id == "srvtoolu_01AAA" + assert code_results[0].code == "echo hello" + assert code_results[0].outputs[0].logs == "hello\n" From 4be1d76fd7da5c4c0b04f1ead59e98bcf54d1dfb Mon Sep 17 00:00:00 2001 From: Andrzej Pomirski Date: Tue, 17 Mar 2026 18:37:32 +0100 Subject: [PATCH 08/66] fix: empty stdout/stderr produces str(content) instead of empty logs When both stdout and stderr are empty strings, the `if parts else str(content)` fallback produced the raw dict representation as logs. Drop the fallback so logs is correctly empty. --- litellm/llms/anthropic/chat/handler.py | 2 +- litellm/llms/anthropic/chat/transformation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 7d5fa2a5591..88d9ee65969 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -711,7 +711,7 @@ class ModelResponseIterator: parts.append(content["stdout"]) if content.get("stderr"): parts.append(f"STDERR: {content['stderr']}") - logs = "".join(parts) if parts else str(content) + logs = "".join(parts) else: logs = str(content) tool_input = self._server_tool_inputs.get(call_id, {}) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 033afea2ffa..21ca8db825c 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1775,7 +1775,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): parts.append(content["stdout"]) if content.get("stderr"): parts.append(f"STDERR: {content['stderr']}") - logs = "".join(parts) if parts else str(content) + logs = "".join(parts) else: logs = str(content) code_interpreter_results.append( From 5b3e84f383627c951c4d7bf5f150e4173c39fdc4 Mon Sep 17 00:00:00 2001 From: Andrzej Pomirski Date: Tue, 17 Mar 2026 18:59:47 +0100 Subject: [PATCH 09/66] fix: address remaining review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Empty stdout/stderr now produces outputs=None (matching OpenAI parity) instead of outputs=[{logs:""}], in both streaming and non-streaming paths - Fix test fixture to use real Anthropic type "bash_code_execution_tool_result" instead of "code_execution_tool_result" - Add test for empty-output → outputs=None behavior - Add unit tests for _extract_tool_result_output_items: Pydantic objects, plain dicts (post-model_dump), empty/missing provider_specific_fields, and in-place substitution preserving output ordering --- litellm/llms/anthropic/chat/handler.py | 5 +- litellm/llms/anthropic/chat/transformation.py | 9 +- .../chat/test_anthropic_chat_handler.py | 72 +++++++- ...est_code_interpreter_results_extraction.py | 163 ++++++++++++++++++ 4 files changed, 243 insertions(+), 6 deletions(-) create mode 100644 tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 88d9ee65969..0b84830a8b0 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -716,6 +716,9 @@ class ModelResponseIterator: logs = str(content) tool_input = self._server_tool_inputs.get(call_id, {}) code = tool_input.get("command", "") if isinstance(tool_input, dict) else "" + log_outputs = ( + [OutputCodeInterpreterCallLog(type="logs", logs=logs)] if logs else None + ) results.append( OutputCodeInterpreterCall( type="code_interpreter_call", @@ -723,7 +726,7 @@ class ModelResponseIterator: code=code, container_id=None, status="completed", - outputs=[OutputCodeInterpreterCallLog(type="logs", logs=logs)], + outputs=log_outputs, ) ) return results diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 21ca8db825c..99b02e7ab46 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1778,6 +1778,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): logs = "".join(parts) else: logs = str(content) + log_outputs = ( + [OutputCodeInterpreterCallLog(type="logs", logs=logs)] + if logs + else None + ) code_interpreter_results.append( OutputCodeInterpreterCall( type="code_interpreter_call", @@ -1785,9 +1790,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): code=code_by_id.get(call_id, ""), container_id=container_id, status="completed", - outputs=[ - OutputCodeInterpreterCallLog(type="logs", logs=logs) - ], + outputs=log_outputs, ) ) provider_specific_fields["code_interpreter_results"] = ( diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index d7a04a054ab..ab298f6809b 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -1410,10 +1410,10 @@ def test_streaming_code_execution_input_assembled_from_deltas(): "type": "content_block_start", "index": 1, "content_block": { - "type": "code_execution_tool_result", + "type": "bash_code_execution_tool_result", "tool_use_id": "srvtoolu_01AAA", "content": { - "type": "code_execution_result", + "type": "bash_code_execution_result", "stdout": "hello\n", "stderr": "", "return_code": 0, @@ -1445,3 +1445,71 @@ def test_streaming_code_execution_input_assembled_from_deltas(): assert code_results[0].id == "srvtoolu_01AAA" assert code_results[0].code == "echo hello" assert code_results[0].outputs[0].logs == "hello\n" + + +def test_empty_output_produces_null_outputs(): + """ + When both stdout and stderr are empty, outputs should be None + (matching OpenAI's native behavior) rather than [{logs: ""}]. + """ + chunks = [ + { + "type": "message_start", + "message": { + "id": "msg_01XYZ", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 100, "output_tokens": 1}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "server_tool_use", + "id": "srvtoolu_01AAA", + "name": "bash_code_execution", + "input": {"command": "true"}, + }, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01AAA", + "content": { + "type": "bash_code_execution_result", + "stdout": "", + "stderr": "", + "return_code": 0, + }, + }, + }, + {"type": "content_block_stop", "index": 1}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 50}, + }, + ] + + iterator = ModelResponseIterator(None, sync_stream=True) + + code_results = None + for chunk in chunks: + parsed = iterator.chunk_parser(chunk) + psf = None + if parsed.choices and parsed.choices[0].delta: + psf = getattr(parsed.choices[0].delta, "provider_specific_fields", None) + if psf and "code_interpreter_results" in psf: + code_results = psf["code_interpreter_results"] + + assert code_results is not None, "No code_interpreter_results emitted" + assert len(code_results) == 1 + assert code_results[0].id == "srvtoolu_01AAA" + assert ( + code_results[0].outputs is None + ), f"Expected outputs=None for empty execution, got {code_results[0].outputs}" diff --git a/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py b/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py new file mode 100644 index 00000000000..c6ff1c7af8f --- /dev/null +++ b/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py @@ -0,0 +1,163 @@ +""" +Tests for the Responses API _extract_tool_result_output_items path +and the non-streaming _hidden_params propagation of code_interpreter_results. +""" + +from unittest.mock import MagicMock + +from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, +) +from litellm.types.responses.main import ( + OutputCodeInterpreterCall, + OutputCodeInterpreterCallLog, +) +from litellm.types.utils import Choices, Message, ModelResponse + + +def _make_model_response(code_interpreter_results=None, provider_specific_fields=None): + """Helper to build a ModelResponse with provider_specific_fields on the message.""" + psf = provider_specific_fields or {} + if code_interpreter_results is not None: + psf["code_interpreter_results"] = code_interpreter_results + msg = Message(content="test", provider_specific_fields=psf if psf else None) + choice = Choices(index=0, message=msg, finish_reason="stop") + resp = ModelResponse() + resp.choices = [choice] + return resp + + +def test_extract_tool_result_output_items_from_pydantic_objects(): + """Non-streaming path: code_interpreter_results are Pydantic OutputCodeInterpreterCall objects.""" + items = [ + OutputCodeInterpreterCall( + type="code_interpreter_call", + id="srvtoolu_01AAA", + code="echo hello", + container_id=None, + status="completed", + outputs=[OutputCodeInterpreterCallLog(type="logs", logs="hello\n")], + ), + OutputCodeInterpreterCall( + type="code_interpreter_call", + id="srvtoolu_01BBB", + code="echo world", + container_id=None, + status="completed", + outputs=[OutputCodeInterpreterCallLog(type="logs", logs="world\n")], + ), + ] + resp = _make_model_response(code_interpreter_results=items) + result = LiteLLMCompletionResponsesConfig._extract_tool_result_output_items(resp) + assert len(result) == 2 + assert result[0].id == "srvtoolu_01AAA" + assert result[1].id == "srvtoolu_01BBB" + + +def test_extract_tool_result_output_items_from_dicts(): + """Streaming path: after model_dump(), code_interpreter_results are plain dicts.""" + items = [ + { + "type": "code_interpreter_call", + "id": "srvtoolu_01AAA", + "code": "echo hello", + "container_id": None, + "status": "completed", + "outputs": [{"type": "logs", "logs": "hello\n"}], + }, + ] + resp = _make_model_response(code_interpreter_results=items) + result = LiteLLMCompletionResponsesConfig._extract_tool_result_output_items(resp) + assert len(result) == 1 + assert result[0]["id"] == "srvtoolu_01AAA" + + +def test_extract_tool_result_output_items_empty(): + """No code_interpreter_results → empty list.""" + resp = _make_model_response() + result = LiteLLMCompletionResponsesConfig._extract_tool_result_output_items(resp) + assert result == [] + + +def test_extract_tool_result_output_items_no_provider_specific_fields(): + """Message with no provider_specific_fields → empty list.""" + msg = Message(content="test") + choice = Choices(index=0, message=msg, finish_reason="stop") + resp = ModelResponse() + resp.choices = [choice] + result = LiteLLMCompletionResponsesConfig._extract_tool_result_output_items(resp) + assert result == [] + + +def test_in_place_substitution_preserves_ordering(): + """ + function_call items matching code_interpreter_results should be replaced + in-place, preserving the original output ordering. + + Simulates: [message, function_call(exec1), function_call(regular), function_call(exec2)] + Expected: [message, code_interpreter_call(exec1), function_call(regular), code_interpreter_call(exec2)] + """ + code_results = [ + OutputCodeInterpreterCall( + type="code_interpreter_call", + id="srvtoolu_01AAA", + code="echo first", + container_id=None, + status="completed", + outputs=[OutputCodeInterpreterCallLog(type="logs", logs="first\n")], + ), + OutputCodeInterpreterCall( + type="code_interpreter_call", + id="srvtoolu_01CCC", + code="echo third", + container_id=None, + status="completed", + outputs=[OutputCodeInterpreterCallLog(type="logs", logs="third\n")], + ), + ] + resp = _make_model_response(code_interpreter_results=code_results) + + # Build a mock responses_output list with interleaved items + class MockItem: + def __init__(self, type, call_id=None): + self.type = type + self.call_id = call_id + + msg_item = MockItem(type="message") + fc_exec1 = MockItem(type="function_call", call_id="srvtoolu_01AAA") + fc_regular = MockItem(type="function_call", call_id="srvtoolu_01BBB") + fc_exec2 = MockItem(type="function_call", call_id="srvtoolu_01CCC") + + responses_output = [msg_item, fc_exec1, fc_regular, fc_exec2] + + # Apply the same logic as _transform_chat_completion_choices_to_responses_output + tool_result_items = ( + LiteLLMCompletionResponsesConfig._extract_tool_result_output_items(resp) + ) + if tool_result_items: + result_by_id = { + (item.get("id") if isinstance(item, dict) else item.id): item + for item in tool_result_items + } + replaced_ids = set(result_by_id.keys()) + responses_output = [ + ( + result_by_id[getattr(item, "call_id", None)] + if ( + getattr(item, "type", None) == "function_call" + and getattr(item, "call_id", None) in replaced_ids + ) + else item + ) + for item in responses_output + ] + + # Verify ordering: message, code_interpreter(AAA), function_call(BBB), code_interpreter(CCC) + assert len(responses_output) == 4 + assert responses_output[0].type == "message" + assert responses_output[1].type == "code_interpreter_call" + assert responses_output[1].id == "srvtoolu_01AAA" + assert responses_output[2].type == "function_call" + assert responses_output[2].call_id == "srvtoolu_01BBB" + assert responses_output[3].type == "code_interpreter_call" + assert responses_output[3].id == "srvtoolu_01CCC" From 3962fbc33ac12c8a6e232dea488779f5570cb5f2 Mon Sep 17 00:00:00 2001 From: Andrzej Pomirski Date: Tue, 17 Mar 2026 19:26:03 +0100 Subject: [PATCH 10/66] fix: non-dict tool result content falls back to outputs=None Replace str(content) fallback with empty string so non-dict content (e.g. list-shaped text_editor results) produces outputs=None instead of raw Python object representations in logs. --- litellm/llms/anthropic/chat/handler.py | 2 +- litellm/llms/anthropic/chat/transformation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 0b84830a8b0..387901588f0 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -713,7 +713,7 @@ class ModelResponseIterator: parts.append(f"STDERR: {content['stderr']}") logs = "".join(parts) else: - logs = str(content) + logs = "" tool_input = self._server_tool_inputs.get(call_id, {}) code = tool_input.get("command", "") if isinstance(tool_input, dict) else "" log_outputs = ( diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 99b02e7ab46..238d72eda44 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1777,7 +1777,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): parts.append(f"STDERR: {content['stderr']}") logs = "".join(parts) else: - logs = str(content) + logs = "" log_outputs = ( [OutputCodeInterpreterCallLog(type="logs", logs=logs)] if logs From 8f60117228821ccde41d4845382afee507dfb70c Mon Sep 17 00:00:00 2001 From: Andrzej Pomirski Date: Wed, 18 Mar 2026 00:42:03 +0100 Subject: [PATCH 11/66] fix: guard code_interpreter conversion to bash_code_execution results only Skip non-bash tool result types (e.g. text_editor_code_execution_tool_result) to avoid producing empty code_interpreter_call items in Responses API output. --- litellm/llms/anthropic/chat/handler.py | 2 ++ litellm/llms/anthropic/chat/transformation.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 387901588f0..91fd3034066 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -703,6 +703,8 @@ class ModelResponseIterator: """ results = [] for tr in self.tool_results: + if tr.get("type") != "bash_code_execution_tool_result": + continue call_id = tr.get("tool_use_id", "") content = tr.get("content", {}) if isinstance(content, dict): diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 238d72eda44..ca101df0e95 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1767,6 +1767,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): pass code_interpreter_results = [] for tr in tool_results: + if tr.get("type") != "bash_code_execution_tool_result": + continue call_id = tr.get("tool_use_id", "") content = tr.get("content", {}) if isinstance(content, dict): From d10007cef49ae278e97c05e40ce367481e983975 Mon Sep 17 00:00:00 2001 From: Andrzej Pomirski Date: Wed, 18 Mar 2026 11:15:15 +0100 Subject: [PATCH 12/66] test: add non-bash skip test and mock end-to-end streaming integration test - test_non_bash_tool_result_skipped: verifies text_editor results produce zero code_interpreter_call items - test_end_to_end_streaming_chunks_to_code_interpreter_output: exercises full path from Anthropic SSE chunks through ModelResponseIterator, stream_chunk_builder, and _extract_tool_result_output_items without a live server --- .../chat/test_anthropic_chat_handler.py | 68 +++++++++++ ...est_code_interpreter_results_extraction.py | 106 +++++++++++++++++- 2 files changed, 172 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index ab298f6809b..20427e8cc94 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -1513,3 +1513,71 @@ def test_empty_output_produces_null_outputs(): assert ( code_results[0].outputs is None ), f"Expected outputs=None for empty execution, got {code_results[0].outputs}" + + +def test_non_bash_tool_result_skipped(): + """ + Tool result types other than bash_code_execution_tool_result (e.g. + text_editor_code_execution_tool_result) should be skipped and NOT + produce code_interpreter_call items. + """ + chunks = [ + { + "type": "message_start", + "message": { + "id": "msg_01XYZ", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 100, "output_tokens": 1}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "server_tool_use", + "id": "srvtoolu_01AAA", + "name": "text_editor", + "input": {"command": "view", "path": "/tmp/test.py"}, + }, + }, + {"type": "content_block_stop", "index": 0}, + # text_editor result — should NOT become a code_interpreter_call + { + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "text_editor_code_execution_tool_result", + "tool_use_id": "srvtoolu_01AAA", + "content": [ + {"type": "text", "text": "file contents here"}, + ], + }, + }, + {"type": "content_block_stop", "index": 1}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 50}, + }, + ] + + iterator = ModelResponseIterator(None, sync_stream=True) + + code_results = None + for chunk in chunks: + parsed = iterator.chunk_parser(chunk) + psf = None + if parsed.choices and parsed.choices[0].delta: + psf = getattr(parsed.choices[0].delta, "provider_specific_fields", None) + if psf and "code_interpreter_results" in psf: + code_results = psf["code_interpreter_results"] + + # code_interpreter_results should be emitted but empty (no bash results) + assert ( + code_results is not None + ), "Expected code_interpreter_results key to be emitted" + assert ( + len(code_results) == 0 + ), f"Expected 0 code_interpreter_results for text_editor result, got {len(code_results)}" diff --git a/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py b/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py index c6ff1c7af8f..eea9be38faa 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py +++ b/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py @@ -1,10 +1,13 @@ """ -Tests for the Responses API _extract_tool_result_output_items path -and the non-streaming _hidden_params propagation of code_interpreter_results. +Tests for the Responses API _extract_tool_result_output_items path, +the non-streaming _hidden_params propagation of code_interpreter_results, +and mock end-to-end streaming integration. """ from unittest.mock import MagicMock +from litellm.llms.anthropic.chat.handler import ModelResponseIterator +from litellm.main import stream_chunk_builder from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) @@ -161,3 +164,102 @@ def test_in_place_substitution_preserves_ordering(): assert responses_output[2].call_id == "srvtoolu_01BBB" assert responses_output[3].type == "code_interpreter_call" assert responses_output[3].id == "srvtoolu_01CCC" + + +def test_end_to_end_streaming_chunks_to_code_interpreter_output(): + """ + Mock end-to-end test: Anthropic SSE chunks → ModelResponseIterator → + stream_chunk_builder → _extract_tool_result_output_items → final output + with code_interpreter_call items replacing function_call items. + + This exercises the full streaming data flow without a live server. + """ + # Realistic Anthropic streaming chunks for a single code execution + raw_chunks = [ + { + "type": "message_start", + "message": { + "id": "msg_01XYZ", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 100, "output_tokens": 1}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "server_tool_use", + "id": "srvtoolu_01AAA", + "name": "bash_code_execution", + "input": {}, + }, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": { + "type": "input_json_delta", + "partial_json": '{"command": "echo e2e_test"}', + }, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01AAA", + "content": { + "type": "bash_code_execution_result", + "stdout": "e2e_test\n", + "stderr": "", + "return_code": 0, + }, + }, + }, + {"type": "content_block_stop", "index": 1}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 50}, + }, + ] + + # Step 1: Parse chunks through ModelResponseIterator (Anthropic handler) + iterator = ModelResponseIterator(None, sync_stream=True) + parsed_chunks = [] + for chunk in raw_chunks: + parsed = iterator.chunk_parser(chunk) + d = parsed.model_dump() + # In production, CustomStreamWrapper sets the model on each chunk; + # stream_chunk_builder requires it. + d["model"] = "claude-sonnet-4-20250514" + parsed_chunks.append(d) + + # Step 2: Assemble via stream_chunk_builder (simulates end-of-stream) + assembled = stream_chunk_builder(chunks=parsed_chunks) + assert assembled is not None + + # Verify stream_chunk_builder picked up code_interpreter_results via last-value-wins + psf = assembled.choices[0].message.provider_specific_fields + assert psf is not None + assert "code_interpreter_results" in psf + code_results = psf["code_interpreter_results"] + assert len(code_results) == 1 + # After model_dump + stream_chunk_builder, results are plain dicts + assert code_results[0]["id"] == "srvtoolu_01AAA" + assert code_results[0]["code"] == "echo e2e_test" + + # Step 3: Extract via _extract_tool_result_output_items (Responses API layer) + tool_result_items = ( + LiteLLMCompletionResponsesConfig._extract_tool_result_output_items(assembled) + ) + assert len(tool_result_items) == 1 + item = tool_result_items[0] + # Items are dicts after the model_dump path + assert item["type"] == "code_interpreter_call" + assert item["id"] == "srvtoolu_01AAA" + assert item["code"] == "echo e2e_test" + assert item["outputs"][0]["logs"] == "e2e_test\n" From cf8d1ac521648fea10bc121cd51da166e96493a4 Mon Sep 17 00:00:00 2001 From: Andrzej Pomirski Date: Wed, 18 Mar 2026 12:05:25 +0100 Subject: [PATCH 13/66] fix: streaming container_id and consistent Pydantic types in output - Populate container_id on streaming code_interpreter_results by re-emitting at message_delta when container info arrives - Reconstruct Pydantic OutputCodeInterpreterCall objects from plain dicts in _extract_tool_result_output_items so responses_output has uniform types across streaming and non-streaming paths --- litellm/llms/anthropic/chat/handler.py | 14 +++++++++++++- .../transformation.py | 14 +++++++++----- .../test_code_interpreter_results_extraction.py | 17 ++++++++++------- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 91fd3034066..70ecf91725d 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -546,6 +546,7 @@ class ModelResponseIterator: self._server_tool_inputs: Dict[str, Any] = {} self.tool_results: List[Dict[str, Any]] = [] self._current_server_tool_id: Optional[str] = None + self._container_id: Optional[str] = None def check_empty_tool_call_args(self) -> bool: """ @@ -726,7 +727,7 @@ class ModelResponseIterator: type="code_interpreter_call", id=call_id, code=code, - container_id=None, + container_id=self._container_id, status="completed", outputs=log_outputs, ) @@ -928,6 +929,17 @@ class ModelResponseIterator: finish_reason, usage, container = self._handle_message_delta(chunk) if container: provider_specific_fields["container"] = container + # Store container_id and re-emit code_interpreter_results + # so stream_chunk_builder's last-value-wins picks up the + # version with container_id populated. + container_id = ( + container.get("id") if isinstance(container, dict) else None + ) + if container_id and self.tool_results: + self._container_id = container_id + provider_specific_fields["code_interpreter_results"] = ( + self._build_code_interpreter_results() + ) elif type_chunk == "message_start": """ Anthropic diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index b7f7e9adda2..cf18511bfa3 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1738,10 +1738,7 @@ class LiteLLMCompletionResponsesConfig: ) ) if tool_result_items: - result_by_id = { - (item.get("id") if isinstance(item, dict) else item.id): item - for item in tool_result_items - } + result_by_id = {item.id: item for item in tool_result_items} replaced_ids = set(result_by_id.keys()) responses_output = [ ( @@ -1778,7 +1775,14 @@ class LiteLLMCompletionResponsesConfig: continue results = psf.get("code_interpreter_results") if results and isinstance(results, list): - output_items.extend(results) + for item in results: + # In the streaming path, items are plain dicts after + # model_dump() in stream_chunk_builder. Reconstruct + # Pydantic objects so responses_output has a uniform type. + if isinstance(item, dict): + output_items.append(OutputCodeInterpreterCall(**item)) + else: + output_items.append(item) return output_items @staticmethod diff --git a/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py b/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py index eea9be38faa..60e45c9b8ce 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py +++ b/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py @@ -58,7 +58,8 @@ def test_extract_tool_result_output_items_from_pydantic_objects(): def test_extract_tool_result_output_items_from_dicts(): - """Streaming path: after model_dump(), code_interpreter_results are plain dicts.""" + """Streaming path: after model_dump(), code_interpreter_results are plain dicts. + _extract_tool_result_output_items reconstructs them as Pydantic objects.""" items = [ { "type": "code_interpreter_call", @@ -72,7 +73,8 @@ def test_extract_tool_result_output_items_from_dicts(): resp = _make_model_response(code_interpreter_results=items) result = LiteLLMCompletionResponsesConfig._extract_tool_result_output_items(resp) assert len(result) == 1 - assert result[0]["id"] == "srvtoolu_01AAA" + assert isinstance(result[0], OutputCodeInterpreterCall) + assert result[0].id == "srvtoolu_01AAA" def test_extract_tool_result_output_items_empty(): @@ -258,8 +260,9 @@ def test_end_to_end_streaming_chunks_to_code_interpreter_output(): ) assert len(tool_result_items) == 1 item = tool_result_items[0] - # Items are dicts after the model_dump path - assert item["type"] == "code_interpreter_call" - assert item["id"] == "srvtoolu_01AAA" - assert item["code"] == "echo e2e_test" - assert item["outputs"][0]["logs"] == "e2e_test\n" + # Items are reconstructed as Pydantic OutputCodeInterpreterCall objects + assert isinstance(item, OutputCodeInterpreterCall) + assert item.type == "code_interpreter_call" + assert item.id == "srvtoolu_01AAA" + assert item.code == "echo e2e_test" + assert item.outputs[0].logs == "e2e_test\n" From b13a7c679033ceea1b93ca82440f5b6bf15a8bd7 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 12:17:19 -0700 Subject: [PATCH 14/66] Fix guardrail_mode.replace crash when backend returns non-string value The backend type for guardrail_mode is Optional[Union[str, List[str], Dict]] but the UI typed it as just string, causing a crash when .replace() was called on null/object/array values. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../GuardrailViewer/GuardrailViewer.test.tsx | 29 ++++++++++++++++ .../GuardrailViewer/GuardrailViewer.tsx | 33 +++++++++++++++---- .../GuardrailViewer/__tests__/fixtures.ts | 2 +- 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx index 28991ebfa03..60778a1337a 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx @@ -151,6 +151,35 @@ describe("GuardrailViewer", () => { expect(screen.queryByText(/Raw Bedrock Guardrail Response/)).not.toBeInTheDocument(); }); + it("renders without crashing when guardrail_mode is null", () => { + const data = makeGuardrailInformation({ guardrail_mode: null }); + renderWithProviders(); + + expect(screen.getByText("Guardrails & Policy Compliance")).toBeInTheDocument(); + // Null mode should display as dash + expect(screen.getByText("—")).toBeInTheDocument(); + }); + + it("renders without crashing when guardrail_mode is an object", () => { + const data = makeGuardrailInformation({ + guardrail_mode: { default: "pre_call", tags: {} }, + }); + renderWithProviders(); + + expect(screen.getByText("Guardrails & Policy Compliance")).toBeInTheDocument(); + expect(screen.getByText("PRE-CALL")).toBeInTheDocument(); + }); + + it("renders without crashing when guardrail_mode is an array", () => { + const data = makeGuardrailInformation({ + guardrail_mode: ["pre_call", "post_call"], + }); + renderWithProviders(); + + expect(screen.getByText("Guardrails & Policy Compliance")).toBeInTheDocument(); + expect(screen.getByText("PRE-CALL")).toBeInTheDocument(); + }); + it("integration: renders with real Bedrock details without mocks", async () => { const user = userEvent.setup(); const data = makeGuardrailInformation({ diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx index 1ab4744b893..0c37b70c8b3 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx @@ -40,7 +40,7 @@ interface GuardrailInformation { duration: number; end_time: number; start_time: number; - guardrail_mode: string; + guardrail_mode: string | string[] | Record | null; guardrail_name: string; guardrail_status: string; guardrail_response: GuardrailEntity[] | BedrockGuardrailResponse | any; @@ -77,9 +77,25 @@ const PROVIDERS_WITH_CUSTOM_RENDERERS = new Set([ "litellm_content_filter", ]); -const formatMode = (mode: unknown): string => { - if (mode == null || mode === "") return "—"; - const s = typeof mode === "string" ? mode : String(mode); +/** + * Extracts a plain string from guardrail_mode, which may be a string, + * an array of strings, an object with a "default" key, or null. + */ +const resolveMode = (mode: GuardrailInformation["guardrail_mode"]): string | null => { + if (mode == null) return null; + if (typeof mode === "string") return mode; + if (Array.isArray(mode)) return mode[0] ?? null; + if (typeof mode === "object" && "default" in mode) { + const def = mode.default; + if (typeof def === "string") return def; + if (Array.isArray(def)) return def[0] ?? null; + } + return null; +}; + +const formatMode = (mode: GuardrailInformation["guardrail_mode"]): string => { + const s = resolveMode(mode); + if (s == null || s === "") return "—"; return s.replace(/_/g, "-").toUpperCase(); }; @@ -302,9 +318,12 @@ const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => { items.push({ type: "request", label: "Request received", offsetMs: 0 }); // Pre-call guardrails - const preCalls = sorted.filter((e) => e.guardrail_mode === "pre_call"); - const postCalls = sorted.filter((e) => e.guardrail_mode === "post_call" || e.guardrail_mode === "logging_only"); - const duringCalls = sorted.filter((e) => e.guardrail_mode === "during_call"); + const preCalls = sorted.filter((e) => resolveMode(e.guardrail_mode) === "pre_call"); + const postCalls = sorted.filter((e) => { + const m = resolveMode(e.guardrail_mode); + return m === "post_call" || m === "logging_only"; + }); + const duringCalls = sorted.filter((e) => resolveMode(e.guardrail_mode) === "during_call"); for (const e of preCalls) { const offsetMs = Math.round((e.end_time - baseTime) * 1000); diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/__tests__/fixtures.ts b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/__tests__/fixtures.ts index 3c78aacf850..fc487b04d7e 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/__tests__/fixtures.ts +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/__tests__/fixtures.ts @@ -23,7 +23,7 @@ export interface GuardrailInformation { duration: number; end_time: number; start_time: number; - guardrail_mode: string; + guardrail_mode: string | string[] | Record | null; guardrail_name: string; guardrail_status: string; guardrail_response: GuardrailEntity[] | BedrockGuardrailResponse; From 20c1d984a610935368bacb215773135ac109cd5c Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 12:18:42 -0700 Subject: [PATCH 15/66] [Test] UI: Add unit tests for 10 previously untested components Add Vitest + RTL tests covering DebugWarningBanner, HelpLink, ExportFormatSelector, ExportSummary, ExportTypeSelector, UsageExportHeader, MetricCard, ScoreChart, GuardrailConfig, and AgentHubTableColumns. 73 tests total. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../AIHub/AgentHubTableColumns.test.tsx | 132 ++++++++++++++++++ .../components/DebugWarningBanner.test.tsx | 38 +++++ .../ExportFormatSelector.test.tsx | 36 +++++ .../EntityUsageExport/ExportSummary.test.tsx | 35 +++++ .../ExportTypeSelector.test.tsx | 37 +++++ .../UsageExportHeader.test.tsx | 73 ++++++++++ .../GuardrailConfig.test.tsx | 84 +++++++++++ .../GuardrailsMonitor/MetricCard.test.tsx | 34 +++++ .../GuardrailsMonitor/ScoreChart.test.tsx | 47 +++++++ .../src/components/HelpLink.test.tsx | 128 +++++++++++++++++ 10 files changed, 644 insertions(+) create mode 100644 ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx create mode 100644 ui/litellm-dashboard/src/components/DebugWarningBanner.test.tsx create mode 100644 ui/litellm-dashboard/src/components/EntityUsageExport/ExportFormatSelector.test.tsx create mode 100644 ui/litellm-dashboard/src/components/EntityUsageExport/ExportSummary.test.tsx create mode 100644 ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.test.tsx create mode 100644 ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx create mode 100644 ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx create mode 100644 ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.test.tsx create mode 100644 ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.test.tsx create mode 100644 ui/litellm-dashboard/src/components/HelpLink.test.tsx diff --git a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx new file mode 100644 index 00000000000..c32df2f5466 --- /dev/null +++ b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx @@ -0,0 +1,132 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { vi } from "vitest"; +import { flexRender, getCoreRowModel, useReactTable } from "@tanstack/react-table"; +import { getAgentHubTableColumns, AgentHubData } from "./AgentHubTableColumns"; + +const mockAgent: AgentHubData = { + agent_id: "agent-1", + protocolVersion: "1.0", + name: "Test Agent", + description: "A test agent for unit testing", + url: "https://agent.example.com", + version: "2.0", + capabilities: { streaming: true, caching: false }, + defaultInputModes: ["text"], + defaultOutputModes: ["text", "image"], + skills: [ + { id: "s1", name: "Skill One", description: "First skill" }, + { id: "s2", name: "Skill Two", description: "Second skill" }, + { id: "s3", name: "Skill Three", description: "Third skill" }, + ], + is_public: true, +}; + +function TestTable({ data, publicPage = false }: { data: AgentHubData[]; publicPage?: boolean }) { + const showModal = vi.fn(); + const copyToClipboard = vi.fn(); + const columns = getAgentHubTableColumns(showModal, copyToClipboard, publicPage); + const table = useReactTable({ data, columns, getCoreRowModel: getCoreRowModel() }); + + return ( + + + {table.getHeaderGroups().map((hg) => ( + + {hg.headers.map((h) => ( + + ))} + + ))} + + + {table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + ))} + + ))} + +
{flexRender(h.column.columnDef.header, h.getContext())}
{flexRender(cell.column.columnDef.cell, cell.getContext())}
+ ); +} + +describe("AgentHubTableColumns", () => { + it("should render", () => { + render(); + expect(screen.getByText("Test Agent")).toBeInTheDocument(); + }); + + it("should display the agent description", () => { + render(); + // Description appears in both the description column and the mobile view within agent name column + expect(screen.getAllByText("A test agent for unit testing").length).toBeGreaterThanOrEqual(1); + }); + + it("should display the version with a 'v' prefix", () => { + render(); + expect(screen.getByText("v2.0")).toBeInTheDocument(); + }); + + it("should display the protocol version", () => { + render(); + expect(screen.getByText("1.0")).toBeInTheDocument(); + }); + + it("should show skill count with correct pluralization", () => { + render(); + expect(screen.getByText("3 skills")).toBeInTheDocument(); + }); + + it("should show first two skills and '+1' for overflow", () => { + render(); + expect(screen.getByText("Skill One")).toBeInTheDocument(); + expect(screen.getByText("Skill Two")).toBeInTheDocument(); + expect(screen.getByText("+1")).toBeInTheDocument(); + }); + + it("should show only true capabilities as badges", () => { + render(); + expect(screen.getByText("streaming")).toBeInTheDocument(); + expect(screen.queryByText("caching")).not.toBeInTheDocument(); + }); + + it("should display I/O modes", () => { + render(); + expect(screen.getByText("text")).toBeInTheDocument(); + expect(screen.getByText("text, image")).toBeInTheDocument(); + }); + + it("should display 'Yes' badge for public agents", () => { + render(); + expect(screen.getByText("Yes")).toBeInTheDocument(); + }); + + it("should display 'No' badge for non-public agents", () => { + const privateAgent = { ...mockAgent, is_public: false }; + render(); + expect(screen.getByText("No")).toBeInTheDocument(); + }); + + it("should display a Details button", () => { + render(); + expect(screen.getByRole("button", { name: /details|info/i })).toBeInTheDocument(); + }); + + it("should show '-' when agent has no capabilities", () => { + const noCapAgent = { ...mockAgent, capabilities: {} }; + render(); + // The dash is rendered in the capabilities column + expect(screen.getByText("-")).toBeInTheDocument(); + }); + + it("should show singular 'skill' for one skill", () => { + const oneSkillAgent = { + ...mockAgent, + skills: [{ id: "s1", name: "Only Skill", description: "One" }], + }; + render(); + expect(screen.getByText("1 skill")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/DebugWarningBanner.test.tsx b/ui/litellm-dashboard/src/components/DebugWarningBanner.test.tsx new file mode 100644 index 00000000000..4c99175162d --- /dev/null +++ b/ui/litellm-dashboard/src/components/DebugWarningBanner.test.tsx @@ -0,0 +1,38 @@ +import { renderWithProviders, screen } from "../../tests/test-utils"; +import { vi } from "vitest"; +import { DebugWarningBanner } from "./DebugWarningBanner"; + +const mockUseHealthReadiness = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/healthReadiness/useHealthReadiness", () => ({ + useHealthReadiness: () => mockUseHealthReadiness(), +})); + +describe("DebugWarningBanner", () => { + afterEach(() => { + vi.resetAllMocks(); + }); + + it("should render", () => { + mockUseHealthReadiness.mockReturnValue({ data: { is_detailed_debug: true } }); + renderWithProviders(); + expect(screen.getByText(/Performance Warning/i)).toBeInTheDocument(); + }); + + it("should render nothing when debug mode is disabled", () => { + mockUseHealthReadiness.mockReturnValue({ data: { is_detailed_debug: false } }); + const { container } = renderWithProviders(); + expect(container.firstChild).toBeNull(); + }); + + it("should render nothing when health data is undefined", () => { + mockUseHealthReadiness.mockReturnValue({ data: undefined }); + const { container } = renderWithProviders(); + expect(container.firstChild).toBeNull(); + }); + + it("should mention LITELLM_LOG=DEBUG in the description", () => { + mockUseHealthReadiness.mockReturnValue({ data: { is_detailed_debug: true } }); + renderWithProviders(); + expect(screen.getByText(/LITELLM_LOG=DEBUG/)).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportFormatSelector.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportFormatSelector.test.tsx new file mode 100644 index 00000000000..a88f9cb116c --- /dev/null +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportFormatSelector.test.tsx @@ -0,0 +1,36 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { vi } from "vitest"; +import ExportFormatSelector from "./ExportFormatSelector"; + +describe("ExportFormatSelector", () => { + it("should render", () => { + render(); + expect(screen.getByText("Format")).toBeInTheDocument(); + }); + + it("should display the current value as csv", () => { + render(); + expect(screen.getByText("CSV (Excel, Google Sheets)")).toBeInTheDocument(); + }); + + it("should display the current value as json", () => { + render(); + expect(screen.getByText("JSON (includes metadata)")).toBeInTheDocument(); + }); + + it("should call onChange when a different format is selected", async () => { + const onChange = vi.fn(); + const user = userEvent.setup(); + render(); + + // Open the Ant Design Select dropdown + await user.click(screen.getByText("CSV (Excel, Google Sheets)")); + // Select JSON option from the dropdown + const jsonOption = await screen.findByText("JSON (includes metadata)", { + selector: ".ant-select-item-option-content", + }); + await user.click(jsonOption); + expect(onChange).toHaveBeenCalledWith("json", expect.anything()); + }); +}); diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportSummary.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportSummary.test.tsx new file mode 100644 index 00000000000..ab426e291b1 --- /dev/null +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportSummary.test.tsx @@ -0,0 +1,35 @@ +import { render, screen } from "@testing-library/react"; +import ExportSummary from "./ExportSummary"; + +describe("ExportSummary", () => { + const dateRange = { + from: new Date("2025-01-01"), + to: new Date("2025-01-31"), + }; + + it("should render", () => { + render(); + expect(screen.getByText(/2025/)).toBeInTheDocument(); + }); + + it("should display formatted date range", () => { + render(); + const text = screen.getByText(/\d+.*-.*\d+/); + expect(text).toBeInTheDocument(); + }); + + it("should show singular 'filter' for one filter", () => { + render(); + expect(screen.getByText(/1 filter$/)).toBeInTheDocument(); + }); + + it("should show plural 'filters' for multiple filters", () => { + render(); + expect(screen.getByText(/2 filters/)).toBeInTheDocument(); + }); + + it("should not show filter text when no filters applied", () => { + render(); + expect(screen.queryByText(/filter/)).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.test.tsx new file mode 100644 index 00000000000..6ccf8822e02 --- /dev/null +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.test.tsx @@ -0,0 +1,37 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { vi } from "vitest"; +import ExportTypeSelector from "./ExportTypeSelector"; + +describe("ExportTypeSelector", () => { + it("should render", () => { + render(); + expect(screen.getByText("Export type")).toBeInTheDocument(); + }); + + it("should render all three radio options", () => { + render(); + expect(screen.getAllByRole("radio")).toHaveLength(3); + }); + + it("should interpolate entity type in labels", () => { + render(); + expect(screen.getByText(/Day-by-day breakdown by organization$/)).toBeInTheDocument(); + expect(screen.getByText(/organization and key/)).toBeInTheDocument(); + expect(screen.getByText(/organization and model/)).toBeInTheDocument(); + }); + + it("should call onChange when a different option is selected", async () => { + const onChange = vi.fn(); + const user = userEvent.setup(); + render(); + await user.click(screen.getByText(/by team and key/)); + expect(onChange).toHaveBeenCalledWith("daily_with_keys"); + }); + + it("should have the correct radio checked based on value prop", () => { + render(); + const radios = screen.getAllByRole("radio"); + expect(radios[2]).toBeChecked(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx new file mode 100644 index 00000000000..729d6fd3406 --- /dev/null +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx @@ -0,0 +1,73 @@ +import { renderWithProviders, screen } from "../../../tests/test-utils"; +import userEvent from "@testing-library/user-event"; +import { vi } from "vitest"; +import UsageExportHeader from "./UsageExportHeader"; +import type { EntitySpendData } from "./types"; + +vi.mock("./EntityUsageExportModal", () => ({ + default: ({ isOpen, onClose }: { isOpen: boolean; onClose: () => void }) => + isOpen ? ( +
+ +
+ ) : null, +})); + +const defaultProps = { + dateValue: { from: new Date("2025-01-01"), to: new Date("2025-01-31") }, + entityType: "team" as const, + spendData: { + results: [], + metadata: { + total_spend: 0, + total_api_requests: 0, + total_successful_requests: 0, + total_failed_requests: 0, + total_tokens: 0, + }, + } satisfies EntitySpendData, +}; + +describe("UsageExportHeader", () => { + it("should render", () => { + renderWithProviders(); + expect(screen.getByRole("button", { name: /export data/i })).toBeInTheDocument(); + }); + + it("should open the export modal when the export button is clicked", async () => { + const user = userEvent.setup(); + renderWithProviders(); + await user.click(screen.getByRole("button", { name: /export data/i })); + expect(screen.getByTestId("export-modal")).toBeInTheDocument(); + }); + + it("should close the export modal when onClose is called", async () => { + const user = userEvent.setup(); + renderWithProviders(); + await user.click(screen.getByRole("button", { name: /export data/i })); + await user.click(screen.getByRole("button", { name: /close/i })); + expect(screen.queryByTestId("export-modal")).not.toBeInTheDocument(); + }); + + it("should not show filter dropdown when showFilters is false", () => { + renderWithProviders(); + expect(screen.queryByText(/filter/i)).not.toBeInTheDocument(); + }); + + it("should show filter dropdown when showFilters is true and options provided", () => { + renderWithProviders( + , + ); + expect(screen.getByText("Team")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx new file mode 100644 index 00000000000..fd9ad61599b --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx @@ -0,0 +1,84 @@ +import { render, screen, act } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { vi } from "vitest"; +import { GuardrailConfig } from "./GuardrailConfig"; + +describe("GuardrailConfig", () => { + const defaultProps = { + guardrailName: "Content Safety", + guardrailType: "Content Safety", + provider: "bedrock", + }; + + it("should render", () => { + render(); + expect(screen.getByText("Parameters")).toBeInTheDocument(); + }); + + it("should display the guardrail name in the parameters description", () => { + render(); + expect(screen.getByText(/Configure Content Safety behavior/)).toBeInTheDocument(); + }); + + it("should show version history when 'View history' is clicked", async () => { + const user = userEvent.setup(); + render(); + await user.click(screen.getByRole("button", { name: /view history/i })); + expect(screen.getByText("Initial configuration")).toBeInTheDocument(); + expect(screen.getByText("Added custom categories list")).toBeInTheDocument(); + }); + + it("should toggle version history text between View/Hide", async () => { + const user = userEvent.setup(); + render(); + const button = screen.getByRole("button", { name: /view history/i }); + await user.click(button); + expect(screen.getByRole("button", { name: /hide history/i })).toBeInTheDocument(); + }); + + it("should show custom code textarea when custom code override is toggled on", async () => { + const user = userEvent.setup(); + render(); + const switches = screen.getAllByRole("switch"); + // The second switch is the custom code override toggle + const customCodeSwitch = switches[1]; + await user.click(customCodeSwitch); + expect(screen.getByPlaceholderText(/async def evaluate/)).toBeInTheDocument(); + }); + + it("should hide custom code textarea when custom code override is off", () => { + render(); + // There's an input for categories, but no textarea + expect(screen.queryByPlaceholderText(/async def evaluate/)).not.toBeInTheDocument(); + }); + + it("should show the re-run button in idle state", () => { + render(); + expect(screen.getByRole("button", { name: /re-run on failing logs/i })).toBeInTheDocument(); + }); + + it("should show loading state when re-run is clicked", async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }); + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); + render(); + await user.click(screen.getByRole("button", { name: /re-run on failing logs/i })); + expect(screen.getByText(/Running on 10 samples/)).toBeInTheDocument(); + vi.useRealTimers(); + }); + + it("should show success message after re-run completes", async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }); + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); + render(); + await user.click(screen.getByRole("button", { name: /re-run on failing logs/i })); + act(() => { vi.advanceTimersByTime(2500); }); + expect(screen.getByText(/7\/10 would now pass/)).toBeInTheDocument(); + vi.useRealTimers(); + }); + + it("should display the Revert and Save buttons", () => { + render(); + expect(screen.getByRole("button", { name: /revert/i })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /save as v4/i })).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.test.tsx new file mode 100644 index 00000000000..9352cf46556 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.test.tsx @@ -0,0 +1,34 @@ +import { render, screen } from "@testing-library/react"; +import { MetricCard } from "./MetricCard"; + +describe("MetricCard", () => { + it("should render", () => { + render(); + expect(screen.getByText("Total Requests")).toBeInTheDocument(); + }); + + it("should display the numeric value", () => { + render(); + expect(screen.getByText("1234")).toBeInTheDocument(); + }); + + it("should display a string value", () => { + render(); + expect(screen.getByText("95.2%")).toBeInTheDocument(); + }); + + it("should display subtitle when provided", () => { + render(); + expect(screen.getByText("Last 7 days")).toBeInTheDocument(); + }); + + it("should not display subtitle when not provided", () => { + render(); + expect(screen.queryByText(/days/)).not.toBeInTheDocument(); + }); + + it("should display icon when provided", () => { + render(!} />); + expect(screen.getByTestId("icon")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.test.tsx new file mode 100644 index 00000000000..b950dd9a302 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.test.tsx @@ -0,0 +1,47 @@ +import { render, screen } from "@testing-library/react"; +import { ScoreChart } from "./ScoreChart"; + +vi.mock("@tremor/react", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + BarChart: ({ data, categories }: { data: unknown[]; categories: string[] }) => ( +
+ {data.length} data points +
+ ), + }; +}); + +describe("ScoreChart", () => { + it("should render", () => { + render(); + expect(screen.getByText("Request Outcomes Over Time")).toBeInTheDocument(); + }); + + it("should show empty state when no data provided", () => { + render(); + expect(screen.getByText("No chart data for this period")).toBeInTheDocument(); + }); + + it("should show empty state when data is an empty array", () => { + render(); + expect(screen.getByText("No chart data for this period")).toBeInTheDocument(); + }); + + it("should render chart when data is provided", () => { + const data = [ + { date: "2025-01-01", passed: 100, blocked: 5 }, + { date: "2025-01-02", passed: 120, blocked: 3 }, + ]; + render(); + expect(screen.getByTestId("bar-chart")).toBeInTheDocument(); + expect(screen.getByText("2 data points")).toBeInTheDocument(); + }); + + it("should pass correct categories to the chart", () => { + const data = [{ date: "2025-01-01", passed: 100, blocked: 5 }]; + render(); + expect(screen.getByTestId("bar-chart")).toHaveAttribute("data-categories", "passed,blocked"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/HelpLink.test.tsx b/ui/litellm-dashboard/src/components/HelpLink.test.tsx new file mode 100644 index 00000000000..2e9b44a9e90 --- /dev/null +++ b/ui/litellm-dashboard/src/components/HelpLink.test.tsx @@ -0,0 +1,128 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { HelpLink, HelpIcon, DocsMenu } from "./HelpLink"; + +describe("HelpLink", () => { + it("should render", () => { + render(); + expect(screen.getByRole("link")).toBeInTheDocument(); + }); + + it("should display default 'Learn more' text when no children provided", () => { + render(); + expect(screen.getByText("Learn more")).toBeInTheDocument(); + }); + + it("should display custom children text", () => { + render(Custom docs link); + expect(screen.getByText("Custom docs link")).toBeInTheDocument(); + }); + + it("should open in a new tab with noopener noreferrer", () => { + render(); + const link = screen.getByRole("link"); + expect(link).toHaveAttribute("target", "_blank"); + expect(link).toHaveAttribute("rel", "noopener noreferrer"); + }); + + it("should have accessible screen reader text", () => { + render(); + expect(screen.getByText("(opens in a new tab)")).toBeInTheDocument(); + }); +}); + +describe("HelpIcon", () => { + it("should render", () => { + render(); + expect(screen.getByRole("button", { name: /help information/i })).toBeInTheDocument(); + }); + + it("should show tooltip content on mouse enter", async () => { + const user = userEvent.setup(); + render(); + await user.hover(screen.getByRole("button", { name: /help information/i })); + expect(screen.getByText("Helpful tooltip text")).toBeInTheDocument(); + }); + + it("should hide tooltip content on mouse leave", async () => { + const user = userEvent.setup(); + render(); + const button = screen.getByRole("button", { name: /help information/i }); + await user.hover(button); + await user.unhover(button); + expect(screen.queryByText("Helpful tooltip text")).not.toBeInTheDocument(); + }); + + it("should show learn more link when learnMoreHref is provided", async () => { + const user = userEvent.setup(); + render(); + await user.hover(screen.getByRole("button", { name: /help information/i })); + expect(screen.getByText("Learn more")).toBeInTheDocument(); + }); + + it("should use custom learnMoreText when provided", async () => { + const user = userEvent.setup(); + render( + , + ); + await user.hover(screen.getByRole("button", { name: /help information/i })); + expect(screen.getByText("Read docs")).toBeInTheDocument(); + }); +}); + +describe("DocsMenu", () => { + const items = [ + { label: "Custom pricing", href: "https://docs.example.com/pricing" }, + { label: "Spend tracking", href: "https://docs.example.com/spend" }, + ]; + + it("should render", () => { + render(); + expect(screen.getByRole("button", { name: /docs/i })).toBeInTheDocument(); + }); + + it("should show menu items when clicked", async () => { + const user = userEvent.setup(); + render(); + await user.click(screen.getByRole("button", { name: /docs/i })); + expect(screen.getByText("Custom pricing")).toBeInTheDocument(); + expect(screen.getByText("Spend tracking")).toBeInTheDocument(); + }); + + it("should hide menu items when clicked again", async () => { + const user = userEvent.setup(); + render(); + const button = screen.getByRole("button", { name: /docs/i }); + await user.click(button); + await user.click(button); + expect(screen.queryByText("Custom pricing")).not.toBeInTheDocument(); + }); + + it("should set aria-expanded correctly", async () => { + const user = userEvent.setup(); + render(); + const button = screen.getByRole("button", { name: /docs/i }); + expect(button).toHaveAttribute("aria-expanded", "false"); + await user.click(button); + expect(button).toHaveAttribute("aria-expanded", "true"); + }); + + it("should close menu when clicking outside", async () => { + const user = userEvent.setup(); + render( +
+ + +
, + ); + await user.click(screen.getByRole("button", { name: /docs/i })); + expect(screen.getByText("Custom pricing")).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: /outside/i })); + expect(screen.queryByText("Custom pricing")).not.toBeInTheDocument(); + }); + + it("should display custom children text", () => { + render(Help); + expect(screen.getByText("Help")).toBeInTheDocument(); + }); +}); From 1c8b5f77c96be048e83a2b35c7679fd8d546e53b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 12:29:57 -0700 Subject: [PATCH 16/66] address greptile review feedback (greploop iteration 1) Add modeMatches() helper so array guardrail_mode values (e.g. ["pre_call", "post_call"]) place the entry in all matching timeline buckets, not just the first. Updated test to verify both buckets. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../GuardrailViewer/GuardrailViewer.test.tsx | 6 ++- .../GuardrailViewer/GuardrailViewer.tsx | 37 ++++++++++++++----- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx index 60778a1337a..b5e04c72440 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx @@ -170,14 +170,18 @@ describe("GuardrailViewer", () => { expect(screen.getByText("PRE-CALL")).toBeInTheDocument(); }); - it("renders without crashing when guardrail_mode is an array", () => { + it("renders without crashing when guardrail_mode is an array and shows in both timeline buckets", () => { const data = makeGuardrailInformation({ guardrail_mode: ["pre_call", "post_call"], }); renderWithProviders(); expect(screen.getByText("Guardrails & Policy Compliance")).toBeInTheDocument(); + // Mode badge shows first element formatted expect(screen.getByText("PRE-CALL")).toBeInTheDocument(); + // Entry should appear in both pre-call and post-call timeline sections + expect(screen.getByText(/Pre-call guardrail:/)).toBeInTheDocument(); + expect(screen.getByText(/Post-call guardrail:/)).toBeInTheDocument(); }); it("integration: renders with real Bedrock details without mocks", async () => { diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx index 0c37b70c8b3..1056076fcd0 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx @@ -78,8 +78,8 @@ const PROVIDERS_WITH_CUSTOM_RENDERERS = new Set([ ]); /** - * Extracts a plain string from guardrail_mode, which may be a string, - * an array of strings, an object with a "default" key, or null. + * Extracts a plain string from guardrail_mode for display purposes. + * Returns the first mode when multiple are present. */ const resolveMode = (mode: GuardrailInformation["guardrail_mode"]): string | null => { if (mode == null) return null; @@ -93,6 +93,25 @@ const resolveMode = (mode: GuardrailInformation["guardrail_mode"]): string | nul return null; }; +/** + * Checks whether guardrail_mode includes the given target stage. + * Handles arrays (multi-stage guardrails) by checking all elements. + */ +const modeMatches = ( + mode: GuardrailInformation["guardrail_mode"], + target: string, +): boolean => { + if (mode == null) return false; + if (typeof mode === "string") return mode === target; + if (Array.isArray(mode)) return mode.includes(target); + if (typeof mode === "object" && "default" in mode) { + const def = mode.default; + if (typeof def === "string") return def === target; + if (Array.isArray(def)) return (def as string[]).includes(target); + } + return false; +}; + const formatMode = (mode: GuardrailInformation["guardrail_mode"]): string => { const s = resolveMode(mode); if (s == null || s === "") return "—"; @@ -317,13 +336,13 @@ const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => { // Request received items.push({ type: "request", label: "Request received", offsetMs: 0 }); - // Pre-call guardrails - const preCalls = sorted.filter((e) => resolveMode(e.guardrail_mode) === "pre_call"); - const postCalls = sorted.filter((e) => { - const m = resolveMode(e.guardrail_mode); - return m === "post_call" || m === "logging_only"; - }); - const duringCalls = sorted.filter((e) => resolveMode(e.guardrail_mode) === "during_call"); + // Pre-call guardrails — use modeMatches so array modes (e.g. ["pre_call", "post_call"]) + // place the entry in every matching bucket. + const preCalls = sorted.filter((e) => modeMatches(e.guardrail_mode, "pre_call")); + const postCalls = sorted.filter( + (e) => modeMatches(e.guardrail_mode, "post_call") || modeMatches(e.guardrail_mode, "logging_only"), + ); + const duringCalls = sorted.filter((e) => modeMatches(e.guardrail_mode, "during_call")); for (const e of preCalls) { const offsetMs = Math.round((e.end_time - baseTime) * 1000); From 40721ab18f15bf4e3df2c1c04d8aba2be155e9e0 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 12:31:25 -0700 Subject: [PATCH 17/66] address greptile review feedback (greploop iteration 1) - Move vi.useRealTimers() to afterEach for proper cleanup - Use label-based DOM queries instead of fragile positional indexes - Remove leftover debug console.log from AgentHubTableColumns.tsx Co-Authored-By: Claude Opus 4.6 (1M context) --- .../components/AIHub/AgentHubTableColumns.tsx | 1 - .../ExportTypeSelector.test.tsx | 4 ++-- .../GuardrailConfig.test.tsx | 20 +++++++++++++------ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.tsx b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.tsx index c6a8c0b9daa..09b1c147615 100644 --- a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.tsx @@ -194,7 +194,6 @@ export const getAgentHubTableColumns = ( return publicA - publicB; }, cell: ({ row }) => { - console.log(`CHECKPOINT 1: ${JSON.stringify(row.original)}`); const agent = row.original; return agent.is_public === true ? ( diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.test.tsx index 6ccf8822e02..d9469f095a3 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.test.tsx +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.test.tsx @@ -31,7 +31,7 @@ describe("ExportTypeSelector", () => { it("should have the correct radio checked based on value prop", () => { render(); - const radios = screen.getAllByRole("radio"); - expect(radios[2]).toBeChecked(); + const modelRadio = screen.getByRole("radio", { name: /by team and model/i }); + expect(modelRadio).toBeChecked(); }); }); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx index fd9ad61599b..38f65681982 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx @@ -10,6 +10,10 @@ describe("GuardrailConfig", () => { provider: "bedrock", }; + afterEach(() => { + vi.useRealTimers(); + }); + it("should render", () => { render(); expect(screen.getByText("Parameters")).toBeInTheDocument(); @@ -39,10 +43,16 @@ describe("GuardrailConfig", () => { it("should show custom code textarea when custom code override is toggled on", async () => { const user = userEvent.setup(); render(); - const switches = screen.getAllByRole("switch"); - // The second switch is the custom code override toggle - const customCodeSwitch = switches[1]; - await user.click(customCodeSwitch); + // Walk up from "Custom Code Override" heading to find the enclosing section, + // then locate the switch within it + const heading = screen.getByText("Custom Code Override"); + let container = heading.parentElement; + let customCodeSwitch: Element | null = null; + while (container && !customCodeSwitch) { + customCodeSwitch = container.querySelector('[role="switch"]'); + container = container.parentElement; + } + await user.click(customCodeSwitch!); expect(screen.getByPlaceholderText(/async def evaluate/)).toBeInTheDocument(); }); @@ -63,7 +73,6 @@ describe("GuardrailConfig", () => { render(); await user.click(screen.getByRole("button", { name: /re-run on failing logs/i })); expect(screen.getByText(/Running on 10 samples/)).toBeInTheDocument(); - vi.useRealTimers(); }); it("should show success message after re-run completes", async () => { @@ -73,7 +82,6 @@ describe("GuardrailConfig", () => { await user.click(screen.getByRole("button", { name: /re-run on failing logs/i })); act(() => { vi.advanceTimersByTime(2500); }); expect(screen.getByText(/7\/10 would now pass/)).toBeInTheDocument(); - vi.useRealTimers(); }); it("should display the Revert and Save buttons", () => { From 6902355f5ba7ae6de8af2366d8ec0ce372c20b9b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 12:34:03 -0700 Subject: [PATCH 18/66] address greptile review feedback (greploop iteration 2) Replace unsafe `as string[]` cast in modeMatches with runtime type check via `.some()`. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../components/view_logs/GuardrailViewer/GuardrailViewer.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx index 1056076fcd0..28247d4b2a0 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx @@ -107,7 +107,7 @@ const modeMatches = ( if (typeof mode === "object" && "default" in mode) { const def = mode.default; if (typeof def === "string") return def === target; - if (Array.isArray(def)) return (def as string[]).includes(target); + if (Array.isArray(def)) return def.some((x) => typeof x === "string" && x === target); } return false; }; From 7714d1be0b236871ffd24424d8513fd4630eaffb Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 12:40:56 -0700 Subject: [PATCH 19/66] address greptile review feedback (greploop iteration 3) Add typeof string guards to all array element returns in resolveMode to prevent non-string values from sneaking through via any-widening. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../view_logs/GuardrailViewer/GuardrailViewer.tsx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx index 28247d4b2a0..5608e3677fa 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx @@ -84,11 +84,17 @@ const PROVIDERS_WITH_CUSTOM_RENDERERS = new Set([ const resolveMode = (mode: GuardrailInformation["guardrail_mode"]): string | null => { if (mode == null) return null; if (typeof mode === "string") return mode; - if (Array.isArray(mode)) return mode[0] ?? null; + if (Array.isArray(mode)) { + const first = mode[0]; + return typeof first === "string" ? first : null; + } if (typeof mode === "object" && "default" in mode) { const def = mode.default; if (typeof def === "string") return def; - if (Array.isArray(def)) return def[0] ?? null; + if (Array.isArray(def)) { + const first = def[0]; + return typeof first === "string" ? first : null; + } } return null; }; From 56ed8379e297e20fb219817690d3b197c440ab69 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 12:42:19 -0700 Subject: [PATCH 20/66] address greptile review feedback (greploop iteration 2) - Add explicit vi import to ScoreChart.test.tsx - Use custom matcher for I/O modes to avoid cross-element text issues - Use version-agnostic regex for Save button assertion - Add comments noting placeholder data in GuardrailConfig tests Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/components/AIHub/AgentHubTableColumns.test.tsx | 10 ++++++++-- .../GuardrailsMonitor/GuardrailConfig.test.tsx | 5 ++++- .../components/GuardrailsMonitor/ScoreChart.test.tsx | 1 + 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx index c32df2f5466..703a24d8d6f 100644 --- a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx @@ -94,8 +94,14 @@ describe("AgentHubTableColumns", () => { it("should display I/O modes", () => { render(); - expect(screen.getByText("text")).toBeInTheDocument(); - expect(screen.getByText("text, image")).toBeInTheDocument(); + // "In:" and "Out:" are in children; getByText with exact:false + // matches against the element's full textContent across child nodes + expect(screen.getByText((_, el) => + el?.tagName === "P" && el.textContent === "In: text" + )).toBeInTheDocument(); + expect(screen.getByText((_, el) => + el?.tagName === "P" && el.textContent === "Out: text, image" + )).toBeInTheDocument(); }); it("should display 'Yes' badge for public agents", () => { diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx index 38f65681982..ac84c270d82 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx @@ -24,6 +24,8 @@ describe("GuardrailConfig", () => { expect(screen.getByText(/Configure Content Safety behavior/)).toBeInTheDocument(); }); + // Note: Version history entries are hardcoded placeholders in the component. + // These assertions will need updating when wired to real API data. it("should show version history when 'View history' is clicked", async () => { const user = userEvent.setup(); render(); @@ -87,6 +89,7 @@ describe("GuardrailConfig", () => { it("should display the Revert and Save buttons", () => { render(); expect(screen.getByRole("button", { name: /revert/i })).toBeInTheDocument(); - expect(screen.getByRole("button", { name: /save as v4/i })).toBeInTheDocument(); + // The component's hardcoded default version is "v3", so Save shows "v4" + expect(screen.getByRole("button", { name: /save as v\d+/i })).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.test.tsx index b950dd9a302..c32e6745784 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.test.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.test.tsx @@ -1,4 +1,5 @@ import { render, screen } from "@testing-library/react"; +import { vi } from "vitest"; import { ScoreChart } from "./ScoreChart"; vi.mock("@tremor/react", async (importOriginal) => { From bbc120095e2dcc9477f76d647d01490699861114 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 12:48:01 -0700 Subject: [PATCH 21/66] address greptile review feedback (greploop iteration 3) - Remove Ant Design CSS class selector coupling in ExportFormatSelector test - Lift mock fns out of TestTable component body to enable callback assertions Co-Authored-By: Claude Opus 4.6 (1M context) --- .../components/AIHub/AgentHubTableColumns.test.tsx | 14 +++++++++++--- .../ExportFormatSelector.test.tsx | 4 +--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx index 703a24d8d6f..083e67c297a 100644 --- a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx @@ -22,9 +22,17 @@ const mockAgent: AgentHubData = { is_public: true, }; -function TestTable({ data, publicPage = false }: { data: AgentHubData[]; publicPage?: boolean }) { - const showModal = vi.fn(); - const copyToClipboard = vi.fn(); +function TestTable({ + data, + publicPage = false, + showModal = vi.fn(), + copyToClipboard = vi.fn(), +}: { + data: AgentHubData[]; + publicPage?: boolean; + showModal?: ReturnType; + copyToClipboard?: ReturnType; +}) { const columns = getAgentHubTableColumns(showModal, copyToClipboard, publicPage); const table = useReactTable({ data, columns, getCoreRowModel: getCoreRowModel() }); diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportFormatSelector.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportFormatSelector.test.tsx index a88f9cb116c..d20d24992f6 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportFormatSelector.test.tsx +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportFormatSelector.test.tsx @@ -27,9 +27,7 @@ describe("ExportFormatSelector", () => { // Open the Ant Design Select dropdown await user.click(screen.getByText("CSV (Excel, Google Sheets)")); // Select JSON option from the dropdown - const jsonOption = await screen.findByText("JSON (includes metadata)", { - selector: ".ant-select-item-option-content", - }); + const jsonOption = await screen.findByText("JSON (includes metadata)"); await user.click(jsonOption); expect(onChange).toHaveBeenCalledWith("json", expect.anything()); }); From fcea5606827552b506fb1de478ec1c6d095e7152 Mon Sep 17 00:00:00 2001 From: Avik Kumar Date: Wed, 18 Mar 2026 15:58:13 -0400 Subject: [PATCH 22/66] fix(langsmith): populate usage_metadata in outputs for Cost column LangSmith reads the Cost column from outputs.usage_metadata.total_cost, but LangsmithLogger._prepare_log_data never wrote to that key. The response_cost was already computed in StandardLoggingPayload but was not forwarded to the outputs dict. Inject usage_metadata with input_tokens, output_tokens, total_tokens, and total_cost into the outputs dict so LangSmith can display cost. Fixes #24001 Made-with: Cursor --- litellm/integrations/langsmith.py | 14 ++++- .../integrations/test_langsmith_init.py | 54 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 03845af521d..ef2d30bb26d 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -153,11 +153,23 @@ class LangsmithLogger(CustomBatchLogger): if key in requester_metadata and key not in extra_metadata: extra_metadata[key] = requester_metadata[key] + outputs = payload["response"] + if isinstance(outputs, dict): + outputs = {**outputs} + else: + outputs = {"output": outputs} + outputs["usage_metadata"] = { + "input_tokens": payload.get("prompt_tokens", 0), + "output_tokens": payload.get("completion_tokens", 0), + "total_tokens": payload.get("total_tokens", 0), + "total_cost": payload.get("response_cost", 0), + } + data = { "name": run_name, "run_type": "llm", # this should always be llm, since litellm always logs llm calls. Langsmith allow us to log "chain" "inputs": payload, - "outputs": payload["response"], + "outputs": outputs, "session_name": project_name, "start_time": payload["startTime"], "end_time": payload["endTime"], diff --git a/tests/test_litellm/integrations/test_langsmith_init.py b/tests/test_litellm/integrations/test_langsmith_init.py index 9f7db4095bc..779e7b4c94b 100644 --- a/tests/test_litellm/integrations/test_langsmith_init.py +++ b/tests/test_litellm/integrations/test_langsmith_init.py @@ -132,3 +132,57 @@ class TestLangsmithLoggerInit: assert ( logger.sampling_rate >= 0.0 ), f"sampling_rate should be non-negative, got {logger.sampling_rate}" + + +class TestLangsmithPrepareLogData: + """Regression test for #24001: _prepare_log_data must inject + usage_metadata into outputs so LangSmith's Cost column is populated.""" + + @patch("asyncio.create_task") + @patch.dict(os.environ, {"LANGSMITH_SAMPLING_RATE": "1"}, clear=False) + def test_outputs_contain_usage_metadata(self, mock_create_task): + logger = LangsmithLogger( + langsmith_api_key="test-key", + langsmith_project="test-project", + ) + + payload = { + "id": "test-id", + "response": {"choices": [{"message": {"content": "hi"}}]}, + "metadata": {}, + "startTime": 1.0, + "endTime": 2.0, + "request_tags": [], + "error_str": None, + "status": "success", + "response_cost": 0.0042, + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + } + + kwargs = { + "litellm_params": {"metadata": {}}, + "standard_logging_object": payload, + } + + credentials = { + "LANGSMITH_API_KEY": "test-key", + "LANGSMITH_PROJECT": "test-project", + "LANGSMITH_BASE_URL": "https://api.smith.langchain.com", + } + + data = logger._prepare_log_data( + kwargs=kwargs, + response_obj=None, + start_time=1.0, + end_time=2.0, + credentials=credentials, + ) + + assert "usage_metadata" in data["outputs"] + um = data["outputs"]["usage_metadata"] + assert um["total_cost"] == 0.0042 + assert um["input_tokens"] == 100 + assert um["output_tokens"] == 50 + assert um["total_tokens"] == 150 From eb7efa36daf03f23d81d137122c0b8e3f1575067 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 13:05:53 -0700 Subject: [PATCH 23/66] Add missing permission options to PERMISSION_OPTIONS list Adds /key/info, /key/list, /key/aliases, and /team/daily/activity to the hardcoded PERMISSION_OPTIONS in TeamSSOSettings.tsx. Co-Authored-By: Claude Opus 4.6 (1M context) --- ui/litellm-dashboard/src/components/TeamSSOSettings.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ui/litellm-dashboard/src/components/TeamSSOSettings.tsx b/ui/litellm-dashboard/src/components/TeamSSOSettings.tsx index a9c07cdbcf8..98745d388a2 100644 --- a/ui/litellm-dashboard/src/components/TeamSSOSettings.tsx +++ b/ui/litellm-dashboard/src/components/TeamSSOSettings.tsx @@ -26,6 +26,10 @@ const PERMISSION_OPTIONS = [ "/key/unblock", "/key/bulk_update", "/key/{key_id}/reset_spend", + "/key/info", + "/key/list", + "/key/aliases", + "/team/daily/activity", ]; interface SettingRowProps { From 51f78b7d7250a86f127d02fbe964968b2a8a41e2 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 13:09:17 -0700 Subject: [PATCH 24/66] address greptile review feedback (greploop iteration 4) - Add guard assertion before non-null click on custom code switch - Use await act(async ...) for timer advancement to avoid act warnings - Pin locale in date range assertion for CI determinism Co-Authored-By: Claude Opus 4.6 (1M context) --- .../components/EntityUsageExport/ExportSummary.test.tsx | 6 ++++-- .../components/GuardrailsMonitor/GuardrailConfig.test.tsx | 7 +++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportSummary.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportSummary.test.tsx index ab426e291b1..1aeee42f74e 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportSummary.test.tsx +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportSummary.test.tsx @@ -14,8 +14,10 @@ describe("ExportSummary", () => { it("should display formatted date range", () => { render(); - const text = screen.getByText(/\d+.*-.*\d+/); - expect(text).toBeInTheDocument(); + // Pin locale to en-US so test is deterministic regardless of CI runner locale + const expectedFrom = dateRange.from!.toLocaleDateString("en-US"); + const expectedTo = dateRange.to!.toLocaleDateString("en-US"); + expect(screen.getByText(`${expectedFrom} - ${expectedTo}`)).toBeInTheDocument(); }); it("should show singular 'filter' for one filter", () => { diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx index ac84c270d82..54c7ebabe77 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx @@ -54,7 +54,10 @@ describe("GuardrailConfig", () => { customCodeSwitch = container.querySelector('[role="switch"]'); container = container.parentElement; } - await user.click(customCodeSwitch!); + if (!customCodeSwitch) { + throw new Error("Could not find the Custom Code Override switch via DOM traversal"); + } + await user.click(customCodeSwitch); expect(screen.getByPlaceholderText(/async def evaluate/)).toBeInTheDocument(); }); @@ -82,7 +85,7 @@ describe("GuardrailConfig", () => { const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); render(); await user.click(screen.getByRole("button", { name: /re-run on failing logs/i })); - act(() => { vi.advanceTimersByTime(2500); }); + await act(async () => { vi.advanceTimersByTime(2500); }); expect(screen.getByText(/7\/10 would now pass/)).toBeInTheDocument(); }); From 98311e0f0a57094eb8cba5b0ec82e5af671b890d Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 18 Mar 2026 15:07:50 -0500 Subject: [PATCH 25/66] Preserve router model_group in generic API logs --- litellm/router.py | 8 ++++ .../test_router_helper_utils.py | 37 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index f34368172ac..d2acd4c1f56 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3874,10 +3874,18 @@ class Router: The response from the handler function """ handler_name = original_function.__name__ + metadata_variable_name = _get_router_metadata_variable_name( + function_name="generic_api_call" + ) try: verbose_router_logger.debug( f"Inside _generic_api_call() - handler: {handler_name}, model: {model}; kwargs: {kwargs}" ) + self._update_kwargs_before_fallbacks( + model=model, + kwargs=kwargs, + metadata_variable_name=metadata_variable_name, + ) deployment = self.get_available_deployment( model=model, messages=kwargs.get("messages", None), diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index 3001aec8b86..75c5250b3dd 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -1568,6 +1568,43 @@ def test_handle_clientside_credential_with_deployment_model_name(model_list): print("✓ _handle_clientside_credential test passed!") +def test_sync_generic_api_call_preserves_requested_model_group_in_logs(): + router = Router( + model_list=[ + { + "model_name": "claude-sonnet-4-6", + "litellm_params": { + "model": "bedrock/global.anthropic.claude-sonnet-4-6", + "aws_access_key_id": "test-access-key", + "aws_secret_access_key": "test-secret-key", + "aws_region_name": "us-west-2", + }, + } + ] + ) + + captured_kwargs = {} + + def mock_original_function(**kwargs): + captured_kwargs.update(kwargs) + return {"status": "ok"} + + response = router._generic_api_call_with_fallbacks( + model="claude-sonnet-4-6", + original_function=mock_original_function, + ) + + assert response == {"status": "ok"} + assert captured_kwargs["model"] == "bedrock/global.anthropic.claude-sonnet-4-6" + assert ( + captured_kwargs["litellm_metadata"]["model_group"] == "claude-sonnet-4-6" + ) + assert ( + captured_kwargs["litellm_metadata"]["deployment"] + == "bedrock/global.anthropic.claude-sonnet-4-6" + ) + + @pytest.mark.parametrize( "function_name, expected_metadata_key", [ From 845ad042913dd3d85d7f5d484fc0acc9ea990f14 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 18 Mar 2026 15:25:44 -0500 Subject: [PATCH 26/66] Address router generic API review feedback --- litellm/router.py | 1 + .../test_router_helper_utils.py | 80 +++++++++++++++---- 2 files changed, 65 insertions(+), 16 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index d2acd4c1f56..5263e966ab0 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3890,6 +3890,7 @@ class Router: model=model, messages=kwargs.get("messages", None), specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, ) self._update_kwargs_with_deployment( deployment=deployment, kwargs=kwargs, function_name="generic_api_call" diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index 75c5250b3dd..34a19f5ce79 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -1583,26 +1583,74 @@ def test_sync_generic_api_call_preserves_requested_model_group_in_logs(): ] ) - captured_kwargs = {} + try: + captured_kwargs = {} - def mock_original_function(**kwargs): - captured_kwargs.update(kwargs) - return {"status": "ok"} + def mock_original_function(**kwargs): + captured_kwargs.update(kwargs) + return {"status": "ok"} - response = router._generic_api_call_with_fallbacks( - model="claude-sonnet-4-6", - original_function=mock_original_function, + response = router._generic_api_call_with_fallbacks( + model="claude-sonnet-4-6", + original_function=mock_original_function, + ) + + assert response == {"status": "ok"} + assert ( + captured_kwargs["model"] == "bedrock/global.anthropic.claude-sonnet-4-6" + ) + assert ( + captured_kwargs["litellm_metadata"]["model_group"] == "claude-sonnet-4-6" + ) + assert ( + captured_kwargs["litellm_metadata"]["deployment"] + == "bedrock/global.anthropic.claude-sonnet-4-6" + ) + finally: + router.discard() + + +def test_sync_generic_api_call_uses_request_kwargs_for_deployment_selection(): + router = Router( + model_list=[ + { + "model_name": "regional-model", + "litellm_params": { + "model": "anthropic/us-model", + "api_key": "test-api-key", + "region_name": "us", + }, + }, + { + "model_name": "regional-model", + "litellm_params": { + "model": "anthropic/eu-model", + "api_key": "test-api-key", + "region_name": "eu", + }, + }, + ], + enable_pre_call_checks=True, ) - assert response == {"status": "ok"} - assert captured_kwargs["model"] == "bedrock/global.anthropic.claude-sonnet-4-6" - assert ( - captured_kwargs["litellm_metadata"]["model_group"] == "claude-sonnet-4-6" - ) - assert ( - captured_kwargs["litellm_metadata"]["deployment"] - == "bedrock/global.anthropic.claude-sonnet-4-6" - ) + try: + captured_kwargs = {} + + def mock_original_function(**kwargs): + captured_kwargs.update(kwargs) + return {"status": "ok"} + + response = router._generic_api_call_with_fallbacks( + model="regional-model", + original_function=mock_original_function, + messages=[{"role": "user", "content": "Hello from Europe"}], + allowed_model_region="eu", + ) + + assert response == {"status": "ok"} + assert captured_kwargs["model"] == "anthropic/eu-model" + finally: + router.discard() @pytest.mark.parametrize( From b00096f2a08a5d56d2a1d84c663b833dcdda5f66 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 18 Mar 2026 20:33:09 +0000 Subject: [PATCH 27/66] chore: regenerate poetry.lock to match pyproject.toml (#2) Co-authored-by: github-actions[bot] --- poetry.lock | 73 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 32 deletions(-) diff --git a/poetry.lock b/poetry.lock index 591d0c270e4..be87e51f801 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. [[package]] name = "a2a-sdk" @@ -7,11 +7,11 @@ description = "A2A Python SDK" optional = false python-versions = ">=3.10" groups = ["main", "proxy-dev"] -markers = "python_version >= \"3.10\"" files = [ {file = "a2a_sdk-0.3.22-py3-none-any.whl", hash = "sha256:b98701135bb90b0ff85d35f31533b6b7a299bf810658c1c65f3814a6c15ea385"}, {file = "a2a_sdk-0.3.22.tar.gz", hash = "sha256:77a5694bfc4f26679c11b70c7f1062522206d430b34bc1215cfbb1eba67b7e7d"}, ] +markers = {main = "python_version >= \"3.10\" and extra == \"extra-proxy\"", proxy-dev = "python_version >= \"3.10\""} [package.dependencies] google-api-core = ">=1.26.0" @@ -385,6 +385,7 @@ files = [ {file = "azure_core-1.36.0-py3-none-any.whl", hash = "sha256:fee9923a3a753e94a259563429f3644aaf05c486d45b1215d098115102d91d3b"}, {file = "azure_core-1.36.0.tar.gz", hash = "sha256:22e5605e6d0bf1d229726af56d9e92bc37b6e726b141a18be0b4d424131741b7"}, ] +markers = {main = "extra == \"proxy\" or extra == \"extra-proxy\""} [package.dependencies] requests = ">=2.21.0" @@ -405,6 +406,7 @@ files = [ {file = "azure_identity-1.25.1-py3-none-any.whl", hash = "sha256:e9edd720af03dff020223cd269fa3a61e8f345ea75443858273bcb44844ab651"}, {file = "azure_identity-1.25.1.tar.gz", hash = "sha256:87ca8328883de6036443e1c37b40e8dc8fb74898240f61071e09d2e369361456"}, ] +markers = {main = "extra == \"proxy\" or extra == \"extra-proxy\""} [package.dependencies] azure-core = ">=1.31.0" @@ -598,7 +600,7 @@ files = [ {file = "cachetools-6.2.2-py3-none-any.whl", hash = "sha256:6c09c98183bf58560c97b2abfcedcbaf6a896a490f534b031b661d3723b45ace"}, {file = "cachetools-6.2.2.tar.gz", hash = "sha256:8e6d266b25e539df852251cfd6f990b4bc3a141db73b939058d809ebd2590fc6"}, ] -markers = {main = "extra == \"google\" or extra == \"extra-proxy\" or python_version >= \"3.10\"", proxy-dev = "python_version >= \"3.10\""} +markers = {main = "python_version >= \"3.10\" and (extra == \"extra-proxy\" or extra == \"google\" or extra == \"mlflow\") or extra == \"google\" or extra == \"extra-proxy\"", proxy-dev = "python_version >= \"3.10\""} [[package]] name = "certifi" @@ -705,7 +707,7 @@ files = [ {file = "cffi-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9"}, {file = "cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529"}, ] -markers = {main = "platform_python_implementation != \"PyPy\" or extra == \"proxy\"", dev = "platform_python_implementation != \"PyPy\"", proxy-dev = "platform_python_implementation != \"PyPy\""} +markers = {main = "(platform_python_implementation != \"PyPy\" or extra == \"proxy\") and (python_version >= \"3.10\" or extra == \"proxy\" or extra == \"extra-proxy\") and (extra == \"proxy\" or extra == \"extra-proxy\" or extra == \"mlflow\")", dev = "platform_python_implementation != \"PyPy\"", proxy-dev = "platform_python_implementation != \"PyPy\""} [package.dependencies] pycparser = {version = "*", markers = "implementation_name != \"PyPy\""} @@ -1055,6 +1057,7 @@ files = [ {file = "cryptography-43.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2ce6fae5bdad59577b44e4dfed356944fbf1d925269114c28be377692643b4ff"}, {file = "cryptography-43.0.3.tar.gz", hash = "sha256:315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805"}, ] +markers = {main = "python_version >= \"3.10\" and (extra == \"proxy\" or extra == \"extra-proxy\" or extra == \"mlflow\") or extra == \"proxy\" or extra == \"extra-proxy\""} [package.dependencies] cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} @@ -1837,11 +1840,11 @@ description = "Google API client core library" optional = false python-versions = ">=3.7" groups = ["main", "proxy-dev"] -markers = "python_version >= \"3.14\"" files = [ {file = "google_api_core-2.25.2-py3-none-any.whl", hash = "sha256:e9a8f62d363dc8424a8497f4c2a47d6bcda6c16514c935629c257ab5d10210e7"}, {file = "google_api_core-2.25.2.tar.gz", hash = "sha256:1c63aa6af0d0d5e37966f157a77f9396d820fba59f9e43e9415bc3dc5baff300"}, ] +markers = {main = "python_version >= \"3.14\" and (extra == \"extra-proxy\" or extra == \"google\")", proxy-dev = "python_version >= \"3.14\""} [package.dependencies] google-auth = ">=2.14.1,<3.0.0" @@ -1869,7 +1872,7 @@ files = [ {file = "google_api_core-2.28.1-py3-none-any.whl", hash = "sha256:4021b0f8ceb77a6fb4de6fde4502cecab45062e66ff4f2895169e0b35bc9466c"}, {file = "google_api_core-2.28.1.tar.gz", hash = "sha256:2b405df02d68e68ce0fbc138559e6036559e685159d148ae5861013dc201baf8"}, ] -markers = {main = "(python_version >= \"3.10\" or extra == \"google\" or extra == \"extra-proxy\") and python_version < \"3.14\"", proxy-dev = "python_version >= \"3.10\" and python_version < \"3.14\""} +markers = {main = "python_version < \"3.14\" and (extra == \"extra-proxy\" or extra == \"google\")", proxy-dev = "python_version >= \"3.10\" and python_version < \"3.14\""} [package.dependencies] google-auth = ">=2.14.1,<3.0.0" @@ -1906,7 +1909,7 @@ files = [ {file = "google_auth-2.43.0-py2.py3-none-any.whl", hash = "sha256:af628ba6fa493f75c7e9dbe9373d148ca9f4399b5ea29976519e0a3848eddd16"}, {file = "google_auth-2.43.0.tar.gz", hash = "sha256:88228eee5fc21b62a1b5fe773ca15e67778cb07dc8363adcb4a8827b52d81483"}, ] -markers = {main = "extra == \"google\" or extra == \"extra-proxy\" or python_version >= \"3.10\"", proxy-dev = "python_version >= \"3.10\""} +markers = {main = "python_version >= \"3.10\" and (extra == \"extra-proxy\" or extra == \"google\" or extra == \"mlflow\") or extra == \"google\" or extra == \"extra-proxy\"", proxy-dev = "python_version >= \"3.10\""} [package.dependencies] cachetools = ">=2.0.0,<7.0" @@ -2078,11 +2081,11 @@ files = [ ] [package.dependencies] -google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} -google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0dev" -grpc-google-iam-v1 = ">=0.12.4,<1.0.0dev" -proto-plus = ">=1.22.3,<2.0.0dev" -protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0dev" +google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0.dev0", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0.dev0" +grpc-google-iam-v1 = ">=0.12.4,<1.0.0.dev0" +proto-plus = ">=1.22.3,<2.0.0.dev0" +protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0" [[package]] name = "google-cloud-resource-manager" @@ -2264,7 +2267,7 @@ files = [ {file = "googleapis_common_protos-1.72.0-py3-none-any.whl", hash = "sha256:4299c5a82d5ae1a9702ada957347726b167f9f8d1fc352477702a1e851ff4038"}, {file = "googleapis_common_protos-1.72.0.tar.gz", hash = "sha256:e55a601c1b32b52d7a3e65f43563e2aa61bcd737998ee672ac9b951cd49319f5"}, ] -markers = {main = "extra == \"google\" or extra == \"extra-proxy\" or python_version >= \"3.10\""} +markers = {main = "python_version >= \"3.10\" and (extra == \"extra-proxy\" or extra == \"google\") or extra == \"google\" or extra == \"extra-proxy\""} [package.dependencies] grpcio = {version = ">=1.44.0,<2.0.0", optional = true, markers = "extra == \"grpc\""} @@ -2673,11 +2676,11 @@ description = "Consume Server-Sent Event (SSE) messages with HTTPX." optional = false python-versions = ">=3.9" groups = ["main", "proxy-dev"] -markers = "python_version >= \"3.10\"" files = [ {file = "httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc"}, {file = "httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d"}, ] +markers = {main = "python_version >= \"3.10\" and (extra == \"proxy\" or extra == \"extra-proxy\")", proxy-dev = "python_version >= \"3.10\""} [[package]] name = "huey" @@ -3042,7 +3045,7 @@ files = [ [package.dependencies] attrs = ">=22.2.0" -jsonschema-specifications = ">=2023.03.6" +jsonschema-specifications = ">=2023.3.6" referencing = ">=0.28.4" rpds-py = ">=0.7.1" @@ -3219,15 +3222,15 @@ files = [ [[package]] name = "litellm-proxy-extras" -version = "0.4.56" +version = "0.4.57" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." optional = true python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" groups = ["main"] markers = "extra == \"proxy\"" files = [ - {file = "litellm_proxy_extras-0.4.56-py3-none-any.whl", hash = "sha256:52dbe3b5358c790e77e12f1ec5ef8e7508b383c2aaf41299750b6fb400908ee7"}, - {file = "litellm_proxy_extras-0.4.56.tar.gz", hash = "sha256:63ad59baa0defccc5c929cfd933ee7e32a6614b0fc5fa0fc45a12d7608e33f08"}, + {file = "litellm_proxy_extras-0.4.57-py3-none-any.whl", hash = "sha256:04538223cd80318a72d70c6e10f701598e58c763368296a6503c674c92fbdb62"}, + {file = "litellm_proxy_extras-0.4.57.tar.gz", hash = "sha256:ef9b95dc42237614216833bd5d46ebf9dea1caa5ea14ea1a66d7f7842b224ec2"}, ] [[package]] @@ -3713,6 +3716,7 @@ files = [ {file = "msal-1.34.0-py3-none-any.whl", hash = "sha256:f669b1644e4950115da7a176441b0e13ec2975c29528d8b9e81316023676d6e1"}, {file = "msal-1.34.0.tar.gz", hash = "sha256:76ba83b716ea5a6d75b0279c0ac353a0e05b820ca1f6682c0eb7f45190c43c2f"}, ] +markers = {main = "extra == \"proxy\" or extra == \"extra-proxy\""} [package.dependencies] cryptography = ">=2.5,<49" @@ -3733,6 +3737,7 @@ files = [ {file = "msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca"}, {file = "msal_extensions-1.3.1.tar.gz", hash = "sha256:c5b0fd10f65ef62b5f1d62f4251d51cbcaf003fcedae8c91b040a488614be1a4"}, ] +markers = {main = "extra == \"proxy\" or extra == \"extra-proxy\""} [package.dependencies] msal = ">=1.29,<2" @@ -3983,6 +3988,7 @@ files = [ {file = "nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"}, {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"}, ] +markers = {main = "extra == \"extra-proxy\""} [[package]] name = "numpy" @@ -4105,7 +4111,7 @@ files = [ {file = "opentelemetry_api-1.39.1-py3-none-any.whl", hash = "sha256:2edd8463432a7f8443edce90972169b195e7d6a05500cd29e6d13898187c9950"}, {file = "opentelemetry_api-1.39.1.tar.gz", hash = "sha256:fbde8c80e1b937a2c61f20347e91c0c18a1940cecf012d62e65a7caf08967c9c"}, ] -markers = {main = "python_version >= \"3.10\""} +markers = {main = "python_version >= \"3.10\" and extra == \"mlflow\""} [package.dependencies] importlib-metadata = ">=6.0,<8.8.0" @@ -4220,7 +4226,7 @@ files = [ {file = "opentelemetry_sdk-1.39.1-py3-none-any.whl", hash = "sha256:4d5482c478513ecb0a5d938dcc61394e647066e0cc2676bee9f3af3f3f45f01c"}, {file = "opentelemetry_sdk-1.39.1.tar.gz", hash = "sha256:cf4d4563caf7bff906c9f7967e2be22d0d6b349b908be0d90fb21c8e9c995cc6"}, ] -markers = {main = "python_version >= \"3.10\""} +markers = {main = "python_version >= \"3.10\" and extra == \"mlflow\""} [package.dependencies] opentelemetry-api = "1.39.1" @@ -4238,7 +4244,7 @@ files = [ {file = "opentelemetry_semantic_conventions-0.60b1-py3-none-any.whl", hash = "sha256:9fa8c8b0c110da289809292b0591220d3a7b53c1526a23021e977d68597893fb"}, {file = "opentelemetry_semantic_conventions-0.60b1.tar.gz", hash = "sha256:87c228b5a0669b748c76d76df6c364c369c28f1c465e50f661e39737e84bc953"}, ] -markers = {main = "python_version >= \"3.10\""} +markers = {main = "python_version >= \"3.10\" and extra == \"mlflow\""} [package.dependencies] opentelemetry-api = "1.39.1" @@ -4722,6 +4728,7 @@ files = [ {file = "prisma-0.11.0-py3-none-any.whl", hash = "sha256:22bb869e59a2968b99f3483bb417717273ffbc569fd1e9ceed95e5614cbaf53a"}, {file = "prisma-0.11.0.tar.gz", hash = "sha256:3f2f2fd2361e1ec5ff655f2a04c7860c2f2a5bc4c91f78ca9c5c6349735bf693"}, ] +markers = {main = "extra == \"extra-proxy\""} [package.dependencies] click = ">=7.1.2" @@ -4895,7 +4902,7 @@ files = [ {file = "proto_plus-1.26.1-py3-none-any.whl", hash = "sha256:13285478c2dcf2abb829db158e1047e2f1e8d63a077d94263c2b88b043c75a66"}, {file = "proto_plus-1.26.1.tar.gz", hash = "sha256:21a515a4c4c0088a773899e23c7bbade3d18f9c66c73edd4c7ee3816bc96a012"}, ] -markers = {main = "extra == \"google\" or extra == \"extra-proxy\" or python_version >= \"3.10\"", proxy-dev = "python_version >= \"3.10\""} +markers = {main = "extra == \"google\" or extra == \"extra-proxy\"", proxy-dev = "python_version >= \"3.10\""} [package.dependencies] protobuf = ">=3.19.0,<7.0.0" @@ -4923,7 +4930,7 @@ files = [ {file = "protobuf-5.29.5-py3-none-any.whl", hash = "sha256:6cf42630262c59b2d8de33954443d94b746c952b01434fc58a417fdbd2e84bd5"}, {file = "protobuf-5.29.5.tar.gz", hash = "sha256:bc1463bafd4b0929216c35f437a8e28731a2b7fe3d98bb77a600efced5a15c84"}, ] -markers = {main = "extra == \"google\" or extra == \"extra-proxy\" or python_version >= \"3.10\""} +markers = {main = "python_version >= \"3.10\" and (extra == \"extra-proxy\" or extra == \"google\" or extra == \"mlflow\") or extra == \"google\" or extra == \"extra-proxy\""} [[package]] name = "psutil" @@ -5083,7 +5090,7 @@ files = [ {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"}, {file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"}, ] -markers = {main = "extra == \"google\" or extra == \"extra-proxy\" or python_version >= \"3.10\"", proxy-dev = "python_version >= \"3.10\""} +markers = {main = "python_version >= \"3.10\" and (extra == \"extra-proxy\" or extra == \"google\" or extra == \"mlflow\") or extra == \"google\" or extra == \"extra-proxy\"", proxy-dev = "python_version >= \"3.10\""} [[package]] name = "pyasn1-modules" @@ -5096,7 +5103,7 @@ files = [ {file = "pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a"}, {file = "pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6"}, ] -markers = {main = "extra == \"google\" or extra == \"extra-proxy\" or python_version >= \"3.10\"", proxy-dev = "python_version >= \"3.10\""} +markers = {main = "python_version >= \"3.10\" and (extra == \"extra-proxy\" or extra == \"google\" or extra == \"mlflow\") or extra == \"google\" or extra == \"extra-proxy\"", proxy-dev = "python_version >= \"3.10\""} [package.dependencies] pyasn1 = ">=0.6.1,<0.7.0" @@ -5124,7 +5131,7 @@ files = [ {file = "pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934"}, {file = "pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2"}, ] -markers = {main = "implementation_name != \"PyPy\" and (platform_python_implementation != \"PyPy\" or extra == \"proxy\")", dev = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\"", proxy-dev = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\""} +markers = {main = "implementation_name != \"PyPy\" and (platform_python_implementation != \"PyPy\" or extra == \"proxy\") and (python_version >= \"3.10\" or extra == \"proxy\" or extra == \"extra-proxy\") and (extra == \"proxy\" or extra == \"extra-proxy\" or extra == \"mlflow\")", dev = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\"", proxy-dev = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\""} [[package]] name = "pydantic" @@ -5347,6 +5354,7 @@ files = [ {file = "pyjwt-2.12.1-py3-none-any.whl", hash = "sha256:28ca37c070cad8ba8cd9790cd940535d40274d22f80ab87f3ac6a713e6e8454c"}, {file = "pyjwt-2.12.1.tar.gz", hash = "sha256:c74a7a2adf861c04d002db713dd85f84beb242228e671280bf709d765b03672b"}, ] +markers = {main = "(python_version <= \"3.13\" or extra == \"proxy\" or extra == \"extra-proxy\") and (extra == \"extra-proxy\" or extra == \"proxy\")"} [package.dependencies] cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} @@ -6290,7 +6298,7 @@ files = [ {file = "rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762"}, {file = "rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75"}, ] -markers = {main = "extra == \"google\" or extra == \"extra-proxy\" or python_version >= \"3.10\"", proxy-dev = "python_version >= \"3.10\""} +markers = {main = "python_version >= \"3.10\" and (extra == \"extra-proxy\" or extra == \"google\" or extra == \"mlflow\") or extra == \"google\" or extra == \"extra-proxy\"", proxy-dev = "python_version >= \"3.10\""} [package.dependencies] pyasn1 = ">=0.1.3" @@ -6336,10 +6344,10 @@ files = [ ] [package.dependencies] -botocore = ">=1.37.4,<2.0a.0" +botocore = ">=1.37.4,<2.0a0" [package.extras] -crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] +crt = ["botocore[crt] (>=1.37.4,<2.0a0)"] [[package]] name = "scikit-learn" @@ -6492,9 +6500,9 @@ tornado = ">=6.4.2,<7" urllib3 = ">=1.26,<3" [package.extras] -all = ["boto3 (>=1.34.98,<2)", "botocore (>=1.34.110,<2)", "cohere (>=5.9.4,<6.00)", "dagger-io (>=0.1.1) ; python_version >= \"3.11\"", "fastembed (>=0.3.0,<0.4) ; python_version < \"3.13\"", "google-cloud-aiplatform (>=1.45.0,<2)", "ipykernel (>=6.25.0,<7)", "llama-cpp-python (>=0.2.28,<0.2.86) ; python_version < \"3.13\"", "mistralai (>=0.0.12,<0.1.0)", "mypy (>=1.7.1,<2)", "ollama (>=0.1.7)", "pillow (>=10.2.0,<11.0.0) ; python_version < \"3.13\"", "pinecone[asyncio] (>=7.0.0,<8.0.0)", "psycopg[binary] (>=3.1.0,<4)", "pytest (>=8.2,<9.0)", "pytest-asyncio (>=0.24.0,<0.25)", "pytest-cov (>=4.1.0,<5)", "pytest-mock (>=3.12.0,<4)", "pytest-timeout", "pytest-xdist (>=3.5.0,<4)", "python-dotenv (>=1.0.0,<2)", "qdrant-client (>=1.11.1,<2)", "requests-mock (>=1.12.1,<2)", "ruff (>=0.11.2,<0.12)", "sentence-transformers (>=5.0.0) ; python_version < \"3.13\"", "tokenizers (>=0.19) ; python_version < \"3.13\"", "torch (>=2.6.0) ; python_version < \"3.13\"", "torchvision (>=0.17.0) ; python_version < \"3.13\"", "transformers (>=4.36.2) ; python_version < \"3.13\"", "types-pyyaml (>=6.0.12.12,<7)", "types-requests (>=2.31.0,<3)"] +all = ["boto3 (>=1.34.98,<2)", "botocore (>=1.34.110,<2)", "cohere (>=5.9.4,<6.0)", "dagger-io (>=0.1.1) ; python_version >= \"3.11\"", "fastembed (>=0.3.0,<0.4) ; python_version < \"3.13\"", "google-cloud-aiplatform (>=1.45.0,<2)", "ipykernel (>=6.25.0,<7)", "llama-cpp-python (>=0.2.28,<0.2.86) ; python_version < \"3.13\"", "mistralai (>=0.0.12,<0.1.0)", "mypy (>=1.7.1,<2)", "ollama (>=0.1.7)", "pillow (>=10.2.0,<11.0.0) ; python_version < \"3.13\"", "pinecone[asyncio] (>=7.0.0,<8.0.0)", "psycopg[binary] (>=3.1.0,<4)", "pytest (>=8.2,<9.0)", "pytest-asyncio (>=0.24.0,<0.25)", "pytest-cov (>=4.1.0,<5)", "pytest-mock (>=3.12.0,<4)", "pytest-timeout", "pytest-xdist (>=3.5.0,<4)", "python-dotenv (>=1.0.0,<2)", "qdrant-client (>=1.11.1,<2)", "requests-mock (>=1.12.1,<2)", "ruff (>=0.11.2,<0.12)", "sentence-transformers (>=5.0.0) ; python_version < \"3.13\"", "tokenizers (>=0.19) ; python_version < \"3.13\"", "torch (>=2.6.0) ; python_version < \"3.13\"", "torchvision (>=0.17.0) ; python_version < \"3.13\"", "transformers (>=4.36.2) ; python_version < \"3.13\"", "types-pyyaml (>=6.0.12.12,<7)", "types-requests (>=2.31.0,<3)"] bedrock = ["boto3 (>=1.34.98,<2)", "botocore (>=1.34.110,<2)"] -cohere = ["cohere (>=5.9.4,<6.00)"] +cohere = ["cohere (>=5.9.4,<6.0)"] dev = ["dagger-io (>=0.1.1) ; python_version >= \"3.11\"", "ipykernel (>=6.25.0,<7)", "mypy (>=1.7.1,<2)", "pytest (>=8.2,<9.0)", "pytest-asyncio (>=0.24.0,<0.25)", "pytest-cov (>=4.1.0,<5)", "pytest-mock (>=3.12.0,<4)", "pytest-timeout", "pytest-xdist (>=3.5.0,<4)", "python-dotenv (>=1.0.0,<2)", "requests-mock (>=1.12.1,<2)", "ruff (>=0.11.2,<0.12)", "types-pyyaml (>=6.0.12.12,<7)", "types-requests (>=2.31.0,<3)"] docs = ["pydoc-markdown (>=4.8.2) ; python_version < \"3.12\""] fastembed = ["fastembed (>=0.3.0,<0.4) ; python_version < \"3.13\""] @@ -7222,6 +7230,7 @@ files = [ {file = "tomlkit-0.13.3-py3-none-any.whl", hash = "sha256:c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0"}, {file = "tomlkit-0.13.3.tar.gz", hash = "sha256:430cf247ee57df2b94ee3fbe588e71d362a941ebb545dec29b53961d61add2a1"}, ] +markers = {main = "extra == \"extra-proxy\""} [[package]] name = "tornado" @@ -7994,4 +8003,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "1f3bbf967451633fb6290ba88980bdf4fbf83420024b14e862d1da717d903684" +content-hash = "22cdc8096e0c8296827734393f5ab6e66088f397b5295caa1d277466d1fde1e8" From 71b687e00a43486ae3171102772a3be02f72ba26 Mon Sep 17 00:00:00 2001 From: Alexey <5122340@mail.ru> Date: Wed, 18 Mar 2026 23:45:57 +0300 Subject: [PATCH 28/66] fix(proxy): sync normalized call_type into model_call_details for proxy-only errors --- litellm/proxy/utils.py | 13 ++-- tests/proxy_unit_tests/test_proxy_utils.py | 69 ++++++++++++++++++++++ 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 76662be1753..e9946ebf975 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1863,28 +1863,33 @@ class ProxyLogging: ) input: Union[list, str, dict] = "" + normalized_call_type: Optional[str] = None if "messages" in request_data and isinstance( request_data["messages"], list ): input = request_data["messages"] litellm_logging_obj.model_call_details["messages"] = input if litellm_logging_obj.call_type != CallTypes.pass_through.value: - litellm_logging_obj.call_type = CallTypes.acompletion.value + normalized_call_type = CallTypes.acompletion.value elif "prompt" in request_data and isinstance(request_data["prompt"], str): input = request_data["prompt"] litellm_logging_obj.model_call_details["prompt"] = input if litellm_logging_obj.call_type != CallTypes.pass_through.value: - litellm_logging_obj.call_type = CallTypes.atext_completion.value + normalized_call_type = CallTypes.atext_completion.value elif "input" in request_data and isinstance(request_data["input"], list): input = request_data["input"] litellm_logging_obj.model_call_details["input"] = input if litellm_logging_obj.call_type != CallTypes.pass_through.value: - litellm_logging_obj.call_type = CallTypes.aembedding.value + normalized_call_type = CallTypes.aembedding.value + if normalized_call_type is not None: + litellm_logging_obj.call_type = normalized_call_type + litellm_logging_obj.model_call_details["call_type"] = ( + normalized_call_type + ) # Pass-through endpoints are logged via the callback loop's # async_post_call_failure_hook — skip pre_call and failure handlers. if litellm_logging_obj.call_type == CallTypes.pass_through.value: return - litellm_logging_obj.pre_call( input=input, api_key="", diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index d77a7465c10..00d4cd24e4b 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -2278,6 +2278,75 @@ async def test_post_call_failure_hook_auth_error_llm_api_route(): mock_handle_logging.assert_called_once() +@pytest.mark.asyncio +@pytest.mark.parametrize( + "request_data, route, expected_call_type", + [ + ( + {"model": "bad-model", "messages": [{"role": "user", "content": "hello"}]}, + "/v1/chat/completions", + "acompletion", + ), + ( + {"model": "bad-model", "prompt": "hello"}, + "/v1/completions", + "atext_completion", + ), + ( + {"model": "bad-model", "input": ["hello"]}, + "/v1/embeddings", + "aembedding", + ), + ], +) +async def test_handle_logging_proxy_only_error_syncs_normalized_call_type( + request_data, route, expected_call_type +): + from fastapi import HTTPException + + from litellm.caching.caching import DualCache + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy.utils import ProxyLogging + + cache = DualCache() + proxy_logging = ProxyLogging(user_api_key_cache=cache) + captured_logging_obj = {} + original_function_setup = litellm.utils.function_setup + + def _capture_function_setup(*args, **kwargs): + logging_obj, data = original_function_setup(*args, **kwargs) + captured_logging_obj["logging_obj"] = logging_obj + return logging_obj, data + + with patch( + "litellm.proxy.utils.litellm.utils.function_setup", + side_effect=_capture_function_setup, + ), patch.object( + Logging, "async_failure_handler", new=AsyncMock(return_value=None) + ), patch.object( + Logging, "failure_handler", return_value=None + ), patch( + "litellm.proxy.utils.threading.Thread" + ) as mock_thread: + mock_thread.return_value.start = Mock() + + await proxy_logging._handle_logging_proxy_only_error( + request_data=request_data, + user_api_key_dict=UserAPIKeyAuth( + api_key="test_key", + user_id="test_user", + token="test_token", + request_route=route, + ), + route=route, + original_exception=HTTPException(status_code=400, detail="bad request"), + ) + + logging_obj = captured_logging_obj["logging_obj"] + assert logging_obj.call_type == expected_call_type + assert logging_obj.model_call_details["call_type"] == expected_call_type + + @pytest.mark.asyncio async def test_during_call_hook_parallel_execution(): """ From 3ba18d708472869f7d6f7bbdb37c1cc2b471b56a Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 16:38:58 -0700 Subject: [PATCH 29/66] [Refactor] UI - Playground: Extract ChatMessageBubble from ChatUI Extract the chat message bubble rendering (~165 lines) into a dedicated ChatMessageBubble component with 15 Vitest tests covering all display branches. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../chat_ui/ChatMessageBubble.test.tsx | 280 ++++++++++++++++++ .../playground/chat_ui/ChatMessageBubble.tsx | 214 +++++++++++++ .../components/playground/chat_ui/ChatUI.tsx | 171 +---------- 3 files changed, 503 insertions(+), 162 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.test.tsx create mode 100644 ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.test.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.test.tsx new file mode 100644 index 00000000000..368043abee6 --- /dev/null +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.test.tsx @@ -0,0 +1,280 @@ +import { render, screen } from "@testing-library/react"; +import { describe, it, expect, vi } from "vitest"; +import ChatMessageBubble from "./ChatMessageBubble"; +import { EndpointType } from "./mode_endpoint_mapping"; +import { MessageType } from "./types"; + +// Mock child components to isolate bubble rendering logic +vi.mock("react-markdown", () => ({ + default: ({ children }: { children: string }) =>
{children}
, +})); + +vi.mock("react-syntax-highlighter", () => ({ + Prism: ({ children }: { children: string }) => 
{children}
, +})); + +vi.mock("react-syntax-highlighter/dist/esm/styles/prism", () => ({ + coy: {}, +})); + +vi.mock("./ReasoningContent", () => ({ + default: ({ reasoningContent }: { reasoningContent: string }) => ( +
{reasoningContent}
+ ), +})); + +vi.mock("./MCPEventsDisplay", () => ({ + default: ({ events }: { events: unknown[] }) => ( +
{events.length} events
+ ), +})); + +vi.mock("./SearchResultsDisplay", () => ({ + SearchResultsDisplay: ({ searchResults }: { searchResults: unknown[] }) => ( +
{searchResults.length} results
+ ), +})); + +vi.mock("./ResponseMetrics", () => ({ + default: ({ timeToFirstToken }: { timeToFirstToken?: number }) => ( +
TTFT: {timeToFirstToken}
+ ), +})); + +vi.mock("./A2AMetrics", () => ({ + default: ({ a2aMetadata }: { a2aMetadata: unknown }) => ( +
A2A
+ ), +})); + +vi.mock("./CodeInterpreterOutput", () => ({ + default: ({ code }: { code: string }) =>
{code}
, +})); + +vi.mock("./AudioRenderer", () => ({ + default: ({ message }: { message: MessageType }) => ( +
{typeof message.content === "string" ? message.content : ""}
+ ), +})); + +vi.mock("./ResponsesImageRenderer", () => ({ + default: () =>
, +})); + +vi.mock("./ChatImageRenderer", () => ({ + default: () =>
, +})); + +const defaultProps = { + isLastMessage: false, + endpointType: EndpointType.CHAT, + mcpEvents: [], + codeInterpreterResult: null, + accessToken: "test-token", +}; + +describe("ChatMessageBubble", () => { + it("should render a user message with right-aligned text", () => { + render( + , + ); + + expect(screen.getByText("user")).toBeInTheDocument(); + expect(screen.getByText("Hello")).toBeInTheDocument(); + }); + + it("should render an assistant message with left-aligned text", () => { + render( + , + ); + + expect(screen.getByText("assistant")).toBeInTheDocument(); + expect(screen.getByText("Hi there")).toBeInTheDocument(); + }); + + it("should show model badge for assistant messages when model is provided", () => { + render( + , + ); + + expect(screen.getByText("gpt-4")).toBeInTheDocument(); + }); + + it("should not show model badge for user messages even when model is set", () => { + render( + , + ); + + expect(screen.queryByText("gpt-4")).not.toBeInTheDocument(); + }); + + it("should render markdown content via ReactMarkdown", () => { + render( + , + ); + + expect(screen.getByTestId("react-markdown")).toHaveTextContent("**bold text**"); + }); + + it("should render an image when isImage is true", () => { + render( + , + ); + + expect(screen.getByAltText("Generated image")).toHaveAttribute("src", "https://example.com/img.png"); + }); + + it("should render AudioRenderer when isAudio is true", () => { + render( + , + ); + + expect(screen.getByTestId("audio-renderer")).toBeInTheDocument(); + }); + + it("should show ReasoningContent when reasoningContent is present", () => { + render( + , + ); + + expect(screen.getByTestId("reasoning-content")).toHaveTextContent("thinking..."); + }); + + it("should show MCP events on the last assistant message for RESPONSES endpoint", () => { + const mcpEvents = [{ type: "tool_call", item_id: "1" }]; + + render( + , + ); + + expect(screen.getByTestId("mcp-events-display")).toHaveTextContent("1 events"); + }); + + it("should not show MCP events when isLastMessage is false", () => { + const mcpEvents = [{ type: "tool_call", item_id: "1" }]; + + render( + , + ); + + expect(screen.queryByTestId("mcp-events-display")).not.toBeInTheDocument(); + }); + + it("should show SearchResultsDisplay when searchResults are present", () => { + render( + , + ); + + expect(screen.getByTestId("search-results-display")).toBeInTheDocument(); + }); + + it("should show ResponseMetrics when usage data is present and no a2aMetadata", () => { + render( + , + ); + + expect(screen.getByTestId("response-metrics")).toBeInTheDocument(); + }); + + it("should show A2AMetrics when a2aMetadata is present instead of ResponseMetrics", () => { + render( + , + ); + + expect(screen.getByTestId("a2a-metrics")).toBeInTheDocument(); + expect(screen.queryByTestId("response-metrics")).not.toBeInTheDocument(); + }); + + it("should show CodeInterpreterOutput on the last assistant message for RESPONSES endpoint", () => { + render( + , + ); + + expect(screen.getByTestId("code-interpreter-output")).toHaveTextContent("print('hello')"); + }); + + it("should render generated image from chat completions via message.image", () => { + render( + , + ); + + const images = screen.getAllByAltText("Generated image"); + expect(images.some((img) => img.getAttribute("src") === "https://example.com/generated.png")).toBe(true); + }); +}); diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx new file mode 100644 index 00000000000..3e1913cb2b2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx @@ -0,0 +1,214 @@ +import { RobotOutlined, UserOutlined } from "@ant-design/icons"; +import React from "react"; +import ReactMarkdown from "react-markdown"; +import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; +import { coy } from "react-syntax-highlighter/dist/esm/styles/prism"; +import { CodeInterpreterResult } from "../llm_calls/code_interpreter_handler"; +import A2AMetrics from "./A2AMetrics"; +import AudioRenderer from "./AudioRenderer"; +import ChatImageRenderer from "./ChatImageRenderer"; +import CodeInterpreterOutput from "./CodeInterpreterOutput"; +import { EndpointType } from "./mode_endpoint_mapping"; +import MCPEventsDisplay from "./MCPEventsDisplay"; +import type { MCPEvent } from "../../mcp_tools/types"; +import ReasoningContent from "./ReasoningContent"; +import ResponseMetrics from "./ResponseMetrics"; +import ResponsesImageRenderer from "./ResponsesImageRenderer"; +import { SearchResultsDisplay } from "./SearchResultsDisplay"; +import { MessageType } from "./types"; + +interface ChatMessageBubbleProps { + message: MessageType; + /** Whether this is the last message in the chat history. */ + isLastMessage: boolean; + endpointType: string; + /** MCP events to display on the last assistant message. */ + mcpEvents: MCPEvent[]; + /** Code interpreter result to display on the last assistant message. */ + codeInterpreterResult: CodeInterpreterResult | null; + /** API key used to fetch code interpreter file downloads. */ + accessToken: string; +} + +function ChatMessageBubble({ + message, + isLastMessage, + endpointType, + mcpEvents, + codeInterpreterResult, + accessToken, +}: ChatMessageBubbleProps) { + const isUser = message.role === "user"; + + return ( +
+
+ {/* Header: role icon + name + model badge */} +
+
+ {isUser ? ( + + ) : ( + + )} +
+ {message.role} + {message.role === "assistant" && message.model && ( + + {message.model} + + )} +
+ + {/* Reasoning content (chain-of-thought) */} + {message.reasoningContent && } + + {/* MCP events at the start of the last assistant message */} + {message.role === "assistant" && + isLastMessage && + mcpEvents.length > 0 && + (endpointType === EndpointType.RESPONSES || endpointType === EndpointType.CHAT) && ( +
+ +
+ )} + + {/* Search results */} + {message.role === "assistant" && message.searchResults && ( + + )} + + {/* Code Interpreter output for the last assistant message */} + {message.role === "assistant" && + isLastMessage && + codeInterpreterResult && + endpointType === EndpointType.RESPONSES && ( + + )} + + {/* Message body */} +
+ {message.isImage ? ( + Generated image + ) : message.isAudio ? ( + + ) : ( + <> + {/* Attached image for user messages based on endpoint */} + {endpointType === EndpointType.RESPONSES && } + {endpointType === EndpointType.CHAT && } + + & { + inline?: boolean; + node?: unknown; + }) { + const match = /language-(\w+)/.exec(className || ""); + return !inline && match ? ( + } + language={match[1]} + PreTag="div" + className="rounded-md my-2" + wrapLines={true} + wrapLongLines={true} + {...props} + > + {String(children).replace(/\n$/, "")} + + ) : ( + + {children} + + ); + }, + pre: ({ node, ...props }) => ( + 
+                  ),
+                }}
+              >
+                {typeof message.content === "string" ? message.content : ""}
+              
+
+              {/* Generated image from chat completions */}
+              {message.image && (
+                

+                  Generated image
+                

+              )}
+            
+          )}
+
+          {/* Response metrics */}
+          {message.role === "assistant" &&
+            (message.timeToFirstToken || message.totalLatency || message.usage) &&
+            !message.a2aMetadata && (
+              
+            )}
+
+          {/* A2A Metrics */}
+          {message.role === "assistant" && message.a2aMetadata && (
+            
+          )}
+
+
+
+ ); +} + +export default ChatMessageBubble; diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx index bc0d52cf581..ec37a9e1e99 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx @@ -63,6 +63,7 @@ import EndpointSelector from "./EndpointSelector"; import FilePreviewCard from "./FilePreviewCard"; import MCPEventsDisplay from "./MCPEventsDisplay"; import type { MCPEvent } from "../../mcp_tools/types"; +import ChatMessageBubble from "./ChatMessageBubble"; import { EndpointType, getEndpointType } from "./mode_endpoint_mapping"; import ReasoningContent from "./ReasoningContent"; import ResponseMetrics, { TokenUsage } from "./ResponseMetrics"; @@ -1932,168 +1933,14 @@ const ChatUI: React.FC = ({ {chatHistory.map((message, index) => (
-
-
-
-
- {message.role === "user" ? ( - - ) : ( - - )} -
- {message.role} - {message.role === "assistant" && message.model && ( - - {message.model} - - )} -
- {message.reasoningContent && } - - {/* Show MCP events at the start of assistant messages */} - {message.role === "assistant" && - index === chatHistory.length - 1 && - mcpEvents.length > 0 && - (endpointType === EndpointType.RESPONSES || endpointType === EndpointType.CHAT) && ( -
- -
- )} - - {/* Show search results at the start of assistant messages */} - {message.role === "assistant" && message.searchResults && ( - - )} - - {/* Show Code Interpreter output for the last assistant message */} - {message.role === "assistant" && - index === chatHistory.length - 1 && - codeInterpreter.result && - endpointType === EndpointType.RESPONSES && ( - - )} - -
- {message.isImage ? ( - Generated image - ) : message.isAudio ? ( - - ) : ( - <> - {/* Show attached image for user messages based on current endpoint */} - {endpointType === EndpointType.RESPONSES && } - {endpointType === EndpointType.CHAT && } - - & { - inline?: boolean; - node?: any; - }) { - const match = /language-(\w+)/.exec(className || ""); - return !inline && match ? ( - - {String(children).replace(/\n$/, "")} - - ) : ( - - {children} - - ); - }, - pre: ({ node, ...props }) => ( - 
-                                ),
-                              }}
-                            >
-                              {typeof message.content === "string" ? message.content : ""}
-                            
-
-                            {/* Show generated image from chat completions */}
-                            {message.image && (
-                              

-                                Generated image
-                              

-                            )}
-                          
-                        )}
-
-                        {message.role === "assistant" &&
-                          (message.timeToFirstToken || message.totalLatency || message.usage) &&
-                          !message.a2aMetadata && (
-                            
-                          )}
-
-                        {/* A2A Metrics - show for A2A agent responses */}
-                        {message.role === "assistant" && message.a2aMetadata && (
-                          
-                        )}
-
-
-
+
))} From b55cb249fe276996e9ca351e31d08e7cff4a9542 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 16:46:41 -0700 Subject: [PATCH 30/66] Address Greptile feedback: use EndpointType enum, add CHAT MCP test - Narrow endpointType prop from string to EndpointType enum - Add missing test for MCP events on CHAT endpoint Co-Authored-By: Claude Opus 4.6 (1M context) --- .../chat_ui/ChatMessageBubble.test.tsx | 16 ++++++++++++++++ .../playground/chat_ui/ChatMessageBubble.tsx | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.test.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.test.tsx index 368043abee6..70c3fdd4f29 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.test.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.test.tsx @@ -180,6 +180,22 @@ describe("ChatMessageBubble", () => { expect(screen.getByTestId("mcp-events-display")).toHaveTextContent("1 events"); }); + it("should show MCP events on the last assistant message for CHAT endpoint", () => { + const mcpEvents = [{ type: "tool_call", item_id: "1" }]; + + render( + , + ); + + expect(screen.getByTestId("mcp-events-display")).toHaveTextContent("1 events"); + }); + it("should not show MCP events when isLastMessage is false", () => { const mcpEvents = [{ type: "tool_call", item_id: "1" }]; diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx index 3e1913cb2b2..148cd082e22 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx @@ -21,7 +21,7 @@ interface ChatMessageBubbleProps { message: MessageType; /** Whether this is the last message in the chat history. */ isLastMessage: boolean; - endpointType: string; + endpointType: EndpointType; /** MCP events to display on the last assistant message. */ mcpEvents: MCPEvent[]; /** Code interpreter result to display on the last assistant message. */ From f6cd0a827ae84cffa838eac859eaea504bd6464a Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 16:53:29 -0700 Subject: [PATCH 31/66] fix: /key/update returns 404 (not 401) for nonexistent body key The /key/update endpoint's get_data() call raises a 401 when the body `key` field doesn't exist in the DB, because get_data() treats the token as an auth credential. This caused the auth layer to resolve the body key instead of the Authorization header bearer token. Replace prisma_client.get_data() with direct Prisma find_unique() in both _get_and_validate_existing_key() and update_key_fn(), matching the pattern used in the /key/block and /key/unblock fix (PR #23977). Also fix the incorrect "Team not found" error message in update_key_fn. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../key_management_endpoints.py | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index ec33816b789..2884c8d374e 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -1662,16 +1662,23 @@ async def _get_and_validate_existing_key( detail={"error": "Database not connected"}, ) - existing_key_row = await prisma_client.get_data( - token=token, - table_name="key", - query_type="find_unique", + from litellm.proxy.proxy_server import hash_token + + if token.startswith("sk-"): + hashed_token = hash_token(token=token) + else: + hashed_token = token + + existing_key_row = await prisma_client.db.litellm_verificationtoken.find_unique( + where={"token": hashed_token} ) if existing_key_row is None: - raise HTTPException( - status_code=404, - detail={"error": f"Key not found: {token}"}, + raise ProxyException( + message=f"Key not found. Passed key={token}", + type=ProxyErrorTypes.not_found_error, + param="key", + code=status.HTTP_404_NOT_FOUND, ) return existing_key_row @@ -2112,14 +2119,23 @@ async def update_key_fn( if prisma_client is None: raise Exception("Not connected to DB!") - existing_key_row = await prisma_client.get_data( - token=data.key, table_name="key", query_type="find_unique" + from litellm.proxy.proxy_server import hash_token + + if data.key.startswith("sk-"): + hashed_token = hash_token(token=data.key) + else: + hashed_token = data.key + + existing_key_row = await prisma_client.db.litellm_verificationtoken.find_unique( + where={"token": hashed_token} ) if existing_key_row is None: - raise HTTPException( - status_code=404, - detail={"error": f"Team not found, passed team_id={data.team_id}"}, + raise ProxyException( + message=f"Key not found. Passed key={data.key}", + type=ProxyErrorTypes.not_found_error, + param="key", + code=status.HTTP_404_NOT_FOUND, ) await _validate_update_key_data( From ebe329cdce15edfd3f0c783ae2bad9053a66bd83 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 17:02:18 -0700 Subject: [PATCH 32/66] Fix build: use `as any` for SyntaxHighlighter style prop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Matches the cast used in ChatUI.tsx — the react-syntax-highlighter type definitions don't accept CSSProperties directly. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/components/playground/chat_ui/ChatMessageBubble.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx index 148cd082e22..15978c17f7e 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx @@ -143,7 +143,7 @@ function ChatMessageBubble({ const match = /language-(\w+)/.exec(className || ""); return !inline && match ? ( } + style={coy as any} language={match[1]} PreTag="div" className="rounded-md my-2" From eceb4981b851659e6ca2152660216577e5f06feb Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 17:03:43 -0700 Subject: [PATCH 33/66] fix: address review feedback - dedup logic, use module-level helper, add test - Deduplicate: update_key_fn now delegates to _get_and_validate_existing_key() instead of inlining its own copy of the lookup logic - Use _hash_token_if_needed (already imported at module level) instead of inline `from proxy_server import hash_token` + manual conditional - Fix stale docstring: _get_and_validate_existing_key raises ProxyException, not HTTPException - Add unit test: test_update_key_nonexistent_key_returns_404 Co-Authored-By: Claude Opus 4.6 (1M context) --- .../key_management_endpoints.py | 32 ++---------- .../test_key_management_endpoints.py | 50 +++++++++++++++++++ 2 files changed, 55 insertions(+), 27 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 2884c8d374e..7be1a5a5e0c 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -1654,7 +1654,7 @@ async def _get_and_validate_existing_key( LiteLLM_VerificationToken: The existing key row Raises: - HTTPException: If key is not found + ProxyException: 404 if key is not found """ if prisma_client is None: raise HTTPException( @@ -1662,12 +1662,7 @@ async def _get_and_validate_existing_key( detail={"error": "Database not connected"}, ) - from litellm.proxy.proxy_server import hash_token - - if token.startswith("sk-"): - hashed_token = hash_token(token=token) - else: - hashed_token = token + hashed_token = _hash_token_if_needed(token=token) existing_key_row = await prisma_client.db.litellm_verificationtoken.find_unique( where={"token": hashed_token} @@ -2116,28 +2111,11 @@ async def update_key_fn( key = data_json.pop("key") # get the row from db - if prisma_client is None: - raise Exception("Not connected to DB!") - - from litellm.proxy.proxy_server import hash_token - - if data.key.startswith("sk-"): - hashed_token = hash_token(token=data.key) - else: - hashed_token = data.key - - existing_key_row = await prisma_client.db.litellm_verificationtoken.find_unique( - where={"token": hashed_token} + existing_key_row = await _get_and_validate_existing_key( + token=data.key, + prisma_client=prisma_client, ) - if existing_key_row is None: - raise ProxyException( - message=f"Key not found. Passed key={data.key}", - type=ProxyErrorTypes.not_found_error, - param="key", - code=status.HTTP_404_NOT_FOUND, - ) - await _validate_update_key_data( data=data, existing_key_row=existing_key_row, diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 3bff35fbd55..48a1f7936c5 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -1678,6 +1678,56 @@ async def test_unblock_key_nonexistent_key_returns_404(monkeypatch): mock_prisma_client.db.litellm_verificationtoken.update.assert_not_called() +@pytest.mark.asyncio +async def test_update_key_nonexistent_key_returns_404(monkeypatch): + """ + Test that update_key_fn returns 404 (not misleading 401) when the body + key doesn't exist in the database, even when the caller is authenticated + as a proxy admin via the Authorization header. + """ + from litellm.proxy.management_endpoints.key_management_endpoints import ( + update_key_fn, + ) + + mock_prisma_client = AsyncMock() + mock_user_api_key_cache = MagicMock() + mock_proxy_logging_obj = MagicMock() + + # find_unique returns None → key does not exist + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=None + ) + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr( + "litellm.proxy.proxy_server.user_api_key_cache", mock_user_api_key_cache + ) + monkeypatch.setattr( + "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging_obj + ) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + + mock_request = MagicMock() + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-admin", user_id="admin_user" + ) + + data = UpdateKeyRequest(key="sk-does-not-exist-key") + + with pytest.raises(ProxyException) as exc_info: + await update_key_fn( + request=mock_request, + data=data, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + + assert exc_info.value.code == "404" + assert "not found" in str(exc_info.value.message).lower() + assert "Authentication Error" not in str(exc_info.value.message) + + @pytest.mark.asyncio async def test_block_key_existing_key_succeeds(monkeypatch): """ From 0b63979d4572bfa6225df9bc62f9fbae8d6b3710 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 17:04:42 -0700 Subject: [PATCH 34/66] Fix build: cast endpointType to EndpointType at call site MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ChatUI stores endpointType as string but the narrowed prop expects EndpointType — add explicit cast at the call site. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/components/playground/chat_ui/ChatUI.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx index ec37a9e1e99..ef57a75062c 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx @@ -1936,7 +1936,7 @@ const ChatUI: React.FC = ({ Date: Wed, 18 Mar 2026 17:56:25 -0700 Subject: [PATCH 35/66] [Feature] UI - Leftnav: Add external link icon to Learning Resources Add ExportOutlined icon next to nav items that link to external pages, making it clear to users when a link opens in a new tab. Co-Authored-By: Claude Opus 4.6 (1M context) --- ui/litellm-dashboard/src/components/leftnav.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index d01fc06bc05..d3789fcffa5 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -13,6 +13,7 @@ import { CreditCardOutlined, DatabaseOutlined, ExperimentOutlined, + ExportOutlined, FileTextOutlined, FolderOutlined, KeyOutlined, @@ -400,7 +401,7 @@ const Sidebar: React.FC = ({ setPage, defaultSelectedKey, collapse onClick={(e) => e.stopPropagation()} style={{ color: "inherit", textDecoration: "none" }} > - {label} + {label} ); } From 4770b657e15a25815eccf211f87843db356625e8 Mon Sep 17 00:00:00 2001 From: Chesars Date: Wed, 18 Mar 2026 22:05:27 -0300 Subject: [PATCH 36/66] =?UTF-8?q?refactor:=20extract=20duplicated=20stdout?= =?UTF-8?q?/stderr=20=E2=86=92=20logs=20logic=20to=20shared=20helper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- litellm/llms/anthropic/chat/handler.py | 15 ++------------- litellm/llms/anthropic/chat/transformation.py | 17 ++--------------- litellm/types/responses/main.py | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 28 deletions(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 70ecf91725d..7dce72f1e82 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -50,7 +50,7 @@ from litellm.types.llms.openai import ( ) from litellm.types.responses.main import ( OutputCodeInterpreterCall, - OutputCodeInterpreterCallLog, + build_code_interpreter_log_outputs, ) from litellm.types.utils import ( Delta, @@ -708,20 +708,9 @@ class ModelResponseIterator: continue call_id = tr.get("tool_use_id", "") content = tr.get("content", {}) - if isinstance(content, dict): - parts = [] - if content.get("stdout"): - parts.append(content["stdout"]) - if content.get("stderr"): - parts.append(f"STDERR: {content['stderr']}") - logs = "".join(parts) - else: - logs = "" + log_outputs = build_code_interpreter_log_outputs(content) tool_input = self._server_tool_inputs.get(call_id, {}) code = tool_input.get("command", "") if isinstance(tool_input, dict) else "" - log_outputs = ( - [OutputCodeInterpreterCallLog(type="logs", logs=logs)] if logs else None - ) results.append( OutputCodeInterpreterCall( type="code_interpreter_call", diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index ca101df0e95..bbc73fcfd40 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -61,7 +61,7 @@ from litellm.types.utils import ( ) from litellm.types.responses.main import ( OutputCodeInterpreterCall, - OutputCodeInterpreterCallLog, + build_code_interpreter_log_outputs, ) from litellm.utils import ( ModelResponse, @@ -1771,20 +1771,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): continue call_id = tr.get("tool_use_id", "") content = tr.get("content", {}) - if isinstance(content, dict): - parts = [] - if content.get("stdout"): - parts.append(content["stdout"]) - if content.get("stderr"): - parts.append(f"STDERR: {content['stderr']}") - logs = "".join(parts) - else: - logs = "" - log_outputs = ( - [OutputCodeInterpreterCallLog(type="logs", logs=logs)] - if logs - else None - ) + log_outputs = build_code_interpreter_log_outputs(content) code_interpreter_results.append( OutputCodeInterpreterCall( type="code_interpreter_call", diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index e46857565c7..ebd2ad5b5a8 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -67,6 +67,24 @@ class OutputCodeInterpreterCall(BaseLiteLLMOpenAIResponseObject): outputs: Optional[List[OutputCodeInterpreterCallLog]] +def build_code_interpreter_log_outputs( + content: Any, +) -> Optional[List[OutputCodeInterpreterCallLog]]: + """Convert Anthropic bash_code_execution stdout/stderr to log outputs. + + Shared by streaming (handler.py) and non-streaming (transformation.py) paths. + """ + if not isinstance(content, dict): + return None + parts = [] + if content.get("stdout"): + parts.append(content["stdout"]) + if content.get("stderr"): + parts.append(f"STDERR: {content['stderr']}") + logs = "".join(parts) + return [OutputCodeInterpreterCallLog(type="logs", logs=logs)] if logs else None + + class GenericResponseOutputItem(BaseLiteLLMOpenAIResponseObject): """ Generic response API output item From 8969a3d1763e7e2bb3b0c48dbb855780cf992ebe Mon Sep 17 00:00:00 2001 From: xianren Date: Thu, 19 Mar 2026 09:10:21 +0800 Subject: [PATCH 37/66] Fixed thinking blocks dropped when thinking field is null (#24026) The check `content.get("thinking", None) is not None` incorrectly drops thinking blocks when the `thinking` key is explicitly null or absent. Changed to `content.get("type") == "thinking"` to match the fix already applied in the experimental pass-through path (PR #15501). Co-Authored-By: Claude Opus 4.6 (1M context) --- litellm/llms/anthropic/chat/transformation.py | 2 +- .../test_anthropic_chat_transformation.py | 51 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 47cdd8287e0..7552becef64 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1522,7 +1522,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): tool_results = [] tool_results.append(content) - elif content.get("thinking", None) is not None: + elif content.get("type") == "thinking": if thinking_blocks is None: thinking_blocks = [] thinking_blocks.append(cast(ChatCompletionThinkingBlock, content)) diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index a95b9413b9d..7f5b7d6158e 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -3321,3 +3321,54 @@ def test_map_tool_helper_empty_parameters_get_default(): assert result is not None assert result["input_schema"]["type"] == "object" assert result["input_schema"].get("properties") == {} + + +def test_extract_response_content_thinking_block_null_thinking(): + """ + Test that thinking blocks are not dropped when the 'thinking' field is null + or missing. Regression test for https://github.com/BerriAI/litellm/issues/24026 + """ + config = AnthropicConfig() + + # Case 1: thinking key is explicitly null + completion_response_null = { + "content": [ + {"type": "thinking", "thinking": None, "signature": "sig123"}, + {"type": "text", "text": "Hello"}, + ] + } + text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content( + completion_response_null + ) + assert thinking_blocks is not None, "thinking blocks should not be None when thinking=null" + assert len(thinking_blocks) == 1 + assert "Hello" in text + + # Case 2: thinking key is absent entirely + completion_response_missing = { + "content": [ + {"type": "thinking", "signature": "sig456"}, + {"type": "text", "text": "World"}, + ] + } + text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content( + completion_response_missing + ) + assert thinking_blocks is not None, "thinking blocks should not be None when thinking key is absent" + assert len(thinking_blocks) == 1 + assert "World" in text + + # Case 3: thinking key has actual content (should still work) + completion_response_text = { + "content": [ + {"type": "thinking", "thinking": "Let me think...", "signature": "sig789"}, + {"type": "text", "text": "Done"}, + ] + } + text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content( + completion_response_text + ) + assert thinking_blocks is not None + assert len(thinking_blocks) == 1 + assert thinking_blocks[0]["thinking"] == "Let me think..." + assert "Done" in text From bd0c3bfdc4d9cd364c8b68b975ed8398c55b0dc0 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 18 Mar 2026 20:58:41 -0700 Subject: [PATCH 38/66] fix: fix logging for response incomplete streaming --- litellm/litellm_core_utils/litellm_logging.py | 294 ++++++++---------- .../anthropic_passthrough_logging_handler.py | 40 ++- litellm/responses/streaming_iterator.py | 52 ++-- .../test_litellm_logging.py | 192 ++++++++++-- 4 files changed, 342 insertions(+), 236 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 4e63dd70760..5e34a4c9925 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -12,47 +12,28 @@ import time import traceback from datetime import datetime as dt_object from functools import lru_cache -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - List, - Literal, - Optional, - Tuple, - Type, - Union, - cast, -) +from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Literal, + Optional, Tuple, Type, Union, cast) from httpx import Response from pydantic import BaseModel import litellm -from litellm import ( - _custom_logger_compatible_callbacks_literal, - json_logs, - log_raw_request_response, - turn_off_message_logging, -) +from litellm import (_custom_logger_compatible_callbacks_literal, json_logs, + log_raw_request_response, turn_off_message_logging) from litellm._logging import _is_debugging_on, verbose_logger from litellm._uuid import uuid from litellm.batches.batch_utils import _handle_completed_batch from litellm.caching.caching import DualCache, InMemoryCache from litellm.caching.caching_handler import LLMCachingHandler -from litellm.constants import ( - DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, - DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, - SENTRY_DENYLIST, - SENTRY_PII_DENYLIST, -) -from litellm.cost_calculator import ( - RealtimeAPITokenUsageProcessor, - _select_model_name_for_cost_calc, -) +from litellm.constants import (DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, + DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, + SENTRY_DENYLIST, SENTRY_PII_DENYLIST) +from litellm.cost_calculator import (RealtimeAPITokenUsageProcessor, + _select_model_name_for_cost_calc) from litellm.integrations.agentops import AgentOps -from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook +from litellm.integrations.anthropic_cache_control_hook import \ + AnthropicCacheControlHook from litellm.integrations.arize.arize import ArizeLogger from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger @@ -61,70 +42,48 @@ from litellm.integrations.mlflow import MlflowLogger from litellm.integrations.sqs import SQSLogger from litellm.litellm_core_utils.core_helpers import reconstruct_model_name from litellm.litellm_core_utils.get_litellm_params import get_litellm_params -from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( - StandardBuiltInToolCostTracking, -) -from litellm.litellm_core_utils.logging_utils import truncate_base64_in_messages +from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import \ + StandardBuiltInToolCostTracking +from litellm.litellm_core_utils.logging_utils import \ + truncate_base64_in_messages from litellm.litellm_core_utils.model_param_helper import ModelParamHelper from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_custom_logger, - redact_message_input_output_from_logging, -) + redact_message_input_output_from_logging) from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.agents import LiteLLMSendMessageResponse from litellm.types.containers.main import ContainerObject -from litellm.types.llms.openai import ( - AllMessageValues, - Batch, - FineTuningJob, - HttpxBinaryResponseContent, - OpenAIFileObject, - OpenAIModerationResponse, - ResponseAPIUsage, - ResponseCompletedEvent, - ResponsesAPIResponse, -) +from litellm.types.llms.openai import (AllMessageValues, Batch, FineTuningJob, + HttpxBinaryResponseContent, + OpenAIFileObject, + OpenAIModerationResponse, + ResponseAPIUsage, + ResponseCompletedEvent, + ResponseFailedEvent, + ResponseIncompleteEvent, + ResponsesAPIResponse) from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse from litellm.types.utils import ( - CachingDetails, - CallTypes, - CostBreakdown, - CostResponseTypes, - CustomPricingLiteLLMParams, - DynamicPromptManagementParamLiteral, - EmbeddingResponse, - GuardrailStatus, - ImageResponse, - LiteLLMBatch, - LiteLLMLoggingBaseClass, - LiteLLMRealtimeStreamLoggingObject, - ModelResponse, - ModelResponseStream, - RawRequestTypedDict, - StandardBuiltInToolsParams, - StandardCallbackDynamicParams, - StandardLoggingAdditionalHeaders, - StandardLoggingHiddenParams, - StandardLoggingMCPToolCall, - StandardLoggingMetadata, - StandardLoggingModelCostFailureDebugInformation, - StandardLoggingModelInformation, - StandardLoggingPayload, - StandardLoggingPayloadErrorInformation, - StandardLoggingPayloadStatus, + CachingDetails, CallTypes, CostBreakdown, CostResponseTypes, + CustomPricingLiteLLMParams, DynamicPromptManagementParamLiteral, + EmbeddingResponse, GuardrailStatus, ImageResponse, LiteLLMBatch, + LiteLLMLoggingBaseClass, LiteLLMRealtimeStreamLoggingObject, ModelResponse, + ModelResponseStream, RawRequestTypedDict, StandardBuiltInToolsParams, + StandardCallbackDynamicParams, StandardLoggingAdditionalHeaders, + StandardLoggingHiddenParams, StandardLoggingMCPToolCall, + StandardLoggingMetadata, StandardLoggingModelCostFailureDebugInformation, + StandardLoggingModelInformation, StandardLoggingPayload, + StandardLoggingPayloadErrorInformation, StandardLoggingPayloadStatus, StandardLoggingPayloadStatusFields, - StandardLoggingPromptManagementMetadata, - StandardLoggingVectorStoreRequest, - TextCompletionResponse, - TranscriptionResponse, - Usage, -) + StandardLoggingPromptManagementMetadata, StandardLoggingVectorStoreRequest, + TextCompletionResponse, TranscriptionResponse, Usage) from litellm.types.videos.main import VideoObject -from litellm.utils import _get_base_model_from_metadata, executor, print_verbose +from litellm.utils import (_get_base_model_from_metadata, executor, + print_verbose) from ..integrations.argilla import ArgillaLogger from ..integrations.arize.arize_phoenix import ArizePhoenixLogger @@ -146,7 +105,8 @@ from ..integrations.humanloop import HumanloopLogger from ..integrations.lago import LagoLogger from ..integrations.langfuse.langfuse import LangFuseLogger from ..integrations.langfuse.langfuse_handler import LangFuseHandler -from ..integrations.langfuse.langfuse_prompt_management import LangfusePromptManagement +from ..integrations.langfuse.langfuse_prompt_management import \ + LangfusePromptManagement from ..integrations.langsmith import LangsmithLogger from ..integrations.litellm_agent import LiteLLMAgentModelResolver from ..integrations.literal_ai import LiteralAILogger @@ -161,34 +121,30 @@ from ..integrations.s3_v2 import S3Logger as S3V2Logger from ..integrations.supabase import Supabase from ..integrations.traceloop import TraceloopLogger from .exception_mapping_utils import _get_response_headers -from .initialize_dynamic_callback_params import ( - initialize_standard_callback_dynamic_params as _initialize_standard_callback_dynamic_params, -) +from .initialize_dynamic_callback_params import \ + initialize_standard_callback_dynamic_params as \ + _initialize_standard_callback_dynamic_params from .specialty_caches.dynamic_logging_cache import DynamicLoggingCache if TYPE_CHECKING: - from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig + from litellm.llms.base_llm.passthrough.transformation import \ + BasePassthroughConfig try: - from litellm_enterprise.enterprise_callbacks.callback_controls import ( - EnterpriseCallbackControls, - ) - from litellm_enterprise.enterprise_callbacks.pagerduty.pagerduty import ( - PagerDutyAlerting, - ) - from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import ( - ResendEmailLogger, - ) - from litellm_enterprise.enterprise_callbacks.send_emails.sendgrid_email import ( - SendGridEmailLogger, - ) - from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import ( - SMTPEmailLogger, - ) - from litellm_enterprise.litellm_core_utils.litellm_logging import ( - StandardLoggingPayloadSetup as EnterpriseStandardLoggingPayloadSetup, - ) + from litellm_enterprise.enterprise_callbacks.callback_controls import \ + EnterpriseCallbackControls + from litellm_enterprise.enterprise_callbacks.pagerduty.pagerduty import \ + PagerDutyAlerting + from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import \ + ResendEmailLogger + from litellm_enterprise.enterprise_callbacks.send_emails.sendgrid_email import \ + SendGridEmailLogger + from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import \ + SMTPEmailLogger + from litellm_enterprise.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup as EnterpriseStandardLoggingPayloadSetup - from litellm.integrations.generic_api.generic_api_callback import GenericAPILogger + from litellm.integrations.generic_api.generic_api_callback import \ + GenericAPILogger EnterpriseStandardLoggingPayloadSetupVAR: Optional[ Type[EnterpriseStandardLoggingPayloadSetup] @@ -516,6 +472,23 @@ class Logging(LiteLLMLoggingBaseClass): ), ) + def get_router_model_id(self) -> Optional[str]: + """Extract the router deployment model_id from litellm_params. + + Checks both litellm_metadata and metadata for model_info.id. + Used by cost calculators to look up custom pricing registered + under the deployment's model_info.id in litellm.model_cost. + """ + if not hasattr(self, "litellm_params"): + return None + for key in ("litellm_metadata", "metadata"): + meta = self.litellm_params.get(key, {}) or {} + info = meta.get("model_info", {}) or {} + model_id = info.get("id") + if model_id is not None: + return model_id + return None + def update_environment_variables( self, litellm_params: Dict, @@ -1458,16 +1431,8 @@ class Logging(LiteLLMLoggingBaseClass): # Fallback: extract router_model_id from litellm_params when not available # from the result object. ResponsesAPIResponse objects (used by /v1/responses # streaming) don't carry _hidden_params["model_id"] like ModelResponse does. - if router_model_id is None and hasattr(self, "litellm_params"): - for metadata_key in ("litellm_metadata", "metadata"): - _metadata: dict = ( - self.litellm_params.get(metadata_key, {}) or {} - ) - _model_info: dict = _metadata.get("model_info", {}) or {} - _model_id = _model_info.get("id") - if _model_id is not None: - router_model_id = _model_id - break + if router_model_id is None: + router_model_id = self.get_router_model_id() ## RESPONSE COST ## custom_pricing = use_custom_pricing_for_model( @@ -1758,9 +1723,8 @@ class Logging(LiteLLMLoggingBaseClass): ) standard_logging_payload["response"] = response_dict elif isinstance(result, TranscriptionResponse): - from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import ( - TranscriptionUsageObjectTransformation, - ) + from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import \ + TranscriptionUsageObjectTransformation result = result.model_copy() transformed_usage = TranscriptionUsageObjectTransformation.transform_transcription_usage_object(result.usage) # type: ignore @@ -2443,9 +2407,8 @@ class Logging(LiteLLMLoggingBaseClass): ): # polling job will query these frequently, don't spam db logs return - from litellm.proxy.openai_files_endpoints.common_utils import ( - _is_base64_encoded_unified_file_id, - ) + from litellm.proxy.openai_files_endpoints.common_utils import \ + _is_base64_encoded_unified_file_id # check if file id is a unified file id is_base64_unified_file_id = _is_base64_encoded_unified_file_id(result.id) @@ -3321,7 +3284,7 @@ class Logging(LiteLLMLoggingBaseClass): return result elif isinstance(result, TextCompletionResponse): return result - elif isinstance(result, ResponseCompletedEvent): + elif isinstance(result, (ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent)): ## return unified Usage object if isinstance(result.response.usage, ResponseAPIUsage): transformed_usage = ( @@ -3588,7 +3551,8 @@ def set_callbacks(callback_list, function_id=None): # noqa: PLR0915 elif callback == "s3": s3Logger = S3Logger() elif callback == "wandb": - from litellm.integrations.weights_biases import WeightsBiasesLogger + from litellm.integrations.weights_biases import \ + WeightsBiasesLogger weightsBiasesLogger = WeightsBiasesLogger() elif callback == "logfire": @@ -3652,7 +3616,8 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(_posthog_logger) return _posthog_logger # type: ignore elif logging_integration == "braintrust": - from litellm.integrations.braintrust_logging import BraintrustLogger + from litellm.integrations.braintrust_logging import \ + BraintrustLogger for callback in _in_memory_loggers: if isinstance(callback, BraintrustLogger): @@ -3773,9 +3738,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 return _opik_logger # type: ignore elif logging_integration == "arize": from litellm.integrations.opentelemetry import ( - OpenTelemetry, - OpenTelemetryConfig, - ) + OpenTelemetry, OpenTelemetryConfig) arize_config = ArizeLogger.get_arize_config() if arize_config.endpoint is None: @@ -3802,9 +3765,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 return _arize_otel_logger # type: ignore elif logging_integration == "arize_phoenix": from litellm.integrations.opentelemetry import ( - OpenTelemetry, - OpenTelemetryConfig, - ) + OpenTelemetry, OpenTelemetryConfig) arize_phoenix_config = ArizePhoenixLogger.get_arize_phoenix_config() otel_config = OpenTelemetryConfig( @@ -3858,9 +3819,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 elif logging_integration == "levo": from litellm.integrations.levo.levo import LevoLogger from litellm.integrations.opentelemetry import ( - OpenTelemetry, - OpenTelemetryConfig, - ) + OpenTelemetry, OpenTelemetryConfig) levo_config = LevoLogger.get_levo_config() otel_config = OpenTelemetryConfig( @@ -3909,7 +3868,8 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(galileo_logger) return galileo_logger # type: ignore elif logging_integration == "cloudzero": - from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger + from litellm.integrations.cloudzero.cloudzero import \ + CloudZeroLogger for callback in _in_memory_loggers: if isinstance(callback, CloudZeroLogger): @@ -3929,7 +3889,8 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(focus_logger) return focus_logger # type: ignore elif logging_integration == "vantage": - from litellm.integrations.vantage.vantage_logger import VantageLogger + from litellm.integrations.vantage.vantage_logger import \ + VantageLogger for callback in _in_memory_loggers: if isinstance(callback, VantageLogger): @@ -3949,9 +3910,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 if "LOGFIRE_TOKEN" not in os.environ: raise ValueError("LOGFIRE_TOKEN not found in environment variables") from litellm.integrations.opentelemetry import ( - OpenTelemetry, - OpenTelemetryConfig, - ) + OpenTelemetry, OpenTelemetryConfig) logfire_base_url = os.getenv( "LOGFIRE_BASE_URL", "https://logfire-api.pydantic.dev" @@ -3969,9 +3928,8 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(_otel_logger) return _otel_logger # type: ignore elif logging_integration == "dynamic_rate_limiter": - from litellm.proxy.hooks.dynamic_rate_limiter import ( - _PROXY_DynamicRateLimitHandler, - ) + from litellm.proxy.hooks.dynamic_rate_limiter import \ + _PROXY_DynamicRateLimitHandler for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandler): @@ -3993,9 +3951,8 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(dynamic_rate_limiter_obj) return dynamic_rate_limiter_obj # type: ignore elif logging_integration == "dynamic_rate_limiter_v3": - from litellm.proxy.hooks.dynamic_rate_limiter_v3 import ( - _PROXY_DynamicRateLimitHandlerV3, - ) + from litellm.proxy.hooks.dynamic_rate_limiter_v3 import \ + _PROXY_DynamicRateLimitHandlerV3 for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandlerV3): @@ -4021,9 +3978,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 raise ValueError("LANGTRACE_API_KEY not found in environment variables") from litellm.integrations.opentelemetry import ( - OpenTelemetry, - OpenTelemetryConfig, - ) + OpenTelemetry, OpenTelemetryConfig) otel_config = OpenTelemetryConfig( exporter="otlp_http", @@ -4059,7 +4014,8 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(langfuse_logger) return langfuse_logger # type: ignore elif logging_integration == "langfuse_otel": - from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger + from litellm.integrations.langfuse.langfuse_otel import \ + LangfuseOtelLogger for callback in _in_memory_loggers: if ( @@ -4077,9 +4033,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 elif logging_integration == "weave_otel": from litellm.integrations.opentelemetry import OpenTelemetryConfig from litellm.integrations.weave.weave_otel import ( - WeaveOtelLogger, - get_weave_otel_config, - ) + WeaveOtelLogger, get_weave_otel_config) weave_otel_config = get_weave_otel_config() @@ -4115,9 +4069,8 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(anthropic_cache_control_hook) return anthropic_cache_control_hook # type: ignore elif logging_integration == "vector_store_pre_call_hook": - from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( - VectorStorePreCallHook, - ) + from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import \ + VectorStorePreCallHook for callback in _in_memory_loggers: if isinstance(callback, VectorStorePreCallHook): @@ -4177,9 +4130,8 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(dotprompt_logger) return dotprompt_logger # type: ignore elif logging_integration == "bitbucket": - from litellm.integrations.bitbucket.bitbucket_prompt_manager import ( - BitBucketPromptManager, - ) + from litellm.integrations.bitbucket.bitbucket_prompt_manager import \ + BitBucketPromptManager for callback in _in_memory_loggers: if isinstance(callback, BitBucketPromptManager): @@ -4196,9 +4148,8 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(bitbucket_logger) return bitbucket_logger # type: ignore elif logging_integration == "gitlab": - from litellm.integrations.gitlab.gitlab_prompt_manager import ( - GitLabPromptManager, - ) + from litellm.integrations.gitlab.gitlab_prompt_manager import \ + GitLabPromptManager for callback in _in_memory_loggers: if isinstance(callback, GitLabPromptManager): @@ -4286,7 +4237,8 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 if isinstance(callback, OpenMeterLogger): return callback elif logging_integration == "braintrust": - from litellm.integrations.braintrust_logging import BraintrustLogger + from litellm.integrations.braintrust_logging import \ + BraintrustLogger for callback in _in_memory_loggers: if isinstance(callback, BraintrustLogger): @@ -4296,7 +4248,8 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 if isinstance(callback, GalileoObserve): return callback elif logging_integration == "cloudzero": - from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger + from litellm.integrations.cloudzero.cloudzero import \ + CloudZeroLogger for callback in _in_memory_loggers: if isinstance(callback, CloudZeroLogger): @@ -4310,7 +4263,8 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 ): # exact match; exclude subclasses like VantageLogger return callback elif logging_integration == "vantage": - from litellm.integrations.vantage.vantage_logger import VantageLogger + from litellm.integrations.vantage.vantage_logger import \ + VantageLogger for callback in _in_memory_loggers: if isinstance(callback, VantageLogger): @@ -4410,17 +4364,15 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 return callback # type: ignore elif logging_integration == "dynamic_rate_limiter": - from litellm.proxy.hooks.dynamic_rate_limiter import ( - _PROXY_DynamicRateLimitHandler, - ) + from litellm.proxy.hooks.dynamic_rate_limiter import \ + _PROXY_DynamicRateLimitHandler for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandler): return callback # type: ignore elif logging_integration == "dynamic_rate_limiter_v3": - from litellm.proxy.hooks.dynamic_rate_limiter_v3 import ( - _PROXY_DynamicRateLimitHandlerV3, - ) + from litellm.proxy.hooks.dynamic_rate_limiter_v3 import \ + _PROXY_DynamicRateLimitHandlerV3 for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandlerV3): @@ -4452,9 +4404,8 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 if isinstance(callback, AnthropicCacheControlHook): return callback elif logging_integration == "vector_store_pre_call_hook": - from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( - VectorStorePreCallHook, - ) + from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import \ + VectorStorePreCallHook for callback in _in_memory_loggers: if isinstance(callback, VectorStorePreCallHook): @@ -5626,7 +5577,6 @@ def _get_traceback_str_for_error(error_str: str) -> str: from decimal import Decimal - # used for unit testing from typing import Any, Dict, List, Optional, Union diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index 20d06b7d531..3241c1ca93f 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -6,20 +6,23 @@ import httpx import litellm from litellm._logging import verbose_proxy_logger -from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.litellm_logging import \ + Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.litellm_logging import \ + use_custom_pricing_for_model from litellm.llms.anthropic import get_anthropic_config -from litellm.llms.anthropic.chat.handler import ( - ModelResponseIterator as AnthropicModelResponseIterator, -) +from litellm.llms.anthropic.chat.handler import \ + ModelResponseIterator as AnthropicModelResponseIterator from litellm.proxy._types import PassThroughEndpointLoggingTypedDict from litellm.proxy.auth.auth_utils import get_end_user_id_from_request_body -from litellm.types.passthrough_endpoints.pass_through_endpoints import ( - PassthroughStandardLoggingPayload, -) -from litellm.types.utils import LiteLLMBatch, ModelResponse, TextCompletionResponse +from litellm.types.passthrough_endpoints.pass_through_endpoints import \ + PassthroughStandardLoggingPayload +from litellm.types.utils import (LiteLLMBatch, ModelResponse, + TextCompletionResponse) if TYPE_CHECKING: - from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType + from litellm.types.passthrough_endpoints.pass_through_endpoints import \ + EndpointType from ..success_handler import PassThroughEndpointLogging else: @@ -124,10 +127,21 @@ class AnthropicPassthroughLoggingHandler: if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/"): model_for_cost = f"{custom_llm_provider}/{model}" + router_model_id = logging_obj.get_router_model_id() + custom_pricing = use_custom_pricing_for_model( + litellm_params=( + logging_obj.litellm_params + if hasattr(logging_obj, "litellm_params") + else None + ) + ) + response_cost = litellm.completion_cost( completion_response=litellm_model_response, model=model_for_cost, custom_llm_provider=custom_llm_provider, + custom_pricing=custom_pricing, + router_model_id=router_model_id, ) kwargs["response_cost"] = response_cost @@ -319,9 +333,8 @@ class AnthropicPassthroughLoggingHandler: import base64 from litellm._uuid import uuid - from litellm.llms.anthropic.batches.transformation import ( - AnthropicBatchesConfig, - ) + from litellm.llms.anthropic.batches.transformation import \ + AnthropicBatchesConfig from litellm.types.utils import Choices, SpecialEnums try: @@ -537,7 +550,8 @@ class AnthropicPassthroughLoggingHandler: managed_files_hook, "store_unified_object_id" ): # Create a mock user API key dict for the managed object storage - from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy._types import (LitellmUserRoles, + UserAPIKeyAuth) user_api_key_dict = UserAPIKeyAuth( user_id=kwargs.get("user_id", "default-user"), diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 073ee926063..a6a1074067d 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -8,31 +8,29 @@ from typing import Any, Dict, List, Optional import httpx import litellm -from litellm.constants import ( - LITELLM_MAX_STREAMING_DURATION_SECONDS, - STREAM_SSE_DONE_STRING, -) +from litellm.constants import (LITELLM_MAX_STREAMING_DURATION_SECONDS, + STREAM_SSE_DONE_STRING) from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import process_response_headers -from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base -from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( - update_response_metadata, -) +from litellm.litellm_core_utils.litellm_logging import \ + Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.llm_response_utils.get_api_base import \ + get_api_base +from litellm.litellm_core_utils.llm_response_utils.response_metadata import \ + update_response_metadata from litellm.litellm_core_utils.thread_pool_executor import executor -from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.llms.base_llm.responses.transformation import \ + BaseResponsesAPIConfig from litellm.responses.utils import ResponsesAPIRequestUtils -from litellm.types.llms.openai import ( - OutputTextDeltaEvent, - ResponseAPIUsage, - ResponseCompletedEvent, - ResponsesAPIRequestParams, - ResponsesAPIResponse, - ResponsesAPIStreamEvents, - ResponsesAPIStreamingResponse, -) +from litellm.types.llms.openai import (OutputTextDeltaEvent, ResponseAPIUsage, + ResponseCompletedEvent, + ResponsesAPIRequestParams, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ResponsesAPIStreamingResponse) from litellm.types.utils import CallTypes -from litellm.utils import CustomStreamWrapper, async_post_call_success_deployment_hook +from litellm.utils import (CustomStreamWrapper, + async_post_call_success_deployment_hook) class BaseResponsesAPIStreamingIterator: @@ -166,11 +164,16 @@ class BaseResponsesAPIStreamingIterator: ) setattr(item, "encrypted_content", wrapped_content) - # Store the completed response + # Store the completed response (also for incomplete/failed so logging still fires) + _chunk_type = getattr(openai_responses_api_chunk, "type", None) if ( openai_responses_api_chunk - and getattr(openai_responses_api_chunk, "type", None) - == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + and _chunk_type + in ( + ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, + ResponsesAPIStreamEvents.RESPONSE_FAILED, + ) ): self.completed_response = openai_responses_api_chunk # Add cost to usage object if include_cost_in_streaming_usage is True @@ -694,7 +697,8 @@ class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): # --------------------------------------------------------------------------- from litellm._logging import verbose_logger -from litellm.litellm_core_utils.thread_pool_executor import executor as _ws_executor +from litellm.litellm_core_utils.thread_pool_executor import \ + executor as _ws_executor RESPONSES_WS_LOGGED_EVENT_TYPES = [ "response.created", diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index fe4851283f4..0f950f6da77 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -11,7 +11,8 @@ sys.path.insert( import time from litellm.constants import SENTRY_DENYLIST, SENTRY_PII_DENYLIST -from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging +from litellm.litellm_core_utils.litellm_logging import \ + Logging as LitellmLogging from litellm.litellm_core_utils.litellm_logging import set_callbacks from litellm.types.utils import ModelResponse, TextCompletionResponse @@ -139,7 +140,8 @@ def test_sentry_environment(): def test_use_custom_pricing_for_model(): - from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model + from litellm.litellm_core_utils.litellm_logging import \ + use_custom_pricing_for_model litellm_params = { "custom_llm_provider": "azure", @@ -154,7 +156,8 @@ def test_use_custom_pricing_for_model_via_litellm_metadata(): Generic API call routes (/messages, /responses) store model_info under litellm_metadata, not metadata. Regression test for #23185. """ - from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model + from litellm.litellm_core_utils.litellm_logging import \ + use_custom_pricing_for_model litellm_params = { "litellm_metadata": { @@ -170,7 +173,8 @@ def test_use_custom_pricing_for_model_via_litellm_metadata(): def test_use_custom_pricing_not_detected_litellm_metadata_no_pricing(): """Should return False when litellm_metadata.model_info has no pricing keys.""" - from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model + from litellm.litellm_core_utils.litellm_logging import \ + use_custom_pricing_for_model litellm_params = { "litellm_metadata": { @@ -186,7 +190,8 @@ def test_response_cost_calculator_uses_router_model_id_from_litellm_metadata(): does not carry _hidden_params (e.g. ResponsesAPIResponse from /v1/responses streaming). Regression test for custom pricing on streaming responses.""" import litellm - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.litellm_logging import \ + Logging as LiteLLMLoggingObj from litellm.types.llms.openai import ResponsesAPIResponse custom_model_id = "gpt-5-custom-pricing" @@ -256,6 +261,121 @@ def test_response_cost_calculator_uses_router_model_id_from_litellm_metadata(): litellm.model_cost.pop(custom_model_id, None) +class TestGetRouterModelId: + """Tests for the get_router_model_id helper method.""" + + def test_returns_id_from_litellm_metadata(self, logging_obj): + """Should extract model_info.id from litellm_metadata.""" + logging_obj.litellm_params = { + "litellm_metadata": { + "model_info": {"id": "custom-deploy-1"}, + }, + } + assert logging_obj.get_router_model_id() == "custom-deploy-1" + + def test_returns_id_from_metadata(self, logging_obj): + """Should fall back to metadata when litellm_metadata has no model_info.""" + logging_obj.litellm_params = { + "metadata": { + "model_info": {"id": "custom-deploy-2"}, + }, + } + assert logging_obj.get_router_model_id() == "custom-deploy-2" + + def test_prefers_litellm_metadata_over_metadata(self, logging_obj): + """litellm_metadata should take priority over metadata.""" + logging_obj.litellm_params = { + "litellm_metadata": { + "model_info": {"id": "from-litellm-meta"}, + }, + "metadata": { + "model_info": {"id": "from-meta"}, + }, + } + assert logging_obj.get_router_model_id() == "from-litellm-meta" + + def test_returns_none_when_no_model_info(self, logging_obj): + """Should return None when no model_info is present.""" + logging_obj.litellm_params = {"api_base": ""} + assert logging_obj.get_router_model_id() is None + + def test_returns_none_when_no_litellm_params(self): + """Should return None when litellm_params is not set.""" + from litellm.litellm_core_utils.litellm_logging import \ + Logging as LiteLLMLoggingObj + + obj = LiteLLMLoggingObj( + model="test", + messages=[], + stream=False, + call_type="completion", + start_time=time.time(), + litellm_call_id="x", + function_id="x", + ) + # litellm_params exists but is empty by default + assert obj.get_router_model_id() is None + + +class TestAnthropicPassthroughCustomPricing: + """Verify the Anthropic pass-through handler forwards custom pricing.""" + + def test_completion_cost_receives_custom_pricing_args(self): + """_create_anthropic_response_logging_payload should pass + custom_pricing and router_model_id to litellm.completion_cost + when the logging object carries custom pricing in model_info.""" + from unittest.mock import patch + + from litellm.litellm_core_utils.litellm_logging import \ + Logging as LiteLLMLoggingObj + from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import \ + AnthropicPassthroughLoggingHandler + + logging_obj = LiteLLMLoggingObj( + model="claude-sonnet-4-20250514", + messages=[{"role": "user", "content": "Hi"}], + stream=False, + call_type="anthropic_messages", + start_time=time.time(), + litellm_call_id="test-456", + function_id="test-fn", + ) + logging_obj.update_environment_variables( + model="claude-sonnet-4-20250514", + user="", + optional_params={}, + litellm_params={ + "api_base": "", + "litellm_metadata": { + "model_info": { + "id": "claude-custom-pricing", + "input_cost_per_token": 0.5, + "output_cost_per_token": 1.5, + }, + }, + }, + ) + logging_obj.model_call_details["custom_llm_provider"] = "anthropic" + + mock_response = ModelResponse() + mock_response.usage = {"prompt_tokens": 10, "completion_tokens": 5} # type: ignore + + with patch("litellm.completion_cost", return_value=42.0) as mock_cost: + AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( + litellm_model_response=mock_response, + model="claude-sonnet-4-20250514", + kwargs={}, + start_time=time.time(), + end_time=time.time(), + logging_obj=logging_obj, + ) + + mock_cost.assert_called_once() + call_kwargs = mock_cost.call_args + assert call_kwargs.kwargs.get("custom_pricing") is True + assert call_kwargs.kwargs.get("router_model_id") == "claude-custom-pricing" + + class TestUpdateFromKwargs: """Tests for the update_from_kwargs convenience wrapper.""" @@ -321,9 +441,8 @@ class TestUpdateFromKwargs: def test_custom_pricing_detected_via_litellm_metadata(self, logging_obj): """Custom pricing in litellm_metadata.model_info should set custom_pricing flag.""" - from litellm.litellm_core_utils.litellm_logging import ( - use_custom_pricing_for_model, - ) + from litellm.litellm_core_utils.litellm_logging import \ + use_custom_pricing_for_model lm_meta = { "model_info": { @@ -382,7 +501,8 @@ async def test_datadog_logger_not_shadowed_by_llm_obs(monkeypatch): monkeypatch.setenv("DD_SITE", "us5.datadoghq.com") from litellm.integrations.datadog.datadog import DataDogLogger - from litellm.integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger + from litellm.integrations.datadog.datadog_llm_obs import \ + DataDogLLMObsLogger from litellm.litellm_core_utils import litellm_logging as logging_module logging_module._in_memory_loggers.clear() @@ -423,7 +543,8 @@ async def test_logfire_logger_accepts_env_vars_for_base_url(monkeypatch): ) # no trailing slash on purpose # Import after env vars are set (important if module-level caching exists) - from litellm.integrations.opentelemetry import OpenTelemetry # logger class + from litellm.integrations.opentelemetry import \ + OpenTelemetry # logger class from litellm.litellm_core_utils import litellm_logging as logging_module logging_module._in_memory_loggers.clear() @@ -752,7 +873,8 @@ def test_success_handler_runs_guardrail_logging_hook_when_enabled(logging_obj): def test_get_user_agent_tags(): - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup tags = StandardLoggingPayloadSetup._get_user_agent_tags( proxy_server_request={ @@ -767,7 +889,8 @@ def test_get_user_agent_tags(): def test_get_request_tags(): - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup tags = StandardLoggingPayloadSetup._get_request_tags( litellm_params={"metadata": {"tags": ["test-tag"]}}, @@ -794,7 +917,8 @@ def test_get_request_tags_from_metadata_and_litellm_metadata(): 4. No tags in either 5. None values for metadata/litellm_metadata """ - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup # Test case 1: Tags in metadata only tags = StandardLoggingPayloadSetup._get_request_tags( @@ -875,7 +999,8 @@ def test_get_request_tags_does_not_mutate_original_tags(): would cause User-Agent tags to be duplicated because the function was mutating the original tags list instead of creating a copy. """ - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup # Create metadata with original tags original_tags = ["custom-tag-1", "custom-tag-2"] @@ -935,7 +1060,8 @@ def test_get_request_tags_does_not_mutate_original_tags(): def test_get_extra_header_tags(): """Test the _get_extra_header_tags method with various scenarios.""" import litellm - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup # Store original value to restore later original_extra_headers = getattr(litellm, "extra_spend_tag_headers", None) @@ -1156,7 +1282,8 @@ async def test_e2e_generate_cold_storage_object_key_successful(): from datetime import datetime, timezone from unittest.mock import patch - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup # Create test data start_time = datetime(2025, 1, 15, 10, 30, 45, 123456, timezone.utc) @@ -1198,7 +1325,8 @@ async def test_e2e_generate_cold_storage_object_key_with_custom_logger_s3_path() from datetime import datetime, timezone from unittest.mock import MagicMock, patch - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup # Create test data start_time = datetime(2025, 1, 15, 10, 30, 45, 123456, timezone.utc) @@ -1249,7 +1377,8 @@ async def test_e2e_generate_cold_storage_object_key_with_logger_no_s3_path(): from datetime import datetime, timezone from unittest.mock import MagicMock, patch - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup # Create test data start_time = datetime(2025, 1, 15, 10, 30, 45, 123456, timezone.utc) @@ -1296,7 +1425,8 @@ async def test_e2e_generate_cold_storage_object_key_not_configured(): from unittest.mock import patch import litellm - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup # Create test data start_time = datetime(2025, 1, 15, 10, 30, 45, 123456, timezone.utc) @@ -1320,7 +1450,8 @@ def test_get_final_response_obj_with_empty_response_obj_and_list_init(): When response_obj is empty (falsy), the method should return init_response_obj if it's a list. """ - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup # Create test objects class TestObject1: @@ -1356,7 +1487,8 @@ def test_get_usage_as_dict(): """ Test get_usage_as_dict returns usage as plain dict from response_obj or combined_usage_object. """ - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup from litellm.types.utils import Usage # Test case 1: None response_obj returns empty usage dict @@ -1394,7 +1526,8 @@ def test_append_system_prompt_messages(): """ Test append_system_prompt_messages prepends system message from kwargs to messages list. """ - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup # Test case 1: system in kwargs with existing messages kwargs = {"system": "You are a helpful assistant"} @@ -1465,7 +1598,8 @@ async def test_async_success_handler_sets_standard_logging_object_for_pass_throu from datetime import datetime from unittest.mock import patch - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.litellm_logging import \ + Logging as LiteLLMLoggingObj from litellm.types.utils import StandardPassThroughResponseObject # Create a logging object for a pass-through endpoint @@ -1546,7 +1680,8 @@ async def test_async_success_handler_prevents_reprocessing_for_pass_through_endp from datetime import datetime from unittest.mock import patch - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.litellm_logging import \ + Logging as LiteLLMLoggingObj from litellm.types.utils import StandardPassThroughResponseObject # Create a logging object for a pass-through endpoint @@ -1622,7 +1757,8 @@ async def test_async_success_handler_sets_standard_logging_object_for_streaming_ from datetime import datetime from unittest.mock import patch - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.litellm_logging import \ + Logging as LiteLLMLoggingObj from litellm.types.utils import StandardPassThroughResponseObject # Create a logging object for a streaming pass-through endpoint @@ -1678,7 +1814,8 @@ def test_get_error_information_error_code_priority(): Test get_error_information prioritizes 'code' attribute over 'status_code' attribute and handles edge cases like empty strings and "None" string values. """ - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup # Test case 1: Exception with 'code' attribute (ProxyException style) class ProxyException(Exception): @@ -1871,7 +2008,8 @@ async def test_async_success_handler_preserves_response_cost_for_pass_through_en by pass-through handlers (Gemini/Vertex).""" from datetime import datetime - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.litellm_logging import \ + Logging as LiteLLMLoggingObj from litellm.types.utils import ModelResponse, Usage logging_obj = LiteLLMLoggingObj( From 08f0cbc2e939c6456feb2f7ef8edf118644748fa Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 18 Mar 2026 21:36:39 -0700 Subject: [PATCH 39/66] fix: address greptile feedback --- litellm/litellm_core_utils/litellm_logging.py | 266 +++++++++++------- litellm/responses/streaming_iterator.py | 73 +++-- ...t_base_responses_api_streaming_iterator.py | 156 +++++++++- 3 files changed, 369 insertions(+), 126 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 5e34a4c9925..27cef858189 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -12,28 +12,47 @@ import time import traceback from datetime import datetime as dt_object from functools import lru_cache -from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Literal, - Optional, Tuple, Type, Union, cast) +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + List, + Literal, + Optional, + Tuple, + Type, + Union, + cast, +) from httpx import Response from pydantic import BaseModel import litellm -from litellm import (_custom_logger_compatible_callbacks_literal, json_logs, - log_raw_request_response, turn_off_message_logging) +from litellm import ( + _custom_logger_compatible_callbacks_literal, + json_logs, + log_raw_request_response, + turn_off_message_logging, +) from litellm._logging import _is_debugging_on, verbose_logger from litellm._uuid import uuid from litellm.batches.batch_utils import _handle_completed_batch from litellm.caching.caching import DualCache, InMemoryCache from litellm.caching.caching_handler import LLMCachingHandler -from litellm.constants import (DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, - DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, - SENTRY_DENYLIST, SENTRY_PII_DENYLIST) -from litellm.cost_calculator import (RealtimeAPITokenUsageProcessor, - _select_model_name_for_cost_calc) +from litellm.constants import ( + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, + DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, + SENTRY_DENYLIST, + SENTRY_PII_DENYLIST, +) +from litellm.cost_calculator import ( + RealtimeAPITokenUsageProcessor, + _select_model_name_for_cost_calc, +) from litellm.integrations.agentops import AgentOps -from litellm.integrations.anthropic_cache_control_hook import \ - AnthropicCacheControlHook +from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook from litellm.integrations.arize.arize import ArizeLogger from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger @@ -42,48 +61,72 @@ from litellm.integrations.mlflow import MlflowLogger from litellm.integrations.sqs import SQSLogger from litellm.litellm_core_utils.core_helpers import reconstruct_model_name from litellm.litellm_core_utils.get_litellm_params import get_litellm_params -from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import \ - StandardBuiltInToolCostTracking -from litellm.litellm_core_utils.logging_utils import \ - truncate_base64_in_messages +from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( + StandardBuiltInToolCostTracking, +) +from litellm.litellm_core_utils.logging_utils import truncate_base64_in_messages from litellm.litellm_core_utils.model_param_helper import ModelParamHelper from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_custom_logger, - redact_message_input_output_from_logging) + redact_message_input_output_from_logging, +) from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.agents import LiteLLMSendMessageResponse from litellm.types.containers.main import ContainerObject -from litellm.types.llms.openai import (AllMessageValues, Batch, FineTuningJob, - HttpxBinaryResponseContent, - OpenAIFileObject, - OpenAIModerationResponse, - ResponseAPIUsage, - ResponseCompletedEvent, - ResponseFailedEvent, - ResponseIncompleteEvent, - ResponsesAPIResponse) +from litellm.types.llms.openai import ( + AllMessageValues, + Batch, + FineTuningJob, + HttpxBinaryResponseContent, + OpenAIFileObject, + OpenAIModerationResponse, + ResponseAPIUsage, + ResponseCompletedEvent, + ResponseFailedEvent, + ResponseIncompleteEvent, + ResponsesAPIResponse, +) from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse from litellm.types.utils import ( - CachingDetails, CallTypes, CostBreakdown, CostResponseTypes, - CustomPricingLiteLLMParams, DynamicPromptManagementParamLiteral, - EmbeddingResponse, GuardrailStatus, ImageResponse, LiteLLMBatch, - LiteLLMLoggingBaseClass, LiteLLMRealtimeStreamLoggingObject, ModelResponse, - ModelResponseStream, RawRequestTypedDict, StandardBuiltInToolsParams, - StandardCallbackDynamicParams, StandardLoggingAdditionalHeaders, - StandardLoggingHiddenParams, StandardLoggingMCPToolCall, - StandardLoggingMetadata, StandardLoggingModelCostFailureDebugInformation, - StandardLoggingModelInformation, StandardLoggingPayload, - StandardLoggingPayloadErrorInformation, StandardLoggingPayloadStatus, + CachingDetails, + CallTypes, + CostBreakdown, + CostResponseTypes, + CustomPricingLiteLLMParams, + DynamicPromptManagementParamLiteral, + EmbeddingResponse, + GuardrailStatus, + ImageResponse, + LiteLLMBatch, + LiteLLMLoggingBaseClass, + LiteLLMRealtimeStreamLoggingObject, + ModelResponse, + ModelResponseStream, + RawRequestTypedDict, + StandardBuiltInToolsParams, + StandardCallbackDynamicParams, + StandardLoggingAdditionalHeaders, + StandardLoggingHiddenParams, + StandardLoggingMCPToolCall, + StandardLoggingMetadata, + StandardLoggingModelCostFailureDebugInformation, + StandardLoggingModelInformation, + StandardLoggingPayload, + StandardLoggingPayloadErrorInformation, + StandardLoggingPayloadStatus, StandardLoggingPayloadStatusFields, - StandardLoggingPromptManagementMetadata, StandardLoggingVectorStoreRequest, - TextCompletionResponse, TranscriptionResponse, Usage) + StandardLoggingPromptManagementMetadata, + StandardLoggingVectorStoreRequest, + TextCompletionResponse, + TranscriptionResponse, + Usage, +) from litellm.types.videos.main import VideoObject -from litellm.utils import (_get_base_model_from_metadata, executor, - print_verbose) +from litellm.utils import _get_base_model_from_metadata, executor, print_verbose from ..integrations.argilla import ArgillaLogger from ..integrations.arize.arize_phoenix import ArizePhoenixLogger @@ -105,8 +148,7 @@ from ..integrations.humanloop import HumanloopLogger from ..integrations.lago import LagoLogger from ..integrations.langfuse.langfuse import LangFuseLogger from ..integrations.langfuse.langfuse_handler import LangFuseHandler -from ..integrations.langfuse.langfuse_prompt_management import \ - LangfusePromptManagement +from ..integrations.langfuse.langfuse_prompt_management import LangfusePromptManagement from ..integrations.langsmith import LangsmithLogger from ..integrations.litellm_agent import LiteLLMAgentModelResolver from ..integrations.literal_ai import LiteralAILogger @@ -121,30 +163,34 @@ from ..integrations.s3_v2 import S3Logger as S3V2Logger from ..integrations.supabase import Supabase from ..integrations.traceloop import TraceloopLogger from .exception_mapping_utils import _get_response_headers -from .initialize_dynamic_callback_params import \ - initialize_standard_callback_dynamic_params as \ - _initialize_standard_callback_dynamic_params +from .initialize_dynamic_callback_params import ( + initialize_standard_callback_dynamic_params as _initialize_standard_callback_dynamic_params, +) from .specialty_caches.dynamic_logging_cache import DynamicLoggingCache if TYPE_CHECKING: - from litellm.llms.base_llm.passthrough.transformation import \ - BasePassthroughConfig + from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig try: - from litellm_enterprise.enterprise_callbacks.callback_controls import \ - EnterpriseCallbackControls - from litellm_enterprise.enterprise_callbacks.pagerduty.pagerduty import \ - PagerDutyAlerting - from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import \ - ResendEmailLogger - from litellm_enterprise.enterprise_callbacks.send_emails.sendgrid_email import \ - SendGridEmailLogger - from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import \ - SMTPEmailLogger - from litellm_enterprise.litellm_core_utils.litellm_logging import \ - StandardLoggingPayloadSetup as EnterpriseStandardLoggingPayloadSetup + from litellm_enterprise.enterprise_callbacks.callback_controls import ( + EnterpriseCallbackControls, + ) + from litellm_enterprise.enterprise_callbacks.pagerduty.pagerduty import ( + PagerDutyAlerting, + ) + from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import ( + ResendEmailLogger, + ) + from litellm_enterprise.enterprise_callbacks.send_emails.sendgrid_email import ( + SendGridEmailLogger, + ) + from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import ( + SMTPEmailLogger, + ) + from litellm_enterprise.litellm_core_utils.litellm_logging import ( + StandardLoggingPayloadSetup as EnterpriseStandardLoggingPayloadSetup, + ) - from litellm.integrations.generic_api.generic_api_callback import \ - GenericAPILogger + from litellm.integrations.generic_api.generic_api_callback import GenericAPILogger EnterpriseStandardLoggingPayloadSetupVAR: Optional[ Type[EnterpriseStandardLoggingPayloadSetup] @@ -1723,8 +1769,9 @@ class Logging(LiteLLMLoggingBaseClass): ) standard_logging_payload["response"] = response_dict elif isinstance(result, TranscriptionResponse): - from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import \ - TranscriptionUsageObjectTransformation + from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import ( + TranscriptionUsageObjectTransformation, + ) result = result.model_copy() transformed_usage = TranscriptionUsageObjectTransformation.transform_transcription_usage_object(result.usage) # type: ignore @@ -2407,8 +2454,9 @@ class Logging(LiteLLMLoggingBaseClass): ): # polling job will query these frequently, don't spam db logs return - from litellm.proxy.openai_files_endpoints.common_utils import \ - _is_base64_encoded_unified_file_id + from litellm.proxy.openai_files_endpoints.common_utils import ( + _is_base64_encoded_unified_file_id, + ) # check if file id is a unified file id is_base64_unified_file_id = _is_base64_encoded_unified_file_id(result.id) @@ -3305,7 +3353,6 @@ class Logging(LiteLLMLoggingBaseClass): return result.response else: return None - return None def _handle_anthropic_messages_response_logging(self, result: Any) -> ModelResponse: """ @@ -3551,8 +3598,7 @@ def set_callbacks(callback_list, function_id=None): # noqa: PLR0915 elif callback == "s3": s3Logger = S3Logger() elif callback == "wandb": - from litellm.integrations.weights_biases import \ - WeightsBiasesLogger + from litellm.integrations.weights_biases import WeightsBiasesLogger weightsBiasesLogger = WeightsBiasesLogger() elif callback == "logfire": @@ -3616,8 +3662,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(_posthog_logger) return _posthog_logger # type: ignore elif logging_integration == "braintrust": - from litellm.integrations.braintrust_logging import \ - BraintrustLogger + from litellm.integrations.braintrust_logging import BraintrustLogger for callback in _in_memory_loggers: if isinstance(callback, BraintrustLogger): @@ -3738,7 +3783,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 return _opik_logger # type: ignore elif logging_integration == "arize": from litellm.integrations.opentelemetry import ( - OpenTelemetry, OpenTelemetryConfig) + OpenTelemetry, + OpenTelemetryConfig, + ) arize_config = ArizeLogger.get_arize_config() if arize_config.endpoint is None: @@ -3765,7 +3812,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 return _arize_otel_logger # type: ignore elif logging_integration == "arize_phoenix": from litellm.integrations.opentelemetry import ( - OpenTelemetry, OpenTelemetryConfig) + OpenTelemetry, + OpenTelemetryConfig, + ) arize_phoenix_config = ArizePhoenixLogger.get_arize_phoenix_config() otel_config = OpenTelemetryConfig( @@ -3819,7 +3868,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 elif logging_integration == "levo": from litellm.integrations.levo.levo import LevoLogger from litellm.integrations.opentelemetry import ( - OpenTelemetry, OpenTelemetryConfig) + OpenTelemetry, + OpenTelemetryConfig, + ) levo_config = LevoLogger.get_levo_config() otel_config = OpenTelemetryConfig( @@ -3868,8 +3919,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(galileo_logger) return galileo_logger # type: ignore elif logging_integration == "cloudzero": - from litellm.integrations.cloudzero.cloudzero import \ - CloudZeroLogger + from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger for callback in _in_memory_loggers: if isinstance(callback, CloudZeroLogger): @@ -3889,8 +3939,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(focus_logger) return focus_logger # type: ignore elif logging_integration == "vantage": - from litellm.integrations.vantage.vantage_logger import \ - VantageLogger + from litellm.integrations.vantage.vantage_logger import VantageLogger for callback in _in_memory_loggers: if isinstance(callback, VantageLogger): @@ -3910,7 +3959,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 if "LOGFIRE_TOKEN" not in os.environ: raise ValueError("LOGFIRE_TOKEN not found in environment variables") from litellm.integrations.opentelemetry import ( - OpenTelemetry, OpenTelemetryConfig) + OpenTelemetry, + OpenTelemetryConfig, + ) logfire_base_url = os.getenv( "LOGFIRE_BASE_URL", "https://logfire-api.pydantic.dev" @@ -3928,8 +3979,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(_otel_logger) return _otel_logger # type: ignore elif logging_integration == "dynamic_rate_limiter": - from litellm.proxy.hooks.dynamic_rate_limiter import \ - _PROXY_DynamicRateLimitHandler + from litellm.proxy.hooks.dynamic_rate_limiter import ( + _PROXY_DynamicRateLimitHandler, + ) for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandler): @@ -3951,8 +4003,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(dynamic_rate_limiter_obj) return dynamic_rate_limiter_obj # type: ignore elif logging_integration == "dynamic_rate_limiter_v3": - from litellm.proxy.hooks.dynamic_rate_limiter_v3 import \ - _PROXY_DynamicRateLimitHandlerV3 + from litellm.proxy.hooks.dynamic_rate_limiter_v3 import ( + _PROXY_DynamicRateLimitHandlerV3, + ) for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandlerV3): @@ -3978,7 +4031,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 raise ValueError("LANGTRACE_API_KEY not found in environment variables") from litellm.integrations.opentelemetry import ( - OpenTelemetry, OpenTelemetryConfig) + OpenTelemetry, + OpenTelemetryConfig, + ) otel_config = OpenTelemetryConfig( exporter="otlp_http", @@ -4014,8 +4069,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(langfuse_logger) return langfuse_logger # type: ignore elif logging_integration == "langfuse_otel": - from litellm.integrations.langfuse.langfuse_otel import \ - LangfuseOtelLogger + from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger for callback in _in_memory_loggers: if ( @@ -4033,7 +4087,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 elif logging_integration == "weave_otel": from litellm.integrations.opentelemetry import OpenTelemetryConfig from litellm.integrations.weave.weave_otel import ( - WeaveOtelLogger, get_weave_otel_config) + WeaveOtelLogger, + get_weave_otel_config, + ) weave_otel_config = get_weave_otel_config() @@ -4069,8 +4125,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(anthropic_cache_control_hook) return anthropic_cache_control_hook # type: ignore elif logging_integration == "vector_store_pre_call_hook": - from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import \ - VectorStorePreCallHook + from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( + VectorStorePreCallHook, + ) for callback in _in_memory_loggers: if isinstance(callback, VectorStorePreCallHook): @@ -4130,8 +4187,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(dotprompt_logger) return dotprompt_logger # type: ignore elif logging_integration == "bitbucket": - from litellm.integrations.bitbucket.bitbucket_prompt_manager import \ - BitBucketPromptManager + from litellm.integrations.bitbucket.bitbucket_prompt_manager import ( + BitBucketPromptManager, + ) for callback in _in_memory_loggers: if isinstance(callback, BitBucketPromptManager): @@ -4148,8 +4206,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(bitbucket_logger) return bitbucket_logger # type: ignore elif logging_integration == "gitlab": - from litellm.integrations.gitlab.gitlab_prompt_manager import \ - GitLabPromptManager + from litellm.integrations.gitlab.gitlab_prompt_manager import ( + GitLabPromptManager, + ) for callback in _in_memory_loggers: if isinstance(callback, GitLabPromptManager): @@ -4237,8 +4296,7 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 if isinstance(callback, OpenMeterLogger): return callback elif logging_integration == "braintrust": - from litellm.integrations.braintrust_logging import \ - BraintrustLogger + from litellm.integrations.braintrust_logging import BraintrustLogger for callback in _in_memory_loggers: if isinstance(callback, BraintrustLogger): @@ -4248,8 +4306,7 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 if isinstance(callback, GalileoObserve): return callback elif logging_integration == "cloudzero": - from litellm.integrations.cloudzero.cloudzero import \ - CloudZeroLogger + from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger for callback in _in_memory_loggers: if isinstance(callback, CloudZeroLogger): @@ -4263,8 +4320,7 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 ): # exact match; exclude subclasses like VantageLogger return callback elif logging_integration == "vantage": - from litellm.integrations.vantage.vantage_logger import \ - VantageLogger + from litellm.integrations.vantage.vantage_logger import VantageLogger for callback in _in_memory_loggers: if isinstance(callback, VantageLogger): @@ -4364,15 +4420,17 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 return callback # type: ignore elif logging_integration == "dynamic_rate_limiter": - from litellm.proxy.hooks.dynamic_rate_limiter import \ - _PROXY_DynamicRateLimitHandler + from litellm.proxy.hooks.dynamic_rate_limiter import ( + _PROXY_DynamicRateLimitHandler, + ) for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandler): return callback # type: ignore elif logging_integration == "dynamic_rate_limiter_v3": - from litellm.proxy.hooks.dynamic_rate_limiter_v3 import \ - _PROXY_DynamicRateLimitHandlerV3 + from litellm.proxy.hooks.dynamic_rate_limiter_v3 import ( + _PROXY_DynamicRateLimitHandlerV3, + ) for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandlerV3): @@ -4404,8 +4462,9 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 if isinstance(callback, AnthropicCacheControlHook): return callback elif logging_integration == "vector_store_pre_call_hook": - from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import \ - VectorStorePreCallHook + from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( + VectorStorePreCallHook, + ) for callback in _in_memory_loggers: if isinstance(callback, VectorStorePreCallHook): @@ -5577,6 +5636,7 @@ def _get_traceback_str_for_error(error_str: str) -> str: from decimal import Decimal + # used for unit testing from typing import Any, Dict, List, Optional, Union diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index a6a1074067d..8a91368dd68 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -8,29 +8,31 @@ from typing import Any, Dict, List, Optional import httpx import litellm -from litellm.constants import (LITELLM_MAX_STREAMING_DURATION_SECONDS, - STREAM_SSE_DONE_STRING) +from litellm.constants import ( + LITELLM_MAX_STREAMING_DURATION_SECONDS, + STREAM_SSE_DONE_STRING, +) from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import process_response_headers -from litellm.litellm_core_utils.litellm_logging import \ - Logging as LiteLLMLoggingObj -from litellm.litellm_core_utils.llm_response_utils.get_api_base import \ - get_api_base -from litellm.litellm_core_utils.llm_response_utils.response_metadata import \ - update_response_metadata +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base +from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + update_response_metadata, +) from litellm.litellm_core_utils.thread_pool_executor import executor -from litellm.llms.base_llm.responses.transformation import \ - BaseResponsesAPIConfig +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.responses.utils import ResponsesAPIRequestUtils -from litellm.types.llms.openai import (OutputTextDeltaEvent, ResponseAPIUsage, - ResponseCompletedEvent, - ResponsesAPIRequestParams, - ResponsesAPIResponse, - ResponsesAPIStreamEvents, - ResponsesAPIStreamingResponse) +from litellm.types.llms.openai import ( + OutputTextDeltaEvent, + ResponseAPIUsage, + ResponseCompletedEvent, + ResponsesAPIRequestParams, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ResponsesAPIStreamingResponse, +) from litellm.types.utils import CallTypes -from litellm.utils import (CustomStreamWrapper, - async_post_call_success_deployment_hook) +from litellm.utils import CustomStreamWrapper, async_post_call_success_deployment_hook class BaseResponsesAPIStreamingIterator: @@ -198,10 +200,12 @@ class BaseResponsesAPIStreamingIterator: if cost is not None: setattr(usage_obj, "cost", cost) except Exception: - # If cost calculation fails, continue without cost pass - self._handle_logging_completed_response() + if _chunk_type == ResponsesAPIStreamEvents.RESPONSE_FAILED: + self._handle_logging_failed_response() + else: + self._handle_logging_completed_response() return openai_responses_api_chunk @@ -219,6 +223,32 @@ class BaseResponsesAPIStreamingIterator: """Base implementation - should be overridden by subclasses""" pass + def _handle_logging_failed_response(self): + """ + Handle logging for RESPONSE_FAILED events by routing to failure handlers. + + Unlike _handle_logging_completed_response (which calls success handlers), + this constructs an exception from the response error and routes to + async_failure_handler / failure_handler so logging integrations correctly + record the call as failed. + """ + response_obj = ( + getattr(self.completed_response, "response", None) + if self.completed_response + else None + ) + error_info = getattr(response_obj, "error", None) if response_obj else None + error_message = "Response failed" + if isinstance(error_info, dict): + error_message = error_info.get("message", str(error_info)) + exception = litellm.APIError( + status_code=500, + message=error_message, + llm_provider=self.custom_llm_provider or "", + model=self.model or "", + ) + self._handle_failure(exception) + async def _call_post_streaming_deployment_hook(self, chunk): """ Allow callbacks to modify streaming chunks before returning (parity with chat). @@ -697,8 +727,7 @@ class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): # --------------------------------------------------------------------------- from litellm._logging import verbose_logger -from litellm.litellm_core_utils.thread_pool_executor import \ - executor as _ws_executor +from litellm.litellm_core_utils.thread_pool_executor import executor as _ws_executor RESPONSES_WS_LOGGED_EVENT_TYPES = [ "response.created", diff --git a/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py b/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py index 860445d875d..e9181d810e1 100644 --- a/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py +++ b/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py @@ -30,9 +30,11 @@ from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterat from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( ResponseCompletedEvent, + ResponseFailedEvent, + ResponseIncompleteEvent, ResponsesAPIResponse, ResponsesAPIStreamEvents, - OutputTextDeltaEvent + OutputTextDeltaEvent, ) @@ -429,3 +431,155 @@ class TestBaseResponsesAPIStreamingIterator: mock_logging_obj.async_failure_handler.assert_not_called() mock_logging_obj.failure_handler.assert_not_called() + def test_process_chunk_response_failed_calls_failure_handler(self): + """ + Test that a RESPONSE_FAILED event routes to failure handlers, + not success handlers. Failed responses represent genuine LLM-level + errors and should be logged as failures. + """ + from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator + + mock_response = Mock() + mock_response.headers = {} + mock_response.aiter_lines = Mock() + mock_logging_obj = Mock(spec=LiteLLMLoggingObj) + mock_logging_obj.model_call_details = {"litellm_params": {}} + mock_logging_obj.async_failure_handler = Mock() + mock_logging_obj.failure_handler = Mock() + mock_logging_obj.async_success_handler = Mock() + mock_logging_obj.success_handler = Mock() + mock_config = Mock(spec=BaseResponsesAPIConfig) + + mock_responses_api_response = Mock(spec=ResponsesAPIResponse) + mock_responses_api_response.id = "resp_failed_123" + mock_responses_api_response.error = { + "type": "server_error", + "message": "The model encountered an error", + } + mock_responses_api_response.usage = None + + mock_failed_event = Mock(spec=ResponseFailedEvent) + mock_failed_event.type = ResponsesAPIStreamEvents.RESPONSE_FAILED + mock_failed_event.response = mock_responses_api_response + + mock_config.transform_streaming_response.return_value = mock_failed_event + + iterator = ResponsesAPIStreamingIterator( + response=mock_response, + model="gpt-4", + responses_api_provider_config=mock_config, + logging_obj=mock_logging_obj, + litellm_metadata={"model_info": {"id": "model_123"}}, + custom_llm_provider="openai", + ) + + test_chunk_data = { + "type": "response.failed", + "response": { + "id": "resp_failed_123", + "error": { + "type": "server_error", + "message": "The model encountered an error", + }, + }, + } + + with patch.object( + ResponsesAPIRequestUtils, + "_update_responses_api_response_id_with_model_id", + return_value=mock_responses_api_response, + ), patch( + "litellm.responses.streaming_iterator.run_async_function" + ) as mock_run_async, patch( + "litellm.responses.streaming_iterator.executor" + ) as mock_executor: + result = iterator._process_chunk(json.dumps(test_chunk_data)) + + assert result is not None + assert result.type == ResponsesAPIStreamEvents.RESPONSE_FAILED + assert iterator.completed_response == result + + # Failure handler should have been called via _handle_failure + mock_run_async.assert_called_once() + call_kwargs = mock_run_async.call_args + assert ( + call_kwargs[1]["async_function"] + == mock_logging_obj.async_failure_handler + ) + + mock_executor.submit.assert_called_once() + submit_args = mock_executor.submit.call_args + assert submit_args[0][0] == mock_logging_obj.failure_handler + + def test_process_chunk_response_incomplete_calls_success_handler(self): + """ + Test that a RESPONSE_INCOMPLETE event routes to success handlers. + Incomplete responses (e.g. max_output_tokens reached) are still valid + responses with usage data — analogous to finish_reason='length' in chat. + """ + from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator + + mock_response = Mock() + mock_response.headers = {} + mock_response.aiter_lines = Mock() + mock_logging_obj = Mock(spec=LiteLLMLoggingObj) + mock_logging_obj.model_call_details = {"litellm_params": {}} + mock_logging_obj.async_failure_handler = Mock() + mock_logging_obj.failure_handler = Mock() + mock_logging_obj.async_success_handler = Mock() + mock_logging_obj.success_handler = Mock() + mock_config = Mock(spec=BaseResponsesAPIConfig) + + mock_responses_api_response = Mock(spec=ResponsesAPIResponse) + mock_responses_api_response.id = "resp_incomplete_123" + mock_responses_api_response.incomplete_details = { + "reason": "max_output_tokens" + } + mock_responses_api_response.usage = None + + mock_incomplete_event = Mock(spec=ResponseIncompleteEvent) + mock_incomplete_event.type = ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE + mock_incomplete_event.response = mock_responses_api_response + + mock_config.transform_streaming_response.return_value = mock_incomplete_event + + iterator = ResponsesAPIStreamingIterator( + response=mock_response, + model="gpt-4", + responses_api_provider_config=mock_config, + logging_obj=mock_logging_obj, + litellm_metadata={"model_info": {"id": "model_123"}}, + custom_llm_provider="openai", + ) + + test_chunk_data = { + "type": "response.incomplete", + "response": { + "id": "resp_incomplete_123", + "incomplete_details": {"reason": "max_output_tokens"}, + }, + } + + with patch.object( + ResponsesAPIRequestUtils, + "_update_responses_api_response_id_with_model_id", + return_value=mock_responses_api_response, + ), patch( + "asyncio.create_task" + ) as mock_create_task, patch( + "litellm.responses.streaming_iterator.executor" + ) as mock_executor: + result = iterator._process_chunk(json.dumps(test_chunk_data)) + + assert result is not None + assert result.type == ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE + assert iterator.completed_response == result + + # Success handler should have been called (via _handle_logging_completed_response) + mock_create_task.assert_called_once() + mock_executor.submit.assert_called_once() + + # Failure handlers should NOT have been called + mock_logging_obj.async_failure_handler.assert_not_called() + mock_logging_obj.failure_handler.assert_not_called() + From bba3b1fe4c468576c145affbb08f6a6587eedc2c Mon Sep 17 00:00:00 2001 From: joereyna Date: Wed, 18 Mar 2026 22:42:25 -0700 Subject: [PATCH 40/66] docs(release-notes): add missing Helicone and Langfuse entries to v1.82.3 changelog Helicone (PRs #19288, #22603) and Langfuse (#22390) were present in the v1.82.0-stable...v1.82.3-stable diff but omitted from the AI Integrations logging section. Also updates the AI Integrations diff summary count from 2 to 4. Co-Authored-By: Claude Sonnet 4.6 --- docs/my-website/release_notes/v1.82.3/index.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/my-website/release_notes/v1.82.3/index.md b/docs/my-website/release_notes/v1.82.3/index.md index 15df33fdb80..b586cab7d6e 100644 --- a/docs/my-website/release_notes/v1.82.3/index.md +++ b/docs/my-website/release_notes/v1.82.3/index.md @@ -297,6 +297,13 @@ pip install litellm==1.82.3 ### Logging +- **[Helicone](../../docs/observability/helicone_integration)** + - Add Gemini and Vertex AI support to HeliconeLogger — routes Gemini and Vertex AI requests through the correct Helicone provider URL - [PR #19288](https://github.com/BerriAI/litellm/pull/19288) + - Fix correct provider URL for Vertex AI Gemini models - [PR #22603](https://github.com/BerriAI/litellm/pull/22603) + +- **[Langfuse](../../docs/proxy/logging#langfuse)** + - Fix failure path kwargs inconsistency causing dropped traces on failed requests - [PR #22390](https://github.com/BerriAI/litellm/pull/22390) + - **[Vantage](https://vantage.sh)** - Add Vantage integration for FOCUS 1.2 CSV export — export LiteLLM proxy spend data as FinOps Open Cost & Usage Specification reports, with time-windowed filenames to prevent overwrites - [PR #23333](https://github.com/BerriAI/litellm/pull/23333) @@ -363,7 +370,7 @@ No major secret manager changes in this release. * New Models / Updated Models: 116 new, 132 removed * LLM API Endpoints: 5 * Management Endpoints / UI: 11 -* AI Integrations: 2 +* AI Integrations: 4 * Performance / Reliability: 5 * Security: 3 * Database / Proxy Operations: 2 From dab8721ba316a6b859d8636c80bc36b94f32a767 Mon Sep 17 00:00:00 2001 From: joereyna Date: Wed, 18 Mar 2026 22:57:38 -0700 Subject: [PATCH 41/66] chore: apply black formatting to fix lint CI --- litellm/litellm_core_utils/litellm_logging.py | 7 ++----- litellm/proxy/management_endpoints/team_endpoints.py | 4 +++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 4e63dd70760..1b612c7091b 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1460,9 +1460,7 @@ class Logging(LiteLLMLoggingBaseClass): # streaming) don't carry _hidden_params["model_id"] like ModelResponse does. if router_model_id is None and hasattr(self, "litellm_params"): for metadata_key in ("litellm_metadata", "metadata"): - _metadata: dict = ( - self.litellm_params.get(metadata_key, {}) or {} - ) + _metadata: dict = self.litellm_params.get(metadata_key, {}) or {} _model_info: dict = _metadata.get("model_info", {}) or {} _model_id = _model_info.get("id") if _model_id is not None: @@ -2972,8 +2970,7 @@ class Logging(LiteLLMLoggingBaseClass): if ( isinstance(callback, CustomLogger) and is_sync_request - and self.call_type - != CallTypes.pass_through.value + and self.call_type != CallTypes.pass_through.value ): # custom logger class callback.log_failure_event( start_time=start_time, diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index b033d3fce0d..3d4488b8a78 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -3334,7 +3334,9 @@ def _convert_teams_to_response_models( use_deleted_table: bool, ) -> List[Union[TeamListItem, LiteLLM_TeamTable, LiteLLM_DeletedTeamTable]]: """Convert raw Prisma team rows to response models.""" - team_list: List[Union[TeamListItem, LiteLLM_TeamTable, LiteLLM_DeletedTeamTable]] = [] + team_list: List[ + Union[TeamListItem, LiteLLM_TeamTable, LiteLLM_DeletedTeamTable] + ] = [] for team in teams: try: team_dict = team.model_dump() From 61df7471bae37e2fa25fcbfe4917b193ff5b1b81 Mon Sep 17 00:00:00 2001 From: joereyna Date: Wed, 18 Mar 2026 23:30:42 -0700 Subject: [PATCH 42/66] docs(release-notes): complete v1.82.3 changelog with 30+ missing features MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Full audit of 371 PRs in v1.82.0-stable...v1.82.3-stable range. Adds previously undocumented user-facing changes: - Key Highlights: Hashicorp Vault, Responses WebSocket, Org Admin RBAC, guardrail mode defaults - New Providers: Google Search API, Bedrock Mantle (7 total, was 5) - LLM API: Anthropic Files API, Mistral Voxtral transcription, WebRTC, Responses WebSocket, litellm.acount_tokens() public API, OpenRouter image edit, Vertex AI VIDEO token tracking, input_fidelity image edit, model cost aliases, per-request json schema validation, 15+ bug fixes - Management: RBAC expansion for Org Admins, Vector Store CRUD, MCP token auth + team scoping, BYOK key precedence, virtual key spend reset, batch expiry for teams, Admin Viewer audit log access, 12+ bug fixes - Guardrails: mode default list, tag-based modes, presidio fix, OTEL fix - Secret Managers: Hashicorp Vault (was "no changes") - Spend Tracking: new section — budget-linked reset fix, flex pricing, spend log cleanup, WebSearch dedup fix - Performance: 4 additional reliability fixes - Diff summary counts updated to reflect actual scope Co-Authored-By: Claude Sonnet 4.6 --- .../my-website/release_notes/v1.82.3/index.md | 147 +++++++++++++++++- 1 file changed, 139 insertions(+), 8 deletions(-) diff --git a/docs/my-website/release_notes/v1.82.3/index.md b/docs/my-website/release_notes/v1.82.3/index.md index b586cab7d6e..a51edc60c2c 100644 --- a/docs/my-website/release_notes/v1.82.3/index.md +++ b/docs/my-website/release_notes/v1.82.3/index.md @@ -47,6 +47,10 @@ pip install litellm==1.82.3 - **FLUX Kontext image editing** — `flux-kontext-pro` and `flux-kontext-max` added to Black Forest Labs, alongside `flux-pro-1.0-fill` and `flux-pro-1.0-expand` for inpainting and outpainting - **116 new models, 132 deprecated models cleaned up** — Major model map refresh including Mistral Magistral, Dashscope Qwen3 VL, xAI Grok via Azure AI, ZAI GLM-5, Serper Search; removal of OpenAI GPT-3.5/GPT-4 legacy variants, Gemini 1.5, and Vertex AI PaLM2 - **SageMaker Nova provider** — [New `sagemaker_nova` provider for Amazon Nova models on SageMaker](../../docs/providers/aws_sagemaker) - [PR #21542](https://github.com/BerriAI/litellm/pull/21542) +- **Hashicorp Vault secret manager** — Config override backend powered by Hashicorp Vault, with full UI for managing vault-sourced credentials - [PR #22939](https://github.com/BerriAI/litellm/pull/22939), [PR #23036](https://github.com/BerriAI/litellm/pull/23036) +- **Responses API WebSocket streaming** — Real-time WebSocket streaming for the Responses API, including support across all providers - [PR #22559](https://github.com/BerriAI/litellm/pull/22559), [PR #22771](https://github.com/BerriAI/litellm/pull/22771) +- **Org Admin RBAC expansion** — Org Admins can now access team management endpoints, view and invite internal users, and manage team membership without requiring a global admin role - [PR #23085](https://github.com/BerriAI/litellm/pull/23085), [PR #23080](https://github.com/BerriAI/litellm/pull/23080) +- **Guardrail mode defaults and tag-based modes** — Set a default guardrail mode list globally, and specify a list of modes in tag-based guardrail configs - [PR #22676](https://github.com/BerriAI/litellm/pull/22676), [PR #23020](https://github.com/BerriAI/litellm/pull/23020) - **Secret redaction in logs** — API keys, tokens, and credentials automatically scrubbed from all proxy log output. Enabled by default; opt out with `LITELLM_DISABLE_REDACT_SECRETS=true` - [PR #23668](https://github.com/BerriAI/litellm/pull/23668) - **Streaming stability fix** — Critical fix for `RuntimeError: Cannot send a request, as the client has been closed.` crashes after ~1 hour in production - [PR #22926](https://github.com/BerriAI/litellm/pull/22926) @@ -54,7 +58,7 @@ pip install litellm==1.82.3 ## New Providers and Endpoints -### New Providers (5 new providers) +### New Providers (7 new providers) | Provider | Supported LiteLLM Endpoints | Description | | -------- | --------------------------- | ----------- | @@ -63,6 +67,8 @@ pip install litellm==1.82.3 | [Black Forest Labs](../../docs/providers/black_forest_labs) (`black_forest_labs/`) | `/images/generations`, `/images/edits` | FLUX image generation and editing — Kontext Pro/Max, Pro 1.0 Fill/Expand | | [Serper](../../docs/providers/serper) (`serper/`) | `/search` | Web search via Serper API | | [SageMaker Nova](../../docs/providers/aws_sagemaker) (`sagemaker_nova/`) | `/chat/completions` | Amazon Nova models via SageMaker endpoint | +| [Google Search API](../../docs/providers/google_search) (`google_search/`) | `/search` | Google Search API integration - [PR #22752](https://github.com/BerriAI/litellm/pull/22752) | +| [Bedrock Mantle](../../docs/providers/bedrock) (`bedrock_mantle/`) | `/chat/completions` | Amazon Bedrock via Mantle — alternative auth and routing path for Bedrock models - [PR #22866](https://github.com/BerriAI/litellm/pull/22866) | --- @@ -238,22 +244,92 @@ pip install litellm==1.82.3 - **[Responses API](../../docs/response_api)** - Handle `response.failed`, `response.incomplete`, and `response.cancelled` terminal event types in background streaming — previously only `response.completed` was handled - [PR #23492](https://github.com/BerriAI/litellm/pull/23492) + - WebSocket streaming support for Responses API — real-time streaming via WebSocket for all providers - [PR #22559](https://github.com/BerriAI/litellm/pull/22559), [PR #22771](https://github.com/BerriAI/litellm/pull/22771) + - WebRTC support for real-time audio/video communication - [PR #23446](https://github.com/BerriAI/litellm/pull/23446) + - Responses API support for OpenAI-compatible JSON providers (`openai_like`) - [PR #21398](https://github.com/BerriAI/litellm/pull/21398) + - Route `gpt-5.4+` calls using both tools and reasoning to the Responses API automatically - [PR #23577](https://github.com/BerriAI/litellm/pull/23577) + +- **[Anthropic Files API](../../docs/providers/anthropic)** + - Full Anthropic Files API support — upload, retrieve, list, and delete files; use file references in messages - [PR #16594](https://github.com/BerriAI/litellm/pull/16594) + +- **[Mistral](../../docs/providers/mistral)** + - Voxtral audio transcription support — `mistral/voxtral-mini-*` and `mistral/voxtral-*` for audio transcription via Mistral - [PR #22801](https://github.com/BerriAI/litellm/pull/22801) + +- **[OpenAI](../../docs/providers/openai)** + - `litellm.acount_tokens()` public API — async token counting with full OpenAI provider support - [PR #22809](https://github.com/BerriAI/litellm/pull/22809) + - Normalize `reasoning_effort` dict to string for chat completion API - [PR #22981](https://github.com/BerriAI/litellm/pull/22981) + +- **[OpenRouter](../../docs/providers/openrouter)** + - Image edit support for OpenRouter models - [PR #22403](https://github.com/BerriAI/litellm/pull/22403) + +- **[Google Vertex AI](../../docs/providers/vertex)** + - VIDEO modality token usage tracking in `completion_tokens_details` - [PR #22550](https://github.com/BerriAI/litellm/pull/22550) + +- **Images API** + - `input_fidelity` parameter for image edit API - [PR #23201](https://github.com/BerriAI/litellm/pull/23201) + +- **General** + - Per-request `enable_json_schema_validation` flag for thread-safe JSON schema validation - [PR #21233](https://github.com/BerriAI/litellm/pull/21233) + - Model cost aliases expansion — define aliases in the cost map that inherit pricing from a parent model - [PR #23314](https://github.com/BerriAI/litellm/pull/23314), [PR #23457](https://github.com/BerriAI/litellm/pull/23457) + - Wildcards model support for the Files API - [PR #22740](https://github.com/BerriAI/litellm/pull/22740) #### Bug Fixes - **[Anthropic](../../docs/providers/anthropic)** - Preserve native tool format (web_search, bash, tool_search, etc.) when guardrails convert tools for the Anthropic Messages API - [PR #23526](https://github.com/BerriAI/litellm/pull/23526) + - Enforce `type: "object"` on tool input schemas in `_map_tool_helper` — fixes tool call failures for strict-schema providers - [PR #23103](https://github.com/BerriAI/litellm/pull/23103) + - Deduplicate `tool_result` messages by `tool_call_id` — prevents duplicate tool result errors in multi-turn conversations - [PR #23104](https://github.com/BerriAI/litellm/pull/23104) + - Map `reasoning_effort` to `output_config` for Claude 4.6 models - [PR #22220](https://github.com/BerriAI/litellm/pull/22220) + +- **[Google Gemini](../../docs/providers/gemini)** + - Correct streaming `finish_reason` for tool calls — was incorrectly returning `null` instead of `tool_calls` - [PR #21577](https://github.com/BerriAI/litellm/pull/21577) + - Preserve `$ref` in JSON Schema for Gemini 2.0+ — schema references were being stripped, breaking structured output - [PR #21597](https://github.com/BerriAI/litellm/pull/21597) + - Handle `minimal` `reasoning_effort` param for Gemini 3.1 models - [PR #22920](https://github.com/BerriAI/litellm/pull/22920) + +- **[Google Vertex AI](../../docs/providers/vertex)** + - Pass through native Gemini `imageConfig` params for image generation - [PR #21585](https://github.com/BerriAI/litellm/pull/21585) + - Prevent content truncation when `finish_reason` races ahead of content chunks in streaming - [PR #22692](https://github.com/BerriAI/litellm/pull/22692) + - Strip LiteLLM-internal keys from `extra_body` before merging to Gemini request body - [PR #23131](https://github.com/BerriAI/litellm/pull/23131) + - Drop unsupported `output_config` parameter from all Vertex AI requests - [PR #22884](https://github.com/BerriAI/litellm/pull/22884) + - Skip schema transforms for Gemini 2.0+ tool parameters — avoids breaking native Gemini schema handling - [PR #23265](https://github.com/BerriAI/litellm/pull/23265) + +- **[OpenRouter](../../docs/providers/openrouter)** + - Pattern-based fix for native model double-stripping when provider prefix matches model name - [PR #22320](https://github.com/BerriAI/litellm/pull/22320) + - Use provider-reported usage in streaming responses when `stream_options` is not set - [PR #21592](https://github.com/BerriAI/litellm/pull/21592) + +- **[AWS Bedrock](../../docs/providers/bedrock)** + - Extract region and model ID from `bedrock/{region}/{model}` path format - [PR #22546](https://github.com/BerriAI/litellm/pull/22546) + - Strip `scope` from `cache_control` for Anthropic messages on Bedrock and Azure AI - [PR #22867](https://github.com/BerriAI/litellm/pull/22867) + - Populate `completion_tokens_details` in Responses API responses - [PR #23243](https://github.com/BerriAI/litellm/pull/23243) + +- **[Azure AI](../../docs/providers/azure_ai)** + - Resolve `api_base` from environment variable in Document Intelligence OCR - [PR #21581](https://github.com/BerriAI/litellm/pull/21581) - **[Moonshot / Kimi](../../docs/providers/openai_compatible)** - Auto-fill `reasoning_content` for Moonshot Kimi reasoning models - [PR #23580](https://github.com/BerriAI/litellm/pull/23580) + - Preserve `image_url` blocks in multimodal messages for Moonshot - [PR #21595](https://github.com/BerriAI/litellm/pull/21595) - **[HuggingFace](../../docs/providers/huggingface)** - Forward `extra_headers` to HuggingFace embedding API - [PR #23525](https://github.com/BerriAI/litellm/pull/23525) +- **Token Counting / Cost** + - Fix `count_tokens` to include system prompts and tools in token counting API requests - [PR #22301](https://github.com/BerriAI/litellm/pull/22301) + - Pass all custom pricing fields to `register_model` in `completion()` and `embedding()` - [PR #22552](https://github.com/BerriAI/litellm/pull/22552) + +- **Tools / Function Calling** + - Gracefully repair truncated JSON in tool call arguments — prevents crashes on malformed tool responses - [PR #22503](https://github.com/BerriAI/litellm/pull/22503) + - Fix `output_item.done` for function calls not emitting `finish_reason` in streaming - [PR #22553](https://github.com/BerriAI/litellm/pull/22553) + - Preserve thinking block order with multiple web searches - [PR #23093](https://github.com/BerriAI/litellm/pull/23093) + - **General** - Normalize `content_filtered` finish reason across providers - [PR #23564](https://github.com/BerriAI/litellm/pull/23564) + - Unify `finish_reason` mapping to OpenAI-compatible values across all providers - [PR #22138](https://github.com/BerriAI/litellm/pull/22138) - Fix custom cost tracking on deployments for `/v1/messages` and `/v1/responses` - [PR #23647](https://github.com/BerriAI/litellm/pull/23647) - Fix per-request custom pricing when `router_model_id` has no pricing data — now falls back to model name + - Fix batch list showing stale `validating` status after completion - [PR #22982](https://github.com/BerriAI/litellm/pull/22982) + - Fix batch retrieve returning raw `output_file_id` when `model_id` is missing - [PR #23194](https://github.com/BerriAI/litellm/pull/23194) + - Encode batch IDs when `x-litellm-model` header is used - [PR #22653](https://github.com/BerriAI/litellm/pull/22653) + - Map `reasoning` to `reasoning_content` in streaming Delta for gpt-oss providers - [PR #22803](https://github.com/BerriAI/litellm/pull/22803) --- @@ -264,10 +340,36 @@ pip install litellm==1.82.3 - **Virtual Keys** - Add Organization dropdown to Create/Edit Key form — `organization_id` is now a first-class field in Key Ownership - [PR #23595](https://github.com/BerriAI/litellm/pull/23595) - Allow setting `organization_id` on `/key/update` — keys can be assigned or moved to a different organization after creation - [PR #23557](https://github.com/BerriAI/litellm/pull/23557) + - Manual Spend Reset for virtual keys from the UI — admins can reset key spend to zero on demand - [PR #22715](https://github.com/BerriAI/litellm/pull/22715) + - BYOK (Bring Your Own Key) — client-side provider API key takes precedence over proxy key for Anthropic `/v1/messages` - [PR #22964](https://github.com/BerriAI/litellm/pull/22964) + - UI login session duration configurable via `LITELLM_UI_SESSION_DURATION` environment variable - [PR #22182](https://github.com/BerriAI/litellm/pull/22182) + - Auto-redirect UI login to SSO via `auto_redirect_ui_login_to_sso: true` in config.yaml - [PR #23367](https://github.com/BerriAI/litellm/pull/23367) + +- **Access Control (RBAC)** + - Org Admins can now access team management endpoints — `/team/new`, `/team/update`, `/team/delete`, `/team/member_add`, `/team/member_delete` - [PR #23085](https://github.com/BerriAI/litellm/pull/23085), [PR #23095](https://github.com/BerriAI/litellm/pull/23095) + - Org Admins can view and invite internal users — full user management without requiring global admin role - [PR #23080](https://github.com/BerriAI/litellm/pull/23080) + - Allow Admin Viewers to access Audit Logs — view-only admin role now includes audit log access - [PR #23419](https://github.com/BerriAI/litellm/pull/23419) + - RBAC for Vector Stores and Agents — key/team-level access control for vector store and agent resources - [PR #22858](https://github.com/BerriAI/litellm/pull/22858) + - User filter scope (`scope_user_search_to_org`) is now opt-in — previously default-on, causing unintended restriction - [PR #23057](https://github.com/BerriAI/litellm/pull/23057) + +- **Vector Stores** + - Vector Store management endpoints — retrieve, list, update, and delete vector stores via `/v1/vector_stores/*` - [PR #23435](https://github.com/BerriAI/litellm/pull/23435) + +- **MCP Servers** + - Token authentication support for MCP servers — configure `auth_type: "bearer"` per MCP server - [PR #23260](https://github.com/BerriAI/litellm/pull/23260) + - Team-scoped MCP server filtering for key creation — keys only see MCP servers available to their team - [PR #23323](https://github.com/BerriAI/litellm/pull/23323) + - Per-server health recheck in the UI - [PR #23328](https://github.com/BerriAI/litellm/pull/23328) + +- **Teams** + - Batch expiry setting for teams — configure a default expiry duration for all team keys - [PR #22705](https://github.com/BerriAI/litellm/pull/22705) + - Team Admin can reset key spend - [PR #22725](https://github.com/BerriAI/litellm/pull/22725) - **Internal Users** - Add/Remove Team Membership directly from the Internal Users info page — includes searchable dropdown and role selector; no longer requires navigating to each team - [PR #23638](https://github.com/BerriAI/litellm/pull/23638) +- **Models** + - Attach knowledge base to model via UI - [PR #22656](https://github.com/BerriAI/litellm/pull/22656) + - **Default Team Settings** - Modernize page to antd (consistent with rest of app) - [PR #23614](https://github.com/BerriAI/litellm/pull/23614) - Fix: default team params (budget, duration, tpm, rpm, permissions) now correctly applied on `/team/new` - [PR #23614](https://github.com/BerriAI/litellm/pull/23614) @@ -290,6 +392,15 @@ pip install litellm==1.82.3 - Fix Public Model Hub not showing config-defined models after save - [PR #23501](https://github.com/BerriAI/litellm/pull/23501) - Fix fallback popup model dropdown z-index issue - [PR #23516](https://github.com/BerriAI/litellm/pull/23516) - Fix double-counting bug in org/team key limit checks on `/key/update` +- Fix invite link allowing multiple password resets for the same link - [PR #22462](https://github.com/BerriAI/litellm/pull/22462) +- Fix key expiry default duration not being applied when `duration` is not set - [PR #22956](https://github.com/BerriAI/litellm/pull/22956) +- Fix all proxy models not including model access groups in key creation - [PR #23236](https://github.com/BerriAI/litellm/pull/23236) +- Fix admin viewers unable to see all organizations - [PR #22940](https://github.com/BerriAI/litellm/pull/22940) +- Fix Audit Logs UI: added server-side pagination, filtering, and drawer view - [PR #22476](https://github.com/BerriAI/litellm/pull/22476) +- Fix MCP server URL and tools management issues - [PR #22751](https://github.com/BerriAI/litellm/pull/22751) +- Fix MCP server health checks triggering on server deletion - [PR #23063](https://github.com/BerriAI/litellm/pull/23063) +- Fix virtual keys in teams view not applying the team filter correctly - [PR #23065](https://github.com/BerriAI/litellm/pull/23065) +- Fix team expiry enforcement validation - [PR #22728](https://github.com/BerriAI/litellm/pull/22728) --- @@ -312,7 +423,10 @@ pip install litellm==1.82.3 ### Guardrails -No major guardrail changes in this release. +- **Guardrail mode default list** — Configure a default list of guardrail modes applied globally when no per-request mode is specified - [PR #22676](https://github.com/BerriAI/litellm/pull/22676) +- **Tag-based guardrail mode lists** — Specify a list of modes in tag-based guardrail configs instead of a single mode - [PR #23020](https://github.com/BerriAI/litellm/pull/23020) +- **Fix presidio PII token leak** — Edge case where Anthropic handle in Presidio caused PII data exposure in token response - [PR #22627](https://github.com/BerriAI/litellm/pull/22627) +- **Fix OTEL orphaned guardrail traces** — Span redundancy and missing response IDs in OpenTelemetry guardrail traces - [PR #23001](https://github.com/BerriAI/litellm/pull/23001) ### Prompt Management @@ -320,7 +434,17 @@ No major prompt management changes in this release. ### Secret Managers -No major secret manager changes in this release. +- **[Hashicorp Vault](../../docs/secret)** — Full Hashicorp Vault integration as a config override backend — secrets defined in Vault are fetched at startup and override `config.yaml` values. UI support for managing vault-sourced credentials included - [PR #22939](https://github.com/BerriAI/litellm/pull/22939), [PR #23036](https://github.com/BerriAI/litellm/pull/23036) + +--- + +## Spend Tracking + +- **Fix budget-linked keys never having spend reset** — Keys linked to budget objects were not having their spend reset on the configured reset interval - [PR #20688](https://github.com/BerriAI/litellm/pull/20688) +- **Flex pricing support** — Add `flex_pricing` field to cost map for providers that offer dynamic pricing tiers - [PR #22992](https://github.com/BerriAI/litellm/pull/22992) +- **Fix spend log cleanup** — Resolved lock tracking, integer retention, and skip-log-level issues in spend log cleanup job - [PR #22687](https://github.com/BerriAI/litellm/pull/22687) +- **Fix WebSearch spend log deduplication** — WebSearch interception was failing with thinking enabled; fixed along with spend log dedup - [PR #22679](https://github.com/BerriAI/litellm/pull/22679) +- **Fix TypeError when request has no API key** — Spend tracking was throwing unhandled exception when API key was absent from request - [PR #23363](https://github.com/BerriAI/litellm/pull/23363) --- @@ -330,6 +454,10 @@ No major secret manager changes in this release. - **Fix OOM / Prisma connection loss** on large installs — unbounded managed-object poll was exhausting Prisma connections after ~60–70 minutes on instances with 336K+ queued response rows - [PR #23472](https://github.com/BerriAI/litellm/pull/23472) - **Centralize logging kwarg updates** — root cause fix migrating all logging updates to a single function, eliminating kwarg inconsistencies across logging paths - [PR #23659](https://github.com/BerriAI/litellm/pull/23659) - **Fix tiktoken cache for non-root offline containers** — tiktoken cache now works correctly in offline environments running as non-root users - [PR #23498](https://github.com/BerriAI/litellm/pull/23498) +- **Block proxy startup when Redis transaction buffer has no Redis** — prevents silent data loss when `use_redis_transaction_buffer: true` is set without a Redis connection - [PR #23019](https://github.com/BerriAI/litellm/pull/23019) +- **Fix `InFlightRequestsMiddleware` crash** — undefined kwargs in middleware were causing request failures - [PR #22523](https://github.com/BerriAI/litellm/pull/22523) +- **Fix `BaseModelResponseIterator` crash on non-string stream chunks** — streaming was crashing when providers returned non-string chunk data - [PR #23497](https://github.com/BerriAI/litellm/pull/23497) +- **Fix `SERVER_ROOT_PATH` prefix handling** — strip prefix before checking mapped pass-through routes to prevent double-prefix issues - [PR #23414](https://github.com/BerriAI/litellm/pull/23414) - **Add CodSpeed continuous performance benchmarks** — automated performance regression tracking on CI - [PR #23676](https://github.com/BerriAI/litellm/pull/23676) --- @@ -366,12 +494,15 @@ No major secret manager changes in this release. ## Diff Summary ## 03/16/2026 -* New Providers: 5 +* New Providers: 7 * New Models / Updated Models: 116 new, 132 removed -* LLM API Endpoints: 5 -* Management Endpoints / UI: 11 -* AI Integrations: 4 -* Performance / Reliability: 5 +* LLM API Endpoints: 37 +* Management Endpoints / UI: 31 +* AI Integrations: 8 +* Guardrails: 4 +* Secret Managers: 1 +* Spend Tracking: 5 +* Performance / Reliability: 9 * Security: 3 * Database / Proxy Operations: 2 From d5ef754950671e2e8a0c17a993bd3837aa008b7c Mon Sep 17 00:00:00 2001 From: joereyna Date: Wed, 18 Mar 2026 23:35:40 -0700 Subject: [PATCH 43/66] docs(release-notes): align v1.82.3 notes with release notes guide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add MCP Gateway section (moved from Management per guide rule §11) - Rename Spend Tracking → Spend Tracking, Budgets and Rate Limiting - Fix Hashicorp Vault doc link: docs/secret → docs/secret_managers - Fix LLM API section: #### Bug Fixes → #### Bugs (matches guide) - Add Documentation Updates section (required by guide §11) - Update Diff Summary: correct section names, add MCP Gateway and Documentation Updates counts Co-Authored-By: Claude Sonnet 4.6 --- .../my-website/release_notes/v1.82.3/index.md | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/docs/my-website/release_notes/v1.82.3/index.md b/docs/my-website/release_notes/v1.82.3/index.md index a51edc60c2c..20be8826718 100644 --- a/docs/my-website/release_notes/v1.82.3/index.md +++ b/docs/my-website/release_notes/v1.82.3/index.md @@ -273,7 +273,7 @@ pip install litellm==1.82.3 - Model cost aliases expansion — define aliases in the cost map that inherit pricing from a parent model - [PR #23314](https://github.com/BerriAI/litellm/pull/23314), [PR #23457](https://github.com/BerriAI/litellm/pull/23457) - Wildcards model support for the Files API - [PR #22740](https://github.com/BerriAI/litellm/pull/22740) -#### Bug Fixes +#### Bugs - **[Anthropic](../../docs/providers/anthropic)** - Preserve native tool format (web_search, bash, tool_search, etc.) when guardrails convert tools for the Anthropic Messages API - [PR #23526](https://github.com/BerriAI/litellm/pull/23526) @@ -355,11 +355,6 @@ pip install litellm==1.82.3 - **Vector Stores** - Vector Store management endpoints — retrieve, list, update, and delete vector stores via `/v1/vector_stores/*` - [PR #23435](https://github.com/BerriAI/litellm/pull/23435) -- **MCP Servers** - - Token authentication support for MCP servers — configure `auth_type: "bearer"` per MCP server - [PR #23260](https://github.com/BerriAI/litellm/pull/23260) - - Team-scoped MCP server filtering for key creation — keys only see MCP servers available to their team - [PR #23323](https://github.com/BerriAI/litellm/pull/23323) - - Per-server health recheck in the UI - [PR #23328](https://github.com/BerriAI/litellm/pull/23328) - - **Teams** - Batch expiry setting for teams — configure a default expiry duration for all team keys - [PR #22705](https://github.com/BerriAI/litellm/pull/22705) - Team Admin can reset key spend - [PR #22725](https://github.com/BerriAI/litellm/pull/22725) @@ -397,8 +392,6 @@ pip install litellm==1.82.3 - Fix all proxy models not including model access groups in key creation - [PR #23236](https://github.com/BerriAI/litellm/pull/23236) - Fix admin viewers unable to see all organizations - [PR #22940](https://github.com/BerriAI/litellm/pull/22940) - Fix Audit Logs UI: added server-side pagination, filtering, and drawer view - [PR #22476](https://github.com/BerriAI/litellm/pull/22476) -- Fix MCP server URL and tools management issues - [PR #22751](https://github.com/BerriAI/litellm/pull/22751) -- Fix MCP server health checks triggering on server deletion - [PR #23063](https://github.com/BerriAI/litellm/pull/23063) - Fix virtual keys in teams view not applying the team filter correctly - [PR #23065](https://github.com/BerriAI/litellm/pull/23065) - Fix team expiry enforcement validation - [PR #22728](https://github.com/BerriAI/litellm/pull/22728) @@ -434,11 +427,26 @@ No major prompt management changes in this release. ### Secret Managers -- **[Hashicorp Vault](../../docs/secret)** — Full Hashicorp Vault integration as a config override backend — secrets defined in Vault are fetched at startup and override `config.yaml` values. UI support for managing vault-sourced credentials included - [PR #22939](https://github.com/BerriAI/litellm/pull/22939), [PR #23036](https://github.com/BerriAI/litellm/pull/23036) +- **[Hashicorp Vault](../../docs/secret_managers)** — Full Hashicorp Vault integration as a config override backend — secrets defined in Vault are fetched at startup and override `config.yaml` values. UI support for managing vault-sourced credentials included - [PR #22939](https://github.com/BerriAI/litellm/pull/22939), [PR #23036](https://github.com/BerriAI/litellm/pull/23036) --- -## Spend Tracking +## MCP Gateway + +#### Features + +- **Token authentication for MCP servers** — configure `auth_type: "bearer"` per MCP server to require token-based auth on tool calls - [PR #23260](https://github.com/BerriAI/litellm/pull/23260) +- **Team-scoped MCP server filtering** — keys created under a team only see MCP servers available to that team - [PR #23323](https://github.com/BerriAI/litellm/pull/23323) +- **Per-server health recheck in UI** — trigger a health check for individual MCP servers without reloading all servers - [PR #23328](https://github.com/BerriAI/litellm/pull/23328) + +#### Bugs + +- Fix MCP server URL and tools management issues causing tool discovery to fail - [PR #22751](https://github.com/BerriAI/litellm/pull/22751) +- Fix MCP server health checks triggering on server deletion - [PR #23063](https://github.com/BerriAI/litellm/pull/23063) + +--- + +## Spend Tracking, Budgets and Rate Limiting - **Fix budget-linked keys never having spend reset** — Keys linked to budget objects were not having their spend reset on the configured reset interval - [PR #20688](https://github.com/BerriAI/litellm/pull/20688) - **Flex pricing support** — Add `flex_pricing` field to cost map for providers that offer dynamic pricing tiers - [PR #22992](https://github.com/BerriAI/litellm/pull/22992) @@ -477,6 +485,16 @@ No major prompt management changes in this release. --- +## Documentation Updates + +- Add Anthropic `/v1/messages` → `/responses` parameter mapping reference - [PR #22893](https://github.com/BerriAI/litellm/pull/22893) +- Update Okta SSO docs and custom SSO handler example - [PR #22786](https://github.com/BerriAI/litellm/pull/22786) +- Add `LITELLM_MAX_BUDGET_PER_SESSION_TTL` to environment variables reference - [PR #23186](https://github.com/BerriAI/litellm/pull/23186) +- Add DB query performance guidelines to `CLAUDE.md` - [PR #23196](https://github.com/BerriAI/litellm/pull/23196) +- Add Gemini Vertex AI PayGo/priority cost tracking docs - [PR #22948](https://github.com/BerriAI/litellm/pull/22948) + +--- + ## New Contributors * @ryanh-ai made their first contribution in [PR #21542](https://github.com/BerriAI/litellm/pull/21542) @@ -499,12 +517,12 @@ No major prompt management changes in this release. * LLM API Endpoints: 37 * Management Endpoints / UI: 31 * AI Integrations: 8 -* Guardrails: 4 -* Secret Managers: 1 -* Spend Tracking: 5 -* Performance / Reliability: 9 +* MCP Gateway: 5 +* Spend Tracking, Budgets and Rate Limiting: 5 +* Performance / Loadbalancing / Reliability improvements: 9 * Security: 3 * Database / Proxy Operations: 2 +* Documentation Updates: 5 --- From 532e0d13df3b3e4532bc805d02083fa7f01980dc Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 19 Mar 2026 15:57:03 +0530 Subject: [PATCH 44/66] feat(proxy): use AZURE_DEFAULT_API_VERSION for proxy --api_version default Aligns proxy default with litellm.AZURE_DEFAULT_API_VERSION (2025-02-01-preview) so Azure response_format + json_schema works without tools fallback. Made-with: Cursor --- litellm/proxy/proxy_cli.py | 3 +- tests/test_litellm/proxy/test_proxy_cli.py | 41 ++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 97d5de0d53d..c638e294268 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -12,6 +12,7 @@ import click import httpx from dotenv import load_dotenv +import litellm from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY from litellm.secret_managers.main import get_secret_bool @@ -387,7 +388,7 @@ class ProxyInitializationHelpers: @click.option("--api_base", default=None, help="API base URL.") @click.option( "--api_version", - default="2024-07-01-preview", + default=litellm.AZURE_DEFAULT_API_VERSION, help="For azure - pass in the api version.", ) @click.option( diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index c5d6c45f9a5..349fe76ed71 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -280,6 +280,47 @@ class TestProxyInitializationHelpers: assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_called_once() + @patch("uvicorn.run") + @patch("atexit.register") + @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) + def test_proxy_default_api_version_uses_azure_default( + self, mock_should_update, mock_setup_db, mock_atexit_register, mock_uvicorn_run + ): + """Proxy default api_version should match litellm.AZURE_DEFAULT_API_VERSION for consistency.""" + from click.testing import CliRunner + + import litellm + from litellm.proxy.proxy_cli import run_server + + runner = CliRunner() + mock_proxy_module = MagicMock( + app=MagicMock(), + ProxyConfig=MagicMock(), + KeyManagementSettings=MagicMock(), + save_worker_config=MagicMock(), + ) + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} + with patch.dict(os.environ, clean_env, clear=True), patch.dict( + "sys.modules", + { + "proxy_server": mock_proxy_module, + "litellm.proxy.proxy_server": mock_proxy_module, + }, + ), patch( + "litellm.proxy.proxy_cli.ProxyInitializationHelpers._get_default_unvicorn_init_args" + ) as mock_get_args: + mock_get_args.return_value = { + "app": "litellm.proxy.proxy_server:app", + "host": "localhost", + "port": 8000, + } + result = runner.invoke(run_server, ["--local", "--skip_server_startup"]) + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" + mock_proxy_module.save_worker_config.assert_called_once() + call_kwargs = mock_proxy_module.save_worker_config.call_args[1] + assert call_kwargs["api_version"] == litellm.AZURE_DEFAULT_API_VERSION + @patch("uvicorn.run") @patch("builtins.print") def test_keepalive_timeout_flag(self, mock_print, mock_uvicorn_run): From 067dab42e6fc8a455434f05926cfae88a6638b47 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 19 Mar 2026 16:16:23 +0530 Subject: [PATCH 45/66] refactor: reduce statement count in langsmith and anthropic methods - Extract helper methods in langsmith._prepare_log_data to reduce from 51 to <50 statements - Extract helper methods in anthropic.transform_parsed_response to reduce from 57 to <50 statements - Fixes PLR0915 linter errors - All existing tests pass (10 langsmith tests, 126 anthropic tests) Made-with: Cursor --- litellm/integrations/langsmith.py | 143 +++++------ litellm/llms/anthropic/chat/transformation.py | 243 ++++++++++-------- 2 files changed, 196 insertions(+), 190 deletions(-) diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index ef2d30bb26d..479b5027ef7 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -5,7 +5,6 @@ import os import random import traceback import types -from litellm._uuid import uuid from datetime import datetime, timezone from typing import Any, Dict, List, Optional @@ -14,10 +13,11 @@ from pydantic import BaseModel # type: ignore import litellm from litellm._logging import verbose_logger +from litellm._uuid import uuid from litellm.integrations.custom_batch_logger import CustomBatchLogger from litellm.integrations.langsmith_mock_client import ( - should_use_langsmith_mock, create_mock_langsmith_client, + should_use_langsmith_mock, ) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, @@ -110,6 +110,56 @@ class LangsmithLogger(CustomBatchLogger): LANGSMITH_TENANT_ID=_credentials_tenant_id, ) + def _extract_metadata_fields( + self, metadata: dict, credentials: LangsmithCredentialsObject + ): + return { + "project_name": metadata.get("project_name", credentials["LANGSMITH_PROJECT"]), + "run_name": metadata.get("run_name", self.langsmith_default_run_name), + "run_id": metadata.get("id", metadata.get("run_id", None)), + "parent_run_id": metadata.get("parent_run_id", None), + "trace_id": metadata.get("trace_id", None), + "session_id": metadata.get("session_id", None), + "dotted_order": metadata.get("dotted_order", None), + } + + def _build_extra_metadata(self, metadata: Dict): + extra_metadata = dict(metadata) + requester_metadata = extra_metadata.get("requester_metadata") + if requester_metadata and isinstance(requester_metadata, dict): + for key in ("session_id", "thread_id", "conversation_id"): + if key in requester_metadata and key not in extra_metadata: + extra_metadata[key] = requester_metadata[key] + return extra_metadata + + def _build_outputs_with_usage(self, payload: StandardLoggingPayload) -> Dict[str, Any]: + response = payload["response"] + outputs: Dict[str, Any] + if isinstance(response, dict): + outputs = {**response} + else: + outputs = {"output": response} + outputs["usage_metadata"] = { + "input_tokens": payload.get("prompt_tokens", 0), + "output_tokens": payload.get("completion_tokens", 0), + "total_tokens": payload.get("total_tokens", 0), + "total_cost": payload.get("response_cost", 0), + } + return outputs + + def _ensure_required_ids(self, data: dict, run_id: Optional[str]): + if "id" not in data or data["id"] is None: + run_id = str(uuid.uuid4()) + data["id"] = run_id + + if "trace_id" not in data or data["trace_id"] is None: + if run_id is not None and isinstance(run_id, str): + data["trace_id"] = run_id + + if "dotted_order" not in data or data["dotted_order"] is None: + if run_id is not None and isinstance(run_id, str): + data["dotted_order"] = self.make_dot_order(run_id=run_id) + def _prepare_log_data( self, kwargs, @@ -121,56 +171,28 @@ class LangsmithLogger(CustomBatchLogger): try: _litellm_params = kwargs.get("litellm_params", {}) or {} metadata = _litellm_params.get("metadata", {}) or {} - project_name = metadata.get( - "project_name", credentials["LANGSMITH_PROJECT"] - ) - run_name = metadata.get("run_name", self.langsmith_default_run_name) - run_id = metadata.get("id", metadata.get("run_id", None)) - parent_run_id = metadata.get("parent_run_id", None) - trace_id = metadata.get("trace_id", None) - session_id = metadata.get("session_id", None) - dotted_order = metadata.get("dotted_order", None) + + fields = self._extract_metadata_fields(metadata, credentials) verbose_logger.debug( - f"Langsmith Logging - project_name: {project_name}, run_name {run_name}" + f"Langsmith Logging - project_name: {fields['project_name']}, run_name {fields['run_name']}" ) - # Ensure everything in the payload is converted to str payload: Optional[StandardLoggingPayload] = kwargs.get( "standard_logging_object", None ) - if payload is None: raise Exception("Error logging request payload. Payload=none.") - metadata = payload[ - "metadata" - ] # ensure logged metadata is json serializable - - extra_metadata = dict(metadata) - requester_metadata = extra_metadata.get("requester_metadata") - if requester_metadata and isinstance(requester_metadata, dict): - for key in ("session_id", "thread_id", "conversation_id"): - if key in requester_metadata and key not in extra_metadata: - extra_metadata[key] = requester_metadata[key] - - outputs = payload["response"] - if isinstance(outputs, dict): - outputs = {**outputs} - else: - outputs = {"output": outputs} - outputs["usage_metadata"] = { - "input_tokens": payload.get("prompt_tokens", 0), - "output_tokens": payload.get("completion_tokens", 0), - "total_tokens": payload.get("total_tokens", 0), - "total_cost": payload.get("response_cost", 0), - } + metadata = payload["metadata"] + extra_metadata = self._build_extra_metadata(dict(metadata)) + outputs = self._build_outputs_with_usage(payload) data = { - "name": run_name, - "run_type": "llm", # this should always be llm, since litellm always logs llm calls. Langsmith allow us to log "chain" + "name": fields["run_name"], + "run_type": "llm", "inputs": payload, "outputs": outputs, - "session_name": project_name, + "session_name": fields["project_name"], "start_time": payload["startTime"], "end_time": payload["endTime"], "tags": payload["request_tags"], @@ -180,46 +202,13 @@ class LangsmithLogger(CustomBatchLogger): if payload["error_str"] is not None and payload["status"] == "failure": data["error"] = payload["error_str"] - if run_id: - data["id"] = run_id - - if parent_run_id: - data["parent_run_id"] = parent_run_id - - if trace_id: - data["trace_id"] = trace_id - - if session_id: - data["session_id"] = session_id - - if dotted_order: - data["dotted_order"] = dotted_order - - run_id: Optional[str] = data.get("id") # type: ignore - if "id" not in data or data["id"] is None: - """ - for /batch langsmith requires id, trace_id and dotted_order passed as params - """ - run_id = str(uuid.uuid4()) - - data["id"] = run_id - - if ( - "trace_id" not in data - or data["trace_id"] is None - and (run_id is not None and isinstance(run_id, str)) - ): - data["trace_id"] = run_id - - if ( - "dotted_order" not in data - or data["dotted_order"] is None - and (run_id is not None and isinstance(run_id, str)) - ): - data["dotted_order"] = self.make_dot_order(run_id=run_id) # type: ignore + for key in ("id", "parent_run_id", "trace_id", "session_id", "dotted_order"): + field_key = "run_id" if key == "id" else key + if fields[field_key]: + data[key] = fields[field_key] + self._ensure_required_ids(data, fields["run_id"]) verbose_logger.debug("Langsmith Logging data on langsmith: %s", data) - return data except Exception: raise diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index f2fc4601cb9..808b68fefdc 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -50,6 +50,10 @@ from litellm.types.llms.openai import ( OpenAIMcpServerTool, OpenAIWebSearchOptions, ) +from litellm.types.responses.main import ( + OutputCodeInterpreterCall, + build_code_interpreter_log_outputs, +) from litellm.types.utils import ( CacheCreationTokenDetails, CompletionTokensDetailsWrapper, @@ -59,10 +63,6 @@ from litellm.types.utils import ( PromptTokensDetailsWrapper, ServerToolUse, ) -from litellm.types.responses.main import ( - OutputCodeInterpreterCall, - build_code_interpreter_log_outputs, -) from litellm.utils import ( ModelResponse, Usage, @@ -1684,6 +1684,85 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) return usage + def _build_code_by_id_map(self, tool_calls: List[ChatCompletionToolCallChunk]) -> Dict[str, str]: + code_by_id: Dict[str, str] = {} + for tc in tool_calls: + try: + args = json.loads(tc.get("function", {}).get("arguments", "{}")) + call_id = tc.get("id") + command = args.get("command", "") + if isinstance(call_id, str): + code_by_id[call_id] = command if isinstance(command, str) else "" + except Exception: + pass + return code_by_id + + def _build_code_interpreter_results( + self, tool_results: List[Any], code_by_id: Dict[str, str], container_id: Optional[str] + ) -> List[OutputCodeInterpreterCall]: + code_interpreter_results = [] + for tr in tool_results: + if tr.get("type") != "bash_code_execution_tool_result": + continue + call_id = tr.get("tool_use_id", "") + content = tr.get("content", {}) + log_outputs = build_code_interpreter_log_outputs(content) + code_interpreter_results.append( + OutputCodeInterpreterCall( + type="code_interpreter_call", + id=call_id, + code=code_by_id.get(call_id, ""), + container_id=container_id, + status="completed", + outputs=log_outputs, + ) + ) + return code_interpreter_results + + def _build_provider_specific_fields( + self, + completion_response: dict, + citations: Optional[List[Any]], + thinking_blocks: Optional[List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]], + web_search_results: Optional[List[Any]], + tool_results: Optional[List[Any]], + compaction_blocks: Optional[List[Any]], + tool_calls: List[ChatCompletionToolCallChunk], + ) -> Dict[str, Any]: + provider_specific_fields: Dict[str, Any] = { + "citations": citations, + "thinking_blocks": thinking_blocks, + } + + context_management = completion_response.get("context_management") + if context_management is not None: + provider_specific_fields["context_management"] = context_management + + if web_search_results is not None: + provider_specific_fields["web_search_results"] = web_search_results + + if tool_results is not None: + provider_specific_fields["tool_results"] = tool_results + container_id = ( + completion_response.get("container", {}).get("id") + if isinstance(completion_response.get("container"), dict) + else None + ) + code_by_id = self._build_code_by_id_map(tool_calls) + code_interpreter_results = self._build_code_interpreter_results( + tool_results, code_by_id, container_id + ) + provider_specific_fields["code_interpreter_results"] = code_interpreter_results + + container = completion_response.get("container") + if container is not None: + provider_specific_fields["container"] = container + + if compaction_blocks is not None: + provider_specific_fields["compaction_blocks"] = compaction_blocks + + return provider_specific_fields + def transform_parsed_response( self, completion_response: dict, @@ -1704,128 +1783,66 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): status_code=raw_response.status_code, headers=response_headers, ) - else: - text_content = "" - citations: Optional[List[Any]] = None - thinking_blocks: Optional[ - List[ - Union[ - ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock - ] - ] - ] = None - reasoning_content: Optional[str] = None - tool_calls: List[ChatCompletionToolCallChunk] = [] - ( - text_content, - citations, - thinking_blocks, - reasoning_content, - tool_calls, - web_search_results, - tool_results, - compaction_blocks, - ) = self.extract_response_content(completion_response=completion_response) + ( + text_content, + citations, + thinking_blocks, + reasoning_content, + tool_calls, + web_search_results, + tool_results, + compaction_blocks, + ) = self.extract_response_content(completion_response=completion_response) - if ( - prefix_prompt is not None - and not text_content.startswith(prefix_prompt) - and not litellm.disable_add_prefix_to_prompt - ): - text_content = prefix_prompt + text_content + if ( + prefix_prompt is not None + and not text_content.startswith(prefix_prompt) + and not litellm.disable_add_prefix_to_prompt + ): + text_content = prefix_prompt + text_content - context_management: Optional[Dict] = completion_response.get( - "context_management" - ) + provider_specific_fields = self._build_provider_specific_fields( + completion_response, + citations, + thinking_blocks, + web_search_results, + tool_results, + compaction_blocks, + tool_calls, + ) - container: Optional[Dict] = completion_response.get("container") + _message = litellm.Message( + tool_calls=tool_calls, + content=text_content or None, + provider_specific_fields=provider_specific_fields, + thinking_blocks=thinking_blocks, + reasoning_content=reasoning_content, + ) + _message.provider_specific_fields = provider_specific_fields - provider_specific_fields: Dict[str, Any] = { - "citations": citations, - "thinking_blocks": thinking_blocks, - } - if context_management is not None: - provider_specific_fields["context_management"] = context_management - if web_search_results is not None: - provider_specific_fields["web_search_results"] = web_search_results - if tool_results is not None: - provider_specific_fields["tool_results"] = tool_results - # Convert to provider-neutral OutputCodeInterpreterCall objects - # so the Responses API layer can use them without Anthropic-specific knowledge. - container_id = ( - completion_response.get("container", {}).get("id") - if isinstance(completion_response.get("container"), dict) - else None - ) - code_by_id: Dict[str, str] = {} - for tc in tool_calls: - try: - args = json.loads(tc.get("function", {}).get("arguments", "{}")) - code_by_id[tc.get("id", "")] = args.get("command", "") - except Exception: - pass - code_interpreter_results = [] - for tr in tool_results: - if tr.get("type") != "bash_code_execution_tool_result": - continue - call_id = tr.get("tool_use_id", "") - content = tr.get("content", {}) - log_outputs = build_code_interpreter_log_outputs(content) - code_interpreter_results.append( - OutputCodeInterpreterCall( - type="code_interpreter_call", - id=call_id, - code=code_by_id.get(call_id, ""), - container_id=container_id, - status="completed", - outputs=log_outputs, - ) - ) - provider_specific_fields["code_interpreter_results"] = ( - code_interpreter_results - ) - if container is not None: - provider_specific_fields["container"] = container - if compaction_blocks is not None: - provider_specific_fields["compaction_blocks"] = compaction_blocks + json_mode_message = self._transform_response_for_json_mode( + json_mode=json_mode, + tool_calls=tool_calls, + ) + if json_mode_message is not None: + completion_response["stop_reason"] = "stop" + _message = json_mode_message - _message = litellm.Message( - tool_calls=tool_calls, - content=text_content or None, - provider_specific_fields=provider_specific_fields, - thinking_blocks=thinking_blocks, - reasoning_content=reasoning_content, - ) - _message.provider_specific_fields = provider_specific_fields + model_response.choices[0].message = _message + model_response._hidden_params["original_response"] = completion_response["content"] + model_response.choices[0].finish_reason = cast( + OpenAIChatCompletionFinishReason, + map_finish_reason(completion_response["stop_reason"]), + ) - ## HANDLE JSON MODE - anthropic returns single function call - json_mode_message = self._transform_response_for_json_mode( - json_mode=json_mode, - tool_calls=tool_calls, - ) - if json_mode_message is not None: - completion_response["stop_reason"] = "stop" - _message = json_mode_message - - model_response.choices[0].message = _message # type: ignore - model_response._hidden_params["original_response"] = completion_response[ - "content" - ] # allow user to access raw anthropic tool calling response - - model_response.choices[0].finish_reason = cast( - OpenAIChatCompletionFinishReason, - map_finish_reason(completion_response["stop_reason"]), - ) - - ## CALCULATING USAGE usage = self.calculate_usage( usage_object=completion_response["usage"], reasoning_content=reasoning_content, completion_response=completion_response, speed=speed, ) - setattr(model_response, "usage", usage) # type: ignore + setattr(model_response, "usage", usage) model_response.created = int(time.time()) model_response.model = completion_response["model"] From b9564834e6eea59a85666685e78a31febbbbbcca Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 19 Mar 2026 16:18:06 +0530 Subject: [PATCH 46/66] Fix mypy errors --- litellm/llms/anthropic/chat/handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 7dce72f1e82..a2389f44295 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -897,7 +897,9 @@ class ModelResponseIterator: args = "" for block in self.content_blocks: if block["delta"]["type"] == "input_json_delta": - args += block["delta"].get("partial_json", "") + partial_json = block["delta"].get("partial_json") + if isinstance(partial_json, str): + args += partial_json if args: try: self._server_tool_inputs[ From f415b72bcfa795c3673de5d13b68658fc9a3482e Mon Sep 17 00:00:00 2001 From: Devin Petersohn Date: Wed, 18 Mar 2026 15:20:45 -0700 Subject: [PATCH 47/66] feat(anthropic): support ANTHROPIC_AUTH_TOKEN and ANTHROPIC_BASE_URL env vars Co-Authored-By: Claude Signed-off-by: Devin Petersohn --- litellm/batches/main.py | 1 + litellm/constants.py | 1 + .../llms/anthropic/batches/transformation.py | 7 +- litellm/llms/anthropic/common_utils.py | 54 ++- .../messages/transformation.py | 10 +- litellm/llms/anthropic/files/handler.py | 10 +- .../llms/anthropic/files/transformation.py | 8 +- .../llms/anthropic/skills/transformation.py | 13 +- .../llm_passthrough_endpoints.py | 4 +- litellm/utils.py | 8 +- .../anthropic/test_anthropic_common_utils.py | 372 +++++++++++++++++- 11 files changed, 441 insertions(+), 47 deletions(-) diff --git a/litellm/batches/main.py b/litellm/batches/main.py index e176dc42921..17d73aae6ad 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -524,6 +524,7 @@ def _handle_retrieve_batch_providers_without_provider_config( optional_params.api_base or litellm.api_base or get_secret_str("ANTHROPIC_API_BASE") + or get_secret_str("ANTHROPIC_BASE_URL") ) api_key = ( optional_params.api_key diff --git a/litellm/constants.py b/litellm/constants.py index 89c59ee9326..c0dd115210c 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1444,6 +1444,7 @@ SENTRY_DENYLIST = [ "credential", "OPENAI_API_KEY", "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", "AZURE_API_KEY", "COHERE_API_KEY", "REPLICATE_API_KEY", diff --git a/litellm/llms/anthropic/batches/transformation.py b/litellm/llms/anthropic/batches/transformation.py index 699f133f0f6..98c0588a091 100644 --- a/litellm/llms/anthropic/batches/transformation.py +++ b/litellm/llms/anthropic/batches/transformation.py @@ -42,9 +42,8 @@ class AnthropicBatchesConfig(BaseBatchesConfig): api_base: Optional[str] = None, ) -> dict: """Validate and prepare environment-specific headers and parameters.""" - # Resolve api_key from environment if not provided - api_key = api_key or self.anthropic_model_info.get_api_key() - if api_key is None: + auth_header = self.anthropic_model_info.get_auth_header(api_key) + if auth_header is None: raise ValueError( "Missing Anthropic API Key - A call is being made to anthropic but no key is set either in the environment variables or via params" ) @@ -52,8 +51,8 @@ class AnthropicBatchesConfig(BaseBatchesConfig): "accept": "application/json", "anthropic-version": "2023-06-01", "content-type": "application/json", - "x-api-key": api_key, } + _headers.update(auth_header) # Add beta header for message batches if "anthropic-beta" not in headers: headers["anthropic-beta"] = "message-batches-2024-09-24" diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index ac352467878..7199e210143 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -359,9 +359,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): Returns: List of beta header strings """ - from litellm.types.llms.anthropic import ( - ANTHROPIC_EFFORT_BETA_HEADER, - ) + from litellm.types.llms.anthropic import ANTHROPIC_EFFORT_BETA_HEADER betas = [] @@ -390,7 +388,8 @@ class AnthropicModelInfo(BaseLLMModelInfo): def get_anthropic_headers( self, - api_key: str, + api_key: Optional[str] = None, + auth_token: Optional[str] = None, anthropic_version: Optional[str] = None, computer_tool_used: Optional[str] = None, prompt_caching_set: bool = False, @@ -451,6 +450,8 @@ class AnthropicModelInfo(BaseLLMModelInfo): headers["authorization"] = f"Bearer {api_key}" headers["anthropic-dangerous-direct-browser-access"] = "true" betas.add(ANTHROPIC_OAUTH_BETA_HEADER) + elif auth_token and not api_key: + headers["authorization"] = f"Bearer {auth_token}" else: headers["x-api-key"] = api_key @@ -485,9 +486,13 @@ class AnthropicModelInfo(BaseLLMModelInfo): headers, api_key = optionally_handle_anthropic_oauth( headers=headers, api_key=api_key ) + # Resolve auth_token from ANTHROPIC_AUTH_TOKEN if api_key is not set + auth_token: Optional[str] = None if api_key is None: + auth_token = AnthropicModelInfo.get_auth_token() + if api_key is None and auth_token is None: raise litellm.AuthenticationError( - message="Missing Anthropic API Key - A call is being made to anthropic but no key is set either in the environment variables or via params. Please set `ANTHROPIC_API_KEY` in your environment vars", + message="Missing Anthropic API Key - A call is being made to anthropic but no key is set either in the environment variables or via params. Please set `ANTHROPIC_API_KEY` or `ANTHROPIC_AUTH_TOKEN` in your environment vars", llm_provider="anthropic", model=model, ) @@ -519,6 +524,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): prompt_caching_set=prompt_caching_set, pdf_used=pdf_used, api_key=api_key, + auth_token=auth_token, file_id_used=file_id_used, web_search_tool_used=web_search_tool_used, is_vertex_request=optional_params.get("is_vertex_request", False), @@ -543,6 +549,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): return ( api_base or get_secret_str("ANTHROPIC_API_BASE") + or get_secret_str("ANTHROPIC_BASE_URL") or "https://api.anthropic.com" ) @@ -552,6 +559,33 @@ class AnthropicModelInfo(BaseLLMModelInfo): return api_key or get_secret_str("ANTHROPIC_API_KEY") + @staticmethod + def get_auth_token(auth_token: Optional[str] = None) -> Optional[str]: + """Get auth token from ANTHROPIC_AUTH_TOKEN env var. + + Unlike api_key (which uses X-Api-Key header), auth_token uses + Authorization: Bearer header, matching the official Anthropic SDK behavior. + """ + from litellm.secret_managers.main import get_secret_str + + return auth_token or get_secret_str("ANTHROPIC_AUTH_TOKEN") + + @staticmethod + def get_auth_header(api_key: Optional[str] = None) -> Optional[dict]: + """Resolve Anthropic credentials and return the appropriate auth header dict. + + Checks ANTHROPIC_API_KEY first (-> x-api-key), then + ANTHROPIC_AUTH_TOKEN (-> Authorization: Bearer). + Returns None if neither is available. + """ + resolved_key = AnthropicModelInfo.get_api_key(api_key) + if resolved_key is not None: + return {"x-api-key": resolved_key} + auth_token = AnthropicModelInfo.get_auth_token() + if auth_token is not None: + return {"authorization": f"Bearer {auth_token}"} + return None + @staticmethod def get_base_model(model: Optional[str] = None) -> Optional[str]: return model.replace("anthropic/", "") if model else None @@ -560,14 +594,16 @@ class AnthropicModelInfo(BaseLLMModelInfo): self, api_key: Optional[str] = None, api_base: Optional[str] = None ) -> List[str]: api_base = AnthropicModelInfo.get_api_base(api_base) - api_key = AnthropicModelInfo.get_api_key(api_key) - if api_base is None or api_key is None: + auth_header = AnthropicModelInfo.get_auth_header(api_key) + if api_base is None or auth_header is None: raise ValueError( - "ANTHROPIC_API_BASE or ANTHROPIC_API_KEY is not set. Please set the environment variable, to query Anthropic's `/models` endpoint." + "ANTHROPIC_API_BASE/ANTHROPIC_BASE_URL or ANTHROPIC_API_KEY/ANTHROPIC_AUTH_TOKEN is not set. Please set the environment variable, to query Anthropic's `/models` endpoint." ) + headers = {"anthropic-version": "2023-06-01"} + headers.update(auth_header) response = litellm.module_level_client.get( url=f"{api_base}/v1/models", - headers={"x-api-key": api_key, "anthropic-version": "2023-06-01"}, + headers=headers, ) try: diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index e9ceea48220..8e7c8506278 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -142,17 +142,15 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): api_key: Optional[str] = None, api_base: Optional[str] = None, ) -> Tuple[dict, Optional[str]]: - import os - # Check for Anthropic OAuth token in Authorization header headers, api_key = optionally_handle_anthropic_oauth( headers=headers, api_key=api_key ) - if api_key is None: - api_key = os.getenv("ANTHROPIC_API_KEY") - if "x-api-key" not in headers and "authorization" not in headers and api_key: - headers["x-api-key"] = api_key + if "x-api-key" not in headers and "authorization" not in headers: + auth_header = AnthropicModelInfo.get_auth_header(api_key) + if auth_header is not None: + headers.update(auth_header) if "anthropic-version" not in headers: headers["anthropic-version"] = DEFAULT_ANTHROPIC_API_VERSION if "content-type" not in headers: diff --git a/litellm/llms/anthropic/files/handler.py b/litellm/llms/anthropic/files/handler.py index 77cc8c27316..c56799f30cf 100644 --- a/litellm/llms/anthropic/files/handler.py +++ b/litellm/llms/anthropic/files/handler.py @@ -8,10 +8,8 @@ import httpx import litellm from litellm._logging import verbose_logger from litellm._uuid import uuid -from litellm.llms.custom_httpx.http_handler import ( - get_async_httpx_client, -) from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.types.llms.openai import ( FileContentRequest, HttpxBinaryResponseContent, @@ -85,9 +83,9 @@ class AnthropicFilesHandler: # Get Anthropic API credentials api_base = self.anthropic_model_info.get_api_base(api_base) - api_key = api_key or self.anthropic_model_info.get_api_key() + auth_header = self.anthropic_model_info.get_auth_header(api_key) - if not api_key: + if auth_header is None: raise ValueError("Missing Anthropic API Key") # Construct the Anthropic batch results URL @@ -97,8 +95,8 @@ class AnthropicFilesHandler: headers = { "accept": "application/json", "anthropic-version": "2023-06-01", - "x-api-key": api_key, } + headers.update(auth_header) # Make the request to Anthropic async_client = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC) diff --git a/litellm/llms/anthropic/files/transformation.py b/litellm/llms/anthropic/files/transformation.py index 98a548a1369..0545cefb071 100644 --- a/litellm/llms/anthropic/files/transformation.py +++ b/litellm/llms/anthropic/files/transformation.py @@ -94,14 +94,14 @@ class AnthropicFilesConfig(BaseFilesConfig): api_key: Optional[str] = None, api_base: Optional[str] = None, ) -> dict: - api_key = AnthropicModelInfo.get_api_key(api_key) - if not api_key: + auth_header = AnthropicModelInfo.get_auth_header(api_key) + if auth_header is None: raise ValueError( - "Anthropic API key is required. Set ANTHROPIC_API_KEY environment variable or pass api_key parameter." + "Anthropic API key is required. Set ANTHROPIC_API_KEY or ANTHROPIC_AUTH_TOKEN environment variable or pass api_key parameter." ) headers.update( { - "x-api-key": api_key, + **auth_header, "anthropic-version": "2023-06-01", "anthropic-beta": ANTHROPIC_FILES_BETA_HEADER, } diff --git a/litellm/llms/anthropic/skills/transformation.py b/litellm/llms/anthropic/skills/transformation.py index af9863534ed..f582aefd81d 100644 --- a/litellm/llms/anthropic/skills/transformation.py +++ b/litellm/llms/anthropic/skills/transformation.py @@ -35,17 +35,16 @@ class AnthropicSkillsConfig(BaseSkillsAPIConfig): """Add Anthropic-specific headers""" from litellm.llms.anthropic.common_utils import AnthropicModelInfo - # Get API key + # Get API key from litellm_params if available api_key = None - if litellm_params: + if litellm_params is not None: api_key = litellm_params.api_key - api_key = AnthropicModelInfo.get_api_key(api_key) - if not api_key: - raise ValueError("ANTHROPIC_API_KEY is required for Skills API") + auth_header = AnthropicModelInfo.get_auth_header(api_key) + if auth_header is None: + raise ValueError("ANTHROPIC_API_KEY or ANTHROPIC_AUTH_TOKEN is required for Skills API") - # Add required headers - headers["x-api-key"] = api_key + headers.update(auth_header) headers["anthropic-version"] = "2023-06-01" # Add beta header for skills API diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 4e3e04a8474..f6b83a68ad5 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -585,7 +585,7 @@ async def anthropic_proxy_route( """ [Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion) """ - base_target_url = os.getenv("ANTHROPIC_API_BASE") or "https://api.anthropic.com" + base_target_url = os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com" encoded_endpoint = httpx.URL(endpoint).path # Ensure endpoint starts with '/' for proper URL construction @@ -609,7 +609,7 @@ async def anthropic_proxy_route( endpoint_func = create_pass_through_route( endpoint=endpoint, target=str(updated_url), - custom_headers={"x-api-key": "{}".format(anthropic_api_key)}, + custom_headers={"x-api-key": "{}".format(anthropic_api_key)} if anthropic_api_key else {}, _forward_headers=True, is_streaming_request=is_streaming_request, ) # dynamically construct pass-through endpoint based on incoming path diff --git a/litellm/utils.py b/litellm/utils.py index 81d749ab821..77625809bf0 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6160,7 +6160,7 @@ def validate_environment( # noqa: PLR0915 ["AZURE_API_BASE", "AZURE_API_VERSION", "AZURE_API_KEY"] ) elif custom_llm_provider == "anthropic": - if "ANTHROPIC_API_KEY" in os.environ: + if "ANTHROPIC_API_KEY" in os.environ or "ANTHROPIC_AUTH_TOKEN" in os.environ: keys_in_environment = True else: missing_keys.append("ANTHROPIC_API_KEY") @@ -6399,7 +6399,7 @@ def validate_environment( # noqa: PLR0915 missing_keys.append("OPENAI_API_KEY") ## anthropic elif model in litellm.anthropic_models: - if "ANTHROPIC_API_KEY" in os.environ: + if "ANTHROPIC_API_KEY" in os.environ or "ANTHROPIC_AUTH_TOKEN" in os.environ: keys_in_environment = True else: missing_keys.append("ANTHROPIC_API_KEY") @@ -8593,9 +8593,7 @@ class ProviderConfigManager: return ManusFilesConfig() elif LlmProviders.ANTHROPIC == provider: - from litellm.llms.anthropic.files.transformation import ( - AnthropicFilesConfig, - ) + from litellm.llms.anthropic.files.transformation import AnthropicFilesConfig return AnthropicFilesConfig() return None diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index b4f2629f8f4..fcec155bae0 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -1,20 +1,27 @@ """ -Tests for Anthropic OAuth token handling in common_utils. +Tests for Anthropic authentication and environment variable handling in common_utils. -Verifies that OAuth tokens (sk-ant-oat*) are sent via Authorization: Bearer -instead of x-api-key, per Anthropic's OAuth specification. +Verifies that: +- OAuth tokens (sk-ant-oat*) produce Authorization: Bearer headers with OAuth beta flags. +- Regular API keys produce x-api-key headers. +- ANTHROPIC_AUTH_TOKEN produces Authorization: Bearer headers, + matching the official Anthropic SDK behavior. +- ANTHROPIC_BASE_URL is used as a fallback for base URL resolution. +- ANTHROPIC_API_KEY / ANTHROPIC_API_BASE take precedence over their aliases. """ import os import sys +from unittest.mock import patch sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) ) -# Fake OAuth token for testing (not a real secret) +# Fake tokens for testing (not real secrets) FAKE_OAUTH_TOKEN = "sk-ant-oat01-fake-token-for-testing-123456789abcdef" FAKE_REGULAR_KEY = "sk-ant-api03-regular-key-for-testing-123456789" +FAKE_AUTH_TOKEN = "sk-ant-aut01-fake-auth-token-for-testing-123456789" class TestOptionallyHandleAnthropicOAuth: @@ -697,3 +704,360 @@ class TestProxyOAuthHeaderForwarding: assert cleaned["authorization"] == oauth_token # Proxy key must be stripped assert "x-litellm-api-key" not in cleaned + + +class TestGetAnthropicHeadersWithAuthToken: + """Tests for get_anthropic_headers with auth_token parameter.""" + + def test_auth_token_uses_bearer_header(self): + """auth_token should produce Authorization: Bearer header.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config = AnthropicModelInfo() + headers = config.get_anthropic_headers( + api_key=None, + auth_token=FAKE_AUTH_TOKEN, + computer_tool_used=False, + prompt_caching_set=False, + pdf_used=False, + is_vertex_request=False, + ) + + assert headers["authorization"] == f"Bearer {FAKE_AUTH_TOKEN}" + assert "x-api-key" not in headers + # auth_token should NOT set OAuth-specific flags + assert "anthropic-dangerous-direct-browser-access" not in headers + + def test_auth_token_includes_standard_headers(self): + """auth_token path should include standard Anthropic headers.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config = AnthropicModelInfo() + headers = config.get_anthropic_headers( + api_key=None, + auth_token=FAKE_AUTH_TOKEN, + computer_tool_used=False, + prompt_caching_set=False, + pdf_used=False, + is_vertex_request=False, + ) + + assert headers["anthropic-version"] == "2023-06-01" + assert headers["accept"] == "application/json" + assert headers["content-type"] == "application/json" + + def test_api_key_takes_precedence_over_auth_token(self): + """When both api_key and auth_token are provided, api_key wins.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config = AnthropicModelInfo() + headers = config.get_anthropic_headers( + api_key=FAKE_REGULAR_KEY, + auth_token=FAKE_AUTH_TOKEN, + computer_tool_used=False, + prompt_caching_set=False, + pdf_used=False, + is_vertex_request=False, + ) + + assert headers["x-api-key"] == FAKE_REGULAR_KEY + assert "authorization" not in headers + + +class TestValidateEnvironmentAuthToken: + """Tests for validate_environment with auth_token resolution.""" + + def test_auth_token_env_var_produces_bearer_header(self): + """validate_environment should use Bearer auth when only ANTHROPIC_AUTH_TOKEN is set.""" + from unittest.mock import patch as mock_patch + + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config = AnthropicModelInfo() + with mock_patch.dict( + "os.environ", + {"ANTHROPIC_AUTH_TOKEN": FAKE_AUTH_TOKEN}, + clear=True, + ): + headers = config.validate_environment( + headers={}, + model="claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Hello"}], + optional_params={}, + litellm_params={}, + api_key=None, + api_base=None, + ) + + assert headers["authorization"] == f"Bearer {FAKE_AUTH_TOKEN}" + assert "x-api-key" not in headers + assert "anthropic-dangerous-direct-browser-access" not in headers + + def test_api_key_param_takes_precedence_over_auth_token_env_var(self): + """validate_environment should prefer explicit api_key over ANTHROPIC_AUTH_TOKEN.""" + from unittest.mock import patch as mock_patch + + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config = AnthropicModelInfo() + with mock_patch.dict( + "os.environ", + {"ANTHROPIC_AUTH_TOKEN": FAKE_AUTH_TOKEN}, + clear=True, + ): + headers = config.validate_environment( + headers={}, + model="claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Hello"}], + optional_params={}, + litellm_params={}, + api_key=FAKE_REGULAR_KEY, + api_base=None, + ) + + assert headers["x-api-key"] == FAKE_REGULAR_KEY + assert "authorization" not in headers + + def test_raises_when_no_credentials(self): + """validate_environment should raise when neither API key nor auth token is available.""" + from unittest.mock import patch as mock_patch + + import pytest + + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config = AnthropicModelInfo() + with mock_patch.dict("os.environ", {}, clear=True): + with pytest.raises( + Exception, match="ANTHROPIC_API_KEY.*ANTHROPIC_AUTH_TOKEN" + ): + config.validate_environment( + headers={}, + model="claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Hello"}], + optional_params={}, + litellm_params={}, + api_key=None, + api_base=None, + ) + + + + +class TestGetAuthToken: + """Tests for AnthropicModelInfo.get_auth_token() static method.""" + + def test_returns_env_var_value(self): + """get_auth_token returns the ANTHROPIC_AUTH_TOKEN env var value.""" + from unittest.mock import patch as mock_patch + + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + with mock_patch.dict( + "os.environ", {"ANTHROPIC_AUTH_TOKEN": FAKE_AUTH_TOKEN}, clear=True + ): + assert AnthropicModelInfo.get_auth_token() == FAKE_AUTH_TOKEN + + def test_returns_none_when_not_set(self): + """get_auth_token returns None when ANTHROPIC_AUTH_TOKEN is not set.""" + from unittest.mock import patch as mock_patch + + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + with mock_patch.dict("os.environ", {}, clear=True): + assert AnthropicModelInfo.get_auth_token() is None + + def test_explicit_param_takes_precedence(self): + """Explicit auth_token param takes precedence over env var.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + explicit_token = "sk-ant-aut01-explicit-token-override-123456789" + assert AnthropicModelInfo.get_auth_token(explicit_token) == explicit_token + + +class TestGetAuthHeader: + """Tests for AnthropicModelInfo.get_auth_header() centralized helper.""" + + def test_returns_x_api_key_when_api_key_provided(self): + """Explicit api_key param should return x-api-key header.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + result = AnthropicModelInfo.get_auth_header(api_key=FAKE_REGULAR_KEY) + assert result == {"x-api-key": FAKE_REGULAR_KEY} + + def test_returns_x_api_key_from_env(self): + """ANTHROPIC_API_KEY env var should return x-api-key header.""" + from unittest.mock import patch as mock_patch + + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + with mock_patch.dict( + "os.environ", + {"ANTHROPIC_API_KEY": FAKE_REGULAR_KEY}, + clear=True, + ): + result = AnthropicModelInfo.get_auth_header() + assert result == {"x-api-key": FAKE_REGULAR_KEY} + + def test_returns_bearer_from_auth_token_env(self): + """ANTHROPIC_AUTH_TOKEN env var should return Authorization: Bearer header.""" + from unittest.mock import patch as mock_patch + + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + with mock_patch.dict( + "os.environ", + {"ANTHROPIC_AUTH_TOKEN": FAKE_AUTH_TOKEN}, + clear=True, + ): + result = AnthropicModelInfo.get_auth_header() + assert result == {"authorization": f"Bearer {FAKE_AUTH_TOKEN}"} + + def test_api_key_takes_precedence_over_auth_token(self): + """ANTHROPIC_API_KEY should take precedence over ANTHROPIC_AUTH_TOKEN.""" + from unittest.mock import patch as mock_patch + + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + with mock_patch.dict( + "os.environ", + { + "ANTHROPIC_API_KEY": FAKE_REGULAR_KEY, + "ANTHROPIC_AUTH_TOKEN": FAKE_AUTH_TOKEN, + }, + clear=True, + ): + result = AnthropicModelInfo.get_auth_header() + assert result == {"x-api-key": FAKE_REGULAR_KEY} + + def test_explicit_api_key_overrides_env_auth_token(self): + """Explicit api_key param should override ANTHROPIC_AUTH_TOKEN env var.""" + from unittest.mock import patch as mock_patch + + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + with mock_patch.dict( + "os.environ", + {"ANTHROPIC_AUTH_TOKEN": FAKE_AUTH_TOKEN}, + clear=True, + ): + result = AnthropicModelInfo.get_auth_header(api_key=FAKE_REGULAR_KEY) + assert result == {"x-api-key": FAKE_REGULAR_KEY} + + def test_returns_none_when_no_credentials(self): + """Should return None when neither api_key nor auth_token is available.""" + from unittest.mock import patch as mock_patch + + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + with mock_patch.dict("os.environ", {}, clear=True): + result = AnthropicModelInfo.get_auth_header() + assert result is None + + +class TestGetApiBaseFallbackChain: + """Tests for AnthropicModelInfo.get_api_base() fallback to ANTHROPIC_BASE_URL.""" + + def test_explicit_param_takes_precedence(self): + """Explicit api_base param takes precedence over all env vars.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + assert ( + AnthropicModelInfo.get_api_base("https://explicit.example.com") + == "https://explicit.example.com" + ) + + def test_defaults_to_anthropic_api(self): + """get_api_base returns the default Anthropic API base when no env vars are set.""" + from unittest.mock import patch as mock_patch + + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + with mock_patch.dict("os.environ", {}, clear=True): + assert AnthropicModelInfo.get_api_base() == "https://api.anthropic.com" + + def test_api_base_env_preferred_over_base_url_env(self): + """ANTHROPIC_API_BASE takes precedence over ANTHROPIC_BASE_URL.""" + from unittest.mock import patch as mock_patch + + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + with mock_patch.dict( + "os.environ", + { + "ANTHROPIC_API_BASE": "https://api-base.example.com", + "ANTHROPIC_BASE_URL": "https://base-url.example.com", + }, + clear=True, + ): + assert AnthropicModelInfo.get_api_base() == "https://api-base.example.com" + + def test_falls_back_to_base_url_env(self): + """get_api_base falls back to ANTHROPIC_BASE_URL when ANTHROPIC_API_BASE is not set.""" + from unittest.mock import patch as mock_patch + + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + with mock_patch.dict( + "os.environ", + {"ANTHROPIC_BASE_URL": "https://base-url.example.com"}, + clear=True, + ): + assert AnthropicModelInfo.get_api_base() == "https://base-url.example.com" + + +class TestPassthroughAuthToken: + """Tests for passthrough messages endpoint with ANTHROPIC_AUTH_TOKEN.""" + + def test_passthrough_auth_token_uses_bearer_header(self): + """Passthrough endpoint should use Bearer auth when only ANTHROPIC_AUTH_TOKEN is set.""" + from unittest.mock import patch as mock_patch + + from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, + ) + + config = AnthropicMessagesConfig() + with mock_patch.dict( + "os.environ", {"ANTHROPIC_AUTH_TOKEN": FAKE_AUTH_TOKEN}, clear=True + ): + updated_headers, _ = config.validate_anthropic_messages_environment( + headers={}, + model="claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Hello"}], + optional_params={}, + litellm_params={}, + api_key=None, + api_base=None, + ) + + assert updated_headers["authorization"] == f"Bearer {FAKE_AUTH_TOKEN}" + assert "x-api-key" not in updated_headers + assert "anthropic-dangerous-direct-browser-access" not in updated_headers + + def test_passthrough_api_key_takes_precedence(self): + """Passthrough endpoint should prefer ANTHROPIC_API_KEY over ANTHROPIC_AUTH_TOKEN.""" + from unittest.mock import patch as mock_patch + + from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, + ) + + config = AnthropicMessagesConfig() + with mock_patch.dict( + "os.environ", + {"ANTHROPIC_API_KEY": FAKE_REGULAR_KEY, "ANTHROPIC_AUTH_TOKEN": FAKE_AUTH_TOKEN}, + clear=True, + ): + updated_headers, _ = config.validate_anthropic_messages_environment( + headers={}, + model="claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Hello"}], + optional_params={}, + litellm_params={}, + api_key=None, + api_base=None, + ) + + assert updated_headers["x-api-key"] == FAKE_REGULAR_KEY + assert "authorization" not in updated_headers From 41d9ecfebc217dc1359d4d8f56b65ccec618ff7f Mon Sep 17 00:00:00 2001 From: Devin Petersohn Date: Thu, 19 Mar 2026 10:36:04 -0700 Subject: [PATCH 48/66] Address feedback Co-Authored-By: Claude --- litellm/llms/anthropic/common_utils.py | 2 +- .../proxy/pass_through_endpoints/llm_passthrough_endpoints.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 7199e210143..5b7937208af 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -452,7 +452,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): betas.add(ANTHROPIC_OAUTH_BETA_HEADER) elif auth_token and not api_key: headers["authorization"] = f"Bearer {auth_token}" - else: + elif api_key: headers["x-api-key"] = api_key if user_anthropic_beta_headers is not None: diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index f6b83a68ad5..96a7b5a27ae 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -22,6 +22,7 @@ from litellm.constants import ( ALLOWED_VERTEX_AI_PASSTHROUGH_HEADERS, BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES, ) +from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.proxy._types import * from litellm.proxy.auth.route_checks import RouteChecks @@ -606,10 +607,11 @@ async def anthropic_proxy_route( is_streaming_request = await is_streaming_request_fn(request) ## CREATE PASS-THROUGH + auth_header = AnthropicModelInfo.get_auth_header(anthropic_api_key or None) endpoint_func = create_pass_through_route( endpoint=endpoint, target=str(updated_url), - custom_headers={"x-api-key": "{}".format(anthropic_api_key)} if anthropic_api_key else {}, + custom_headers=auth_header if auth_header is not None else {}, _forward_headers=True, is_streaming_request=is_streaming_request, ) # dynamically construct pass-through endpoint based on incoming path From b7e2269942883d9f26bbf21c2140840514b498a7 Mon Sep 17 00:00:00 2001 From: Devin Petersohn Date: Thu, 19 Mar 2026 12:17:00 -0700 Subject: [PATCH 49/66] Address review feedback: fix OAuth routing in get_auth_header and self-contained validate_environment Co-Authored-By: Claude --- litellm/llms/anthropic/common_utils.py | 3 ++ .../anthropic/test_anthropic_common_utils.py | 46 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 5b7937208af..7d2d0a74961 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -486,6 +486,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): headers, api_key = optionally_handle_anthropic_oauth( headers=headers, api_key=api_key ) + api_key = AnthropicModelInfo.get_api_key(api_key) # Resolve auth_token from ANTHROPIC_AUTH_TOKEN if api_key is not set auth_token: Optional[str] = None if api_key is None: @@ -580,6 +581,8 @@ class AnthropicModelInfo(BaseLLMModelInfo): """ resolved_key = AnthropicModelInfo.get_api_key(api_key) if resolved_key is not None: + if is_anthropic_oauth_key(resolved_key): + return {"authorization": f"Bearer {resolved_key}"} return {"x-api-key": resolved_key} auth_token = AnthropicModelInfo.get_auth_token() if auth_token is not None: diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index fcec155bae0..dd5c49258ee 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -841,6 +841,31 @@ class TestValidateEnvironmentAuthToken: api_base=None, ) + def test_resolves_api_key_from_env_when_param_is_none(self): + """validate_environment should resolve ANTHROPIC_API_KEY from env when api_key param is None.""" + from unittest.mock import patch as mock_patch + + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config = AnthropicModelInfo() + with mock_patch.dict( + "os.environ", + {"ANTHROPIC_API_KEY": FAKE_REGULAR_KEY}, + clear=True, + ): + headers = config.validate_environment( + headers={}, + model="claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Hello"}], + optional_params={}, + litellm_params={}, + api_key=None, + api_base=None, + ) + + assert headers["x-api-key"] == FAKE_REGULAR_KEY + assert "authorization" not in headers + @@ -954,6 +979,27 @@ class TestGetAuthHeader: result = AnthropicModelInfo.get_auth_header() assert result is None + def test_oauth_token_uses_bearer_not_x_api_key(self): + """OAuth token (sk-ant-oat*) should return Authorization: Bearer, not x-api-key.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + result = AnthropicModelInfo.get_auth_header(api_key=FAKE_OAUTH_TOKEN) + assert result == {"authorization": f"Bearer {FAKE_OAUTH_TOKEN}"} + + def test_oauth_token_from_env_uses_bearer(self): + """OAuth token in ANTHROPIC_API_KEY env var should return Authorization: Bearer.""" + from unittest.mock import patch as mock_patch + + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + with mock_patch.dict( + "os.environ", + {"ANTHROPIC_API_KEY": FAKE_OAUTH_TOKEN}, + clear=True, + ): + result = AnthropicModelInfo.get_auth_header() + assert result == {"authorization": f"Bearer {FAKE_OAUTH_TOKEN}"} + class TestGetApiBaseFallbackChain: """Tests for AnthropicModelInfo.get_api_base() fallback to ANTHROPIC_BASE_URL.""" From c2b8ba8b1ba0832d34ed7432f3eb870e749e3c54 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 19 Mar 2026 12:28:56 -0700 Subject: [PATCH 50/66] [Fix] Resolve mypy errors in key_management_endpoints.py Add None guard for prisma_client before calling update_data, and add "unblocked" to AUDIT_ACTIONS literal type. Co-Authored-By: Claude Opus 4.6 --- litellm/proxy/_types.py | 2 +- litellm/proxy/management_endpoints/key_management_endpoints.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 9e86680e355..f5568757a22 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2955,7 +2955,7 @@ class LiteLLM_ErrorLogs(LiteLLMPydanticObjectBase): endTime: Union[str, datetime, None] -AUDIT_ACTIONS = Literal["created", "updated", "deleted", "blocked", "rotated"] +AUDIT_ACTIONS = Literal["created", "updated", "deleted", "blocked", "unblocked", "rotated"] class LiteLLM_AuditLogs(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 7be1a5a5e0c..5e573d0bbe2 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -2150,6 +2150,8 @@ async def update_key_fn( ) _data = {**non_default_values, "token": key} + if prisma_client is None: + raise Exception("Not connected to DB!") response = await prisma_client.update_data(token=key, data=_data) # Delete - key from cache, since it's been updated! From f784da41af57df1b241b6ea8973a5c69c04105d7 Mon Sep 17 00:00:00 2001 From: Devin Petersohn Date: Thu, 19 Mar 2026 12:37:12 -0700 Subject: [PATCH 51/66] Fix get_complete_url to honour ANTHROPIC_BASE_URL in experimental passthrough Co-Authored-By: Claude --- .../messages/transformation.py | 3 +-- .../anthropic/test_anthropic_common_utils.py | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 8e7c8506278..6f1a7718fc5 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -23,7 +23,6 @@ from ...common_utils import ( optionally_handle_anthropic_oauth, ) -DEFAULT_ANTHROPIC_API_BASE = "https://api.anthropic.com" DEFAULT_ANTHROPIC_API_VERSION = "2023-06-01" @@ -127,7 +126,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): litellm_params: dict, stream: Optional[bool] = None, ) -> str: - api_base = api_base or DEFAULT_ANTHROPIC_API_BASE + api_base = AnthropicModelInfo.get_api_base(api_base) if not api_base.endswith("/v1/messages"): api_base = f"{api_base}/v1/messages" return api_base diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index dd5c49258ee..22470b93540 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -1107,3 +1107,27 @@ class TestPassthroughAuthToken: assert updated_headers["x-api-key"] == FAKE_REGULAR_KEY assert "authorization" not in updated_headers + + def test_passthrough_get_complete_url_honours_base_url_env(self): + """get_complete_url should use ANTHROPIC_BASE_URL when api_base is None.""" + from unittest.mock import patch as mock_patch + + from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, + ) + + config = AnthropicMessagesConfig() + with mock_patch.dict( + "os.environ", + {"ANTHROPIC_BASE_URL": "https://custom.example.com"}, + clear=True, + ): + url = config.get_complete_url( + api_base=None, + api_key=FAKE_REGULAR_KEY, + model="claude-sonnet-4-5-20250929", + optional_params={}, + litellm_params={}, + ) + + assert url == "https://custom.example.com/v1/messages" From 004d8d01f6840ecf0b542b09293cb8c59c665787 Mon Sep 17 00:00:00 2001 From: joereyna Date: Thu, 19 Mar 2026 12:39:39 -0700 Subject: [PATCH 52/66] chore: apply black formatting to fix lint CI --- litellm/integrations/datadog/datadog.py | 4 +++- litellm/litellm_core_utils/streaming_handler.py | 12 +++--------- litellm/proxy/auth/auth_checks.py | 8 ++++++-- litellm/proxy/auth/user_api_key_auth.py | 10 ++++++---- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/litellm/integrations/datadog/datadog.py b/litellm/integrations/datadog/datadog.py index da7e84a0254..4de3644b581 100644 --- a/litellm/integrations/datadog/datadog.py +++ b/litellm/integrations/datadog/datadog.py @@ -442,7 +442,9 @@ class DataDogLogger( verbose_logger.debug("Datadog: Logger - Logging payload = %s", json_payload) dd_payload = DatadogPayload( ddsource=get_datadog_source(), - ddtags=",".join(get_datadog_tags(standard_logging_object=standard_logging_object)), + ddtags=",".join( + get_datadog_tags(standard_logging_object=standard_logging_object) + ), hostname=get_datadog_hostname(), message=json_payload, service=get_datadog_service(), diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index ca78e72c693..370b53244b0 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1894,12 +1894,8 @@ class CustomStreamWrapper: getattr(complete_streaming_response, "usage"), ) try: - _cache_copy = complete_streaming_response.model_copy( - deep=True - ) - _log_copy = complete_streaming_response.model_copy( - deep=True - ) + _cache_copy = complete_streaming_response.model_copy(deep=True) + _log_copy = complete_streaming_response.model_copy(deep=True) except RuntimeError: _cache_copy = complete_streaming_response.model_copy() _log_copy = complete_streaming_response.model_copy() @@ -2122,9 +2118,7 @@ class CustomStreamWrapper: getattr(complete_streaming_response, "usage"), ) try: - _copy = complete_streaming_response.model_copy( - deep=True - ) + _copy = complete_streaming_response.model_copy(deep=True) except RuntimeError: _copy = complete_streaming_response.model_copy() asyncio.create_task( diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 6cf1f7ed6b0..1aa14fff574 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -413,7 +413,9 @@ async def common_checks( # noqa: PLR0915 model=_model, team_object=team_object, llm_router=llm_router, - team_model_aliases=valid_token.team_model_aliases if valid_token else None, + team_model_aliases=valid_token.team_model_aliases + if valid_token + else None, ): raise ProxyException( message=f"Team not allowed to access model. Team={team_object.team_id}, Model={_model}. Allowed team models = {team_object.models}", @@ -482,7 +484,9 @@ async def common_checks( # noqa: PLR0915 ) # 3.1. If organization is in budget - with tracer.trace("litellm.proxy.auth.common_checks.organization_max_budget_check"): + with tracer.trace( + "litellm.proxy.auth.common_checks.organization_max_budget_check" + ): await _organization_max_budget_check( valid_token=valid_token, team_object=team_object, diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 30e59f77e6e..eb6a5bdb994 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -1438,10 +1438,12 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 ): # user set proxy max budget cache_key = "{}:spend".format(litellm_proxy_admin_name) with tracer.trace("litellm.proxy.auth.get_global_proxy_spend"): - global_proxy_spend = await _fetch_global_spend_with_event_coordination( - cache_key=cache_key, - user_api_key_cache=user_api_key_cache, - prisma_client=prisma_client, + global_proxy_spend = ( + await _fetch_global_spend_with_event_coordination( + cache_key=cache_key, + user_api_key_cache=user_api_key_cache, + prisma_client=prisma_client, + ) ) if global_proxy_spend is not None: From cf6369770310844fc5855b8a47820f7f1fabf0bc Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 19 Mar 2026 12:41:11 -0700 Subject: [PATCH 53/66] [Fix] Update tests for _get_and_validate_existing_key refactor Tests were outdated after _get_and_validate_existing_key was refactored to use prisma_client.db.litellm_verificationtoken.find_unique() instead of prisma_client.get_data(), and to raise ProxyException instead of HTTPException. Also fix bulk_update_keys error handler to extract ProxyException.message (str(ProxyException) returns empty string). Co-Authored-By: Claude Opus 4.6 --- .../key_management_endpoints.py | 2 + .../test_key_management_endpoints.py | 233 ++++++++++-------- 2 files changed, 128 insertions(+), 107 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 5e573d0bbe2..a550af31e71 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -2324,6 +2324,8 @@ async def bulk_update_keys( error_message = error_detail.get("error", str(e)) else: error_message = str(error_detail) + elif isinstance(e, ProxyException): + error_message = e.message else: error_message = str(e) diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 48a1f7936c5..37551fe3871 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -5108,6 +5108,8 @@ async def test_get_and_validate_existing_key(): """ from fastapi import HTTPException + from litellm.proxy.utils import ProxyException + # Test Case 1: Successfully retrieve existing key mock_prisma_client = AsyncMock() mock_key = LiteLLM_VerificationToken( @@ -5116,31 +5118,37 @@ async def test_get_and_validate_existing_key(): models=["gpt-4"], team_id=None, ) - mock_prisma_client.get_data = AsyncMock(return_value=mock_key) - - result = await _get_and_validate_existing_key( - token="test-key-123", - prisma_client=mock_prisma_client, - ) - - assert result == mock_key - mock_prisma_client.get_data.assert_called_once_with( - token="test-key-123", - table_name="key", - query_type="find_unique", - ) - - # Test Case 2: Key not found raises HTTPException - mock_prisma_client.get_data = AsyncMock(return_value=None) - - with pytest.raises(HTTPException) as exc_info: - await _get_and_validate_existing_key( - token="non-existent-key", + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=mock_key) + + with patch( + "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", + return_value="hashed-test-key-123", + ): + result = await _get_and_validate_existing_key( + token="test-key-123", prisma_client=mock_prisma_client, ) - - assert exc_info.value.status_code == 404 - assert "Key not found" in str(exc_info.value.detail) + + assert result == mock_key + mock_prisma_client.db.litellm_verificationtoken.find_unique.assert_called_once_with( + where={"token": "hashed-test-key-123"} + ) + + # Test Case 2: Key not found raises ProxyException + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None) + + with patch( + "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", + return_value="hashed-non-existent-key", + ): + with pytest.raises(ProxyException) as exc_info: + await _get_and_validate_existing_key( + token="non-existent-key", + prisma_client=mock_prisma_client, + ) + + assert str(exc_info.value.code) == "404" + assert "Key not found" in str(exc_info.value.message) # Test Case 3: Database not connected raises HTTPException with pytest.raises(HTTPException) as exc_info: @@ -5189,75 +5197,80 @@ async def test_process_single_key_update(): "tags": ["production"], } - mock_prisma_client.get_data = AsyncMock(return_value=existing_key) + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=existing_key) mock_updated_key_obj = MagicMock() mock_updated_key_obj.model_dump.return_value = updated_key_data mock_prisma_client.update_data = AsyncMock( return_value={"data": mock_updated_key_obj} ) - + # Mock prepare_key_update_data with patch( "litellm.proxy.management_endpoints.key_management_endpoints.prepare_key_update_data" ) as mock_prepare: mock_prepare.return_value = {"max_budget": 100.0, "tags": ["production"]} - + # Mock TeamMemberPermissionChecks with patch( "litellm.proxy.management_endpoints.key_management_endpoints.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint" ) as mock_permission_check: mock_permission_check.return_value = None - + # Mock _delete_cache_key_object with patch( "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object" ) as mock_delete_cache: mock_delete_cache.return_value = None - + # Mock hash_token (imported from litellm.proxy._types) with patch( "litellm.proxy._types.hash_token" ) as mock_hash: mock_hash.return_value = "hashed-test-key-123" - - # Mock KeyManagementEventHooks + + # Mock _hash_token_if_needed with patch( - "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" + "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", + return_value="hashed-test-key-123", ): - # Create update request - key_update_item = BulkUpdateKeyRequestItem( - key="test-key-123", - max_budget=100.0, - tags=["production"], - ) - - user_api_key_dict = UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, - api_key="sk-admin", - user_id="admin-user", - ) - - # Call the function - result = await _process_single_key_update( - key_update_item=key_update_item, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=None, - prisma_client=mock_prisma_client, - user_api_key_cache=mock_user_api_key_cache, - proxy_logging_obj=mock_proxy_logging_obj, - llm_router=mock_llm_router, - ) - - # Verify results - assert result is not None - assert "token" not in result # Token should be removed - assert result.get("max_budget") == 100.0 - assert result.get("tags") == ["production"] - - # Verify mocks were called - mock_prisma_client.get_data.assert_called_once() - mock_prisma_client.update_data.assert_called_once() - mock_delete_cache.assert_called_once() + # Mock KeyManagementEventHooks + with patch( + "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" + ): + # Create update request + key_update_item = BulkUpdateKeyRequestItem( + key="test-key-123", + max_budget=100.0, + tags=["production"], + ) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-admin", + user_id="admin-user", + ) + + # Call the function + result = await _process_single_key_update( + key_update_item=key_update_item, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + prisma_client=mock_prisma_client, + user_api_key_cache=mock_user_api_key_cache, + proxy_logging_obj=mock_proxy_logging_obj, + llm_router=mock_llm_router, + ) + + # Verify results + assert result is not None + assert "token" not in result # Token should be removed + assert result.get("max_budget") == 100.0 + assert result.get("tags") == ["production"] + + # Verify mocks were called + mock_prisma_client.db.litellm_verificationtoken.find_unique.assert_called_once() + mock_prisma_client.update_data.assert_called_once() + mock_delete_cache.assert_called_once() @pytest.mark.asyncio @@ -5447,15 +5460,17 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): } # First key exists, second key doesn't exist - mock_prisma_client.get_data = AsyncMock( + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( side_effect=[existing_key_1, None] # Second key not found ) + # get_data is used by the error handler to fetch key info for failed updates + mock_prisma_client.get_data = AsyncMock(return_value=None) mock_updated_key_1_obj = MagicMock() mock_updated_key_1_obj.model_dump.return_value = updated_key_1_data mock_prisma_client.update_data = AsyncMock( return_value={"data": mock_updated_key_1_obj} ) - + # Patch dependencies monkeypatch.setattr( "litellm.proxy.proxy_server.prisma_client", mock_prisma_client @@ -5467,13 +5482,13 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging_obj ) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", mock_llm_router) - + # Mock helper functions with patch( "litellm.proxy.management_endpoints.key_management_endpoints.prepare_key_update_data" ) as mock_prepare: mock_prepare.return_value = {"max_budget": 100.0, "tags": ["production"]} - + with patch( "litellm.proxy.management_endpoints.key_management_endpoints.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint" ): @@ -5484,46 +5499,50 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): "litellm.proxy._types.hash_token" ) as mock_hash: mock_hash.return_value = "hashed-key-1" - + with patch( - "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" + "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", + side_effect=lambda token: f"hashed-{token}", ): - # Create request with one valid and one invalid key - request_data = BulkUpdateKeyRequest( - keys=[ - BulkUpdateKeyRequestItem( - key="test-key-1", - max_budget=100.0, - tags=["production"], - ), - BulkUpdateKeyRequestItem( - key="non-existent-key", - max_budget=200.0, - tags=["staging"], - ), - ] - ) - - user_api_key_dict = UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, - api_key="sk-admin", - user_id="admin-user", - ) - - # Call endpoint - response = await bulk_update_keys( - data=request_data, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=None, - ) - - # Verify response - assert response.total_requested == 2 - assert len(response.successful_updates) == 1 - assert len(response.failed_updates) == 1 - assert response.successful_updates[0].key == "test-key-1" - assert response.failed_updates[0].key == "non-existent-key" - assert "Key not found" in response.failed_updates[0].failed_reason + with patch( + "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" + ): + # Create request with one valid and one invalid key + request_data = BulkUpdateKeyRequest( + keys=[ + BulkUpdateKeyRequestItem( + key="test-key-1", + max_budget=100.0, + tags=["production"], + ), + BulkUpdateKeyRequestItem( + key="non-existent-key", + max_budget=200.0, + tags=["staging"], + ), + ] + ) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-admin", + user_id="admin-user", + ) + + # Call endpoint + response = await bulk_update_keys( + data=request_data, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + + # Verify response + assert response.total_requested == 2 + assert len(response.successful_updates) == 1 + assert len(response.failed_updates) == 1 + assert response.successful_updates[0].key == "test-key-1" + assert response.failed_updates[0].key == "non-existent-key" + assert "Key not found" in response.failed_updates[0].failed_reason @pytest.mark.parametrize( From e86ca7f34d41d764d885fc64ce86dc97508035f8 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 19 Mar 2026 14:32:14 -0700 Subject: [PATCH 54/66] Revert "[Fix] Update tests for _get_and_validate_existing_key refactor" This reverts commit cf6369770310844fc5855b8a47820f7f1fabf0bc. --- .../key_management_endpoints.py | 2 - .../test_key_management_endpoints.py | 233 ++++++++---------- 2 files changed, 107 insertions(+), 128 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index a550af31e71..5e573d0bbe2 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -2324,8 +2324,6 @@ async def bulk_update_keys( error_message = error_detail.get("error", str(e)) else: error_message = str(error_detail) - elif isinstance(e, ProxyException): - error_message = e.message else: error_message = str(e) diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 37551fe3871..48a1f7936c5 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -5108,8 +5108,6 @@ async def test_get_and_validate_existing_key(): """ from fastapi import HTTPException - from litellm.proxy.utils import ProxyException - # Test Case 1: Successfully retrieve existing key mock_prisma_client = AsyncMock() mock_key = LiteLLM_VerificationToken( @@ -5118,37 +5116,31 @@ async def test_get_and_validate_existing_key(): models=["gpt-4"], team_id=None, ) - mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=mock_key) - - with patch( - "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", - return_value="hashed-test-key-123", - ): - result = await _get_and_validate_existing_key( - token="test-key-123", + mock_prisma_client.get_data = AsyncMock(return_value=mock_key) + + result = await _get_and_validate_existing_key( + token="test-key-123", + prisma_client=mock_prisma_client, + ) + + assert result == mock_key + mock_prisma_client.get_data.assert_called_once_with( + token="test-key-123", + table_name="key", + query_type="find_unique", + ) + + # Test Case 2: Key not found raises HTTPException + mock_prisma_client.get_data = AsyncMock(return_value=None) + + with pytest.raises(HTTPException) as exc_info: + await _get_and_validate_existing_key( + token="non-existent-key", prisma_client=mock_prisma_client, ) - - assert result == mock_key - mock_prisma_client.db.litellm_verificationtoken.find_unique.assert_called_once_with( - where={"token": "hashed-test-key-123"} - ) - - # Test Case 2: Key not found raises ProxyException - mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None) - - with patch( - "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", - return_value="hashed-non-existent-key", - ): - with pytest.raises(ProxyException) as exc_info: - await _get_and_validate_existing_key( - token="non-existent-key", - prisma_client=mock_prisma_client, - ) - - assert str(exc_info.value.code) == "404" - assert "Key not found" in str(exc_info.value.message) + + assert exc_info.value.status_code == 404 + assert "Key not found" in str(exc_info.value.detail) # Test Case 3: Database not connected raises HTTPException with pytest.raises(HTTPException) as exc_info: @@ -5197,80 +5189,75 @@ async def test_process_single_key_update(): "tags": ["production"], } - mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=existing_key) + mock_prisma_client.get_data = AsyncMock(return_value=existing_key) mock_updated_key_obj = MagicMock() mock_updated_key_obj.model_dump.return_value = updated_key_data mock_prisma_client.update_data = AsyncMock( return_value={"data": mock_updated_key_obj} ) - + # Mock prepare_key_update_data with patch( "litellm.proxy.management_endpoints.key_management_endpoints.prepare_key_update_data" ) as mock_prepare: mock_prepare.return_value = {"max_budget": 100.0, "tags": ["production"]} - + # Mock TeamMemberPermissionChecks with patch( "litellm.proxy.management_endpoints.key_management_endpoints.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint" ) as mock_permission_check: mock_permission_check.return_value = None - + # Mock _delete_cache_key_object with patch( "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object" ) as mock_delete_cache: mock_delete_cache.return_value = None - + # Mock hash_token (imported from litellm.proxy._types) with patch( "litellm.proxy._types.hash_token" ) as mock_hash: mock_hash.return_value = "hashed-test-key-123" - - # Mock _hash_token_if_needed + + # Mock KeyManagementEventHooks with patch( - "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", - return_value="hashed-test-key-123", + "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" ): - # Mock KeyManagementEventHooks - with patch( - "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" - ): - # Create update request - key_update_item = BulkUpdateKeyRequestItem( - key="test-key-123", - max_budget=100.0, - tags=["production"], - ) - - user_api_key_dict = UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, - api_key="sk-admin", - user_id="admin-user", - ) - - # Call the function - result = await _process_single_key_update( - key_update_item=key_update_item, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=None, - prisma_client=mock_prisma_client, - user_api_key_cache=mock_user_api_key_cache, - proxy_logging_obj=mock_proxy_logging_obj, - llm_router=mock_llm_router, - ) - - # Verify results - assert result is not None - assert "token" not in result # Token should be removed - assert result.get("max_budget") == 100.0 - assert result.get("tags") == ["production"] - - # Verify mocks were called - mock_prisma_client.db.litellm_verificationtoken.find_unique.assert_called_once() - mock_prisma_client.update_data.assert_called_once() - mock_delete_cache.assert_called_once() + # Create update request + key_update_item = BulkUpdateKeyRequestItem( + key="test-key-123", + max_budget=100.0, + tags=["production"], + ) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-admin", + user_id="admin-user", + ) + + # Call the function + result = await _process_single_key_update( + key_update_item=key_update_item, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + prisma_client=mock_prisma_client, + user_api_key_cache=mock_user_api_key_cache, + proxy_logging_obj=mock_proxy_logging_obj, + llm_router=mock_llm_router, + ) + + # Verify results + assert result is not None + assert "token" not in result # Token should be removed + assert result.get("max_budget") == 100.0 + assert result.get("tags") == ["production"] + + # Verify mocks were called + mock_prisma_client.get_data.assert_called_once() + mock_prisma_client.update_data.assert_called_once() + mock_delete_cache.assert_called_once() @pytest.mark.asyncio @@ -5460,17 +5447,15 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): } # First key exists, second key doesn't exist - mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + mock_prisma_client.get_data = AsyncMock( side_effect=[existing_key_1, None] # Second key not found ) - # get_data is used by the error handler to fetch key info for failed updates - mock_prisma_client.get_data = AsyncMock(return_value=None) mock_updated_key_1_obj = MagicMock() mock_updated_key_1_obj.model_dump.return_value = updated_key_1_data mock_prisma_client.update_data = AsyncMock( return_value={"data": mock_updated_key_1_obj} ) - + # Patch dependencies monkeypatch.setattr( "litellm.proxy.proxy_server.prisma_client", mock_prisma_client @@ -5482,13 +5467,13 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging_obj ) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", mock_llm_router) - + # Mock helper functions with patch( "litellm.proxy.management_endpoints.key_management_endpoints.prepare_key_update_data" ) as mock_prepare: mock_prepare.return_value = {"max_budget": 100.0, "tags": ["production"]} - + with patch( "litellm.proxy.management_endpoints.key_management_endpoints.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint" ): @@ -5499,50 +5484,46 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): "litellm.proxy._types.hash_token" ) as mock_hash: mock_hash.return_value = "hashed-key-1" - + with patch( - "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", - side_effect=lambda token: f"hashed-{token}", + "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" ): - with patch( - "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" - ): - # Create request with one valid and one invalid key - request_data = BulkUpdateKeyRequest( - keys=[ - BulkUpdateKeyRequestItem( - key="test-key-1", - max_budget=100.0, - tags=["production"], - ), - BulkUpdateKeyRequestItem( - key="non-existent-key", - max_budget=200.0, - tags=["staging"], - ), - ] - ) - - user_api_key_dict = UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, - api_key="sk-admin", - user_id="admin-user", - ) - - # Call endpoint - response = await bulk_update_keys( - data=request_data, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=None, - ) - - # Verify response - assert response.total_requested == 2 - assert len(response.successful_updates) == 1 - assert len(response.failed_updates) == 1 - assert response.successful_updates[0].key == "test-key-1" - assert response.failed_updates[0].key == "non-existent-key" - assert "Key not found" in response.failed_updates[0].failed_reason + # Create request with one valid and one invalid key + request_data = BulkUpdateKeyRequest( + keys=[ + BulkUpdateKeyRequestItem( + key="test-key-1", + max_budget=100.0, + tags=["production"], + ), + BulkUpdateKeyRequestItem( + key="non-existent-key", + max_budget=200.0, + tags=["staging"], + ), + ] + ) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-admin", + user_id="admin-user", + ) + + # Call endpoint + response = await bulk_update_keys( + data=request_data, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + + # Verify response + assert response.total_requested == 2 + assert len(response.successful_updates) == 1 + assert len(response.failed_updates) == 1 + assert response.successful_updates[0].key == "test-key-1" + assert response.failed_updates[0].key == "non-existent-key" + assert "Key not found" in response.failed_updates[0].failed_reason @pytest.mark.parametrize( From d118bf48188fb72ebd23f625df5a72189a052114 Mon Sep 17 00:00:00 2001 From: joereyna Date: Thu, 19 Mar 2026 14:36:02 -0700 Subject: [PATCH 55/66] chore: add poetry check --lock to lint CI to prevent stale lockfile merges --- .github/workflows/test-linting.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index fc0f84a20d4..4cedb8b5bae 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -28,9 +28,12 @@ jobs: find . -type d -name "__pycache__" -exec rm -rf {} + || true find . -name "*.pyc" -delete || true + - name: Check poetry.lock is up to date + run: | + poetry check --lock || (echo "❌ poetry.lock is out of sync with pyproject.toml. Run 'poetry lock' locally and commit the result." && exit 1) + - name: Install dependencies run: | - poetry lock poetry install --with dev - name: Check Black formatting From 05620c87e38b7cc03a9483db7f334c5b8aa2eb0c Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 19 Mar 2026 15:34:44 -0700 Subject: [PATCH 56/66] [Fix] Update bulk key update tests for find_unique refactor Tests were outdated after _get_and_validate_existing_key was refactored to use prisma_client.db.litellm_verificationtoken.find_unique() and ProxyException. Also add ProxyException handling in bulk_update_keys error extractor so error messages aren't empty. Co-Authored-By: Claude Opus 4.6 --- .../key_management_endpoints.py | 2 + .../test_key_management_endpoints.py | 331 ++++++++++-------- 2 files changed, 182 insertions(+), 151 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 5e573d0bbe2..a550af31e71 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -2324,6 +2324,8 @@ async def bulk_update_keys( error_message = error_detail.get("error", str(e)) else: error_message = str(error_detail) + elif isinstance(e, ProxyException): + error_message = e.message else: error_message = str(e) diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 48a1f7936c5..12ec79d3e0b 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -5100,14 +5100,16 @@ async def test_validate_max_budget(): async def test_get_and_validate_existing_key(): """ Test _get_and_validate_existing_key helper function. - + Tests: 1. Successfully retrieve existing key - 2. Key not found raises HTTPException + 2. Key not found raises ProxyException 3. Database not connected raises HTTPException """ from fastapi import HTTPException + from litellm.proxy._types import ProxyException + # Test Case 1: Successfully retrieve existing key mock_prisma_client = AsyncMock() mock_key = LiteLLM_VerificationToken( @@ -5116,39 +5118,49 @@ async def test_get_and_validate_existing_key(): models=["gpt-4"], team_id=None, ) - mock_prisma_client.get_data = AsyncMock(return_value=mock_key) - - result = await _get_and_validate_existing_key( - token="test-key-123", - prisma_client=mock_prisma_client, + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=mock_key ) - - assert result == mock_key - mock_prisma_client.get_data.assert_called_once_with( - token="test-key-123", - table_name="key", - query_type="find_unique", - ) - - # Test Case 2: Key not found raises HTTPException - mock_prisma_client.get_data = AsyncMock(return_value=None) - - with pytest.raises(HTTPException) as exc_info: - await _get_and_validate_existing_key( - token="non-existent-key", + + with patch( + "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", + return_value="hashed-test-key-123", + ): + result = await _get_and_validate_existing_key( + token="test-key-123", prisma_client=mock_prisma_client, ) - - assert exc_info.value.status_code == 404 - assert "Key not found" in str(exc_info.value.detail) - + + assert result == mock_key + mock_prisma_client.db.litellm_verificationtoken.find_unique.assert_called_once_with( + where={"token": "hashed-test-key-123"} + ) + + # Test Case 2: Key not found raises ProxyException + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=None + ) + + with patch( + "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", + return_value="hashed-non-existent-key", + ): + with pytest.raises(ProxyException) as exc_info: + await _get_and_validate_existing_key( + token="non-existent-key", + prisma_client=mock_prisma_client, + ) + + assert str(exc_info.value.code) == "404" + assert "Key not found" in exc_info.value.message + # Test Case 3: Database not connected raises HTTPException with pytest.raises(HTTPException) as exc_info: await _get_and_validate_existing_key( token="test-key-123", prisma_client=None, ) - + assert exc_info.value.status_code == 500 assert "Database not connected" in str(exc_info.value.detail) @@ -5189,75 +5201,82 @@ async def test_process_single_key_update(): "tags": ["production"], } - mock_prisma_client.get_data = AsyncMock(return_value=existing_key) + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=existing_key + ) mock_updated_key_obj = MagicMock() mock_updated_key_obj.model_dump.return_value = updated_key_data mock_prisma_client.update_data = AsyncMock( return_value={"data": mock_updated_key_obj} ) - + # Mock prepare_key_update_data with patch( "litellm.proxy.management_endpoints.key_management_endpoints.prepare_key_update_data" ) as mock_prepare: mock_prepare.return_value = {"max_budget": 100.0, "tags": ["production"]} - + # Mock TeamMemberPermissionChecks with patch( "litellm.proxy.management_endpoints.key_management_endpoints.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint" ) as mock_permission_check: mock_permission_check.return_value = None - + # Mock _delete_cache_key_object with patch( "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object" ) as mock_delete_cache: mock_delete_cache.return_value = None - + # Mock hash_token (imported from litellm.proxy._types) with patch( "litellm.proxy._types.hash_token" ) as mock_hash: mock_hash.return_value = "hashed-test-key-123" - - # Mock KeyManagementEventHooks + + # Mock _hash_token_if_needed with patch( - "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" + "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", + return_value="hashed-test-key-123", ): - # Create update request - key_update_item = BulkUpdateKeyRequestItem( - key="test-key-123", - max_budget=100.0, - tags=["production"], - ) - - user_api_key_dict = UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, - api_key="sk-admin", - user_id="admin-user", - ) - - # Call the function - result = await _process_single_key_update( - key_update_item=key_update_item, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=None, - prisma_client=mock_prisma_client, - user_api_key_cache=mock_user_api_key_cache, - proxy_logging_obj=mock_proxy_logging_obj, - llm_router=mock_llm_router, - ) - - # Verify results - assert result is not None - assert "token" not in result # Token should be removed - assert result.get("max_budget") == 100.0 - assert result.get("tags") == ["production"] - - # Verify mocks were called - mock_prisma_client.get_data.assert_called_once() - mock_prisma_client.update_data.assert_called_once() - mock_delete_cache.assert_called_once() + # Mock KeyManagementEventHooks + with patch( + "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" + ): + # Create update request + key_update_item = BulkUpdateKeyRequestItem( + key="test-key-123", + max_budget=100.0, + tags=["production"], + ) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-admin", + user_id="admin-user", + ) + + # Call the function + result = await _process_single_key_update( + key_update_item=key_update_item, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + prisma_client=mock_prisma_client, + user_api_key_cache=mock_user_api_key_cache, + proxy_logging_obj=mock_proxy_logging_obj, + llm_router=mock_llm_router, + ) + + # Verify results + assert result is not None + assert "token" not in result # Token should be removed + assert result.get("max_budget") == 100.0 + assert result.get("tags") == ["production"] + + # Verify mocks were called + mock_prisma_client.db.litellm_verificationtoken.find_unique.assert_called_once() + mock_prisma_client.update_data.assert_called_once() + mock_delete_cache.assert_called_once() @pytest.mark.asyncio @@ -5319,7 +5338,7 @@ async def test_bulk_update_keys_success(monkeypatch): "tags": ["staging"], } - mock_prisma_client.get_data = AsyncMock( + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( side_effect=[existing_key_1, existing_key_2] ) mock_updated_key_1_obj = MagicMock() @@ -5332,7 +5351,7 @@ async def test_bulk_update_keys_success(monkeypatch): {"data": mock_updated_key_2_obj}, ] ) - + # Patch dependencies monkeypatch.setattr( "litellm.proxy.proxy_server.prisma_client", mock_prisma_client @@ -5344,7 +5363,7 @@ async def test_bulk_update_keys_success(monkeypatch): "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging_obj ) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", mock_llm_router) - + # Mock helper functions with patch( "litellm.proxy.management_endpoints.key_management_endpoints.prepare_key_update_data" @@ -5353,7 +5372,7 @@ async def test_bulk_update_keys_success(monkeypatch): {"max_budget": 100.0, "tags": ["production"]}, {"max_budget": 200.0, "tags": ["staging"]}, ] - + with patch( "litellm.proxy.management_endpoints.key_management_endpoints.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint" ): @@ -5364,45 +5383,49 @@ async def test_bulk_update_keys_success(monkeypatch): "litellm.proxy._types.hash_token" ) as mock_hash: mock_hash.side_effect = ["hashed-key-1", "hashed-key-2"] - + with patch( - "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" + "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", + side_effect=["hashed-key-1", "hashed-key-2"], ): - # Create request - request_data = BulkUpdateKeyRequest( - keys=[ - BulkUpdateKeyRequestItem( - key="test-key-1", - max_budget=100.0, - tags=["production"], - ), - BulkUpdateKeyRequestItem( - key="test-key-2", - max_budget=200.0, - tags=["staging"], - ), - ] - ) - - user_api_key_dict = UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, - api_key="sk-admin", - user_id="admin-user", - ) - - # Call endpoint - response = await bulk_update_keys( - data=request_data, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=None, - ) - - # Verify response - assert response.total_requested == 2 - assert len(response.successful_updates) == 2 - assert len(response.failed_updates) == 0 - assert response.successful_updates[0].key == "test-key-1" - assert response.successful_updates[1].key == "test-key-2" + with patch( + "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" + ): + # Create request + request_data = BulkUpdateKeyRequest( + keys=[ + BulkUpdateKeyRequestItem( + key="test-key-1", + max_budget=100.0, + tags=["production"], + ), + BulkUpdateKeyRequestItem( + key="test-key-2", + max_budget=200.0, + tags=["staging"], + ), + ] + ) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-admin", + user_id="admin-user", + ) + + # Call endpoint + response = await bulk_update_keys( + data=request_data, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + + # Verify response + assert response.total_requested == 2 + assert len(response.successful_updates) == 2 + assert len(response.failed_updates) == 0 + assert response.successful_updates[0].key == "test-key-1" + assert response.successful_updates[1].key == "test-key-2" @pytest.mark.asyncio @@ -5447,7 +5470,7 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): } # First key exists, second key doesn't exist - mock_prisma_client.get_data = AsyncMock( + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( side_effect=[existing_key_1, None] # Second key not found ) mock_updated_key_1_obj = MagicMock() @@ -5455,7 +5478,9 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): mock_prisma_client.update_data = AsyncMock( return_value={"data": mock_updated_key_1_obj} ) - + # Mock get_data for the error handler path (used to fetch key_info on failure) + mock_prisma_client.get_data = AsyncMock(return_value=None) + # Patch dependencies monkeypatch.setattr( "litellm.proxy.proxy_server.prisma_client", mock_prisma_client @@ -5467,13 +5492,13 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging_obj ) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", mock_llm_router) - + # Mock helper functions with patch( "litellm.proxy.management_endpoints.key_management_endpoints.prepare_key_update_data" ) as mock_prepare: mock_prepare.return_value = {"max_budget": 100.0, "tags": ["production"]} - + with patch( "litellm.proxy.management_endpoints.key_management_endpoints.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint" ): @@ -5484,46 +5509,50 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): "litellm.proxy._types.hash_token" ) as mock_hash: mock_hash.return_value = "hashed-key-1" - + with patch( - "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" + "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", + side_effect=["hashed-key-1", "hashed-non-existent-key"], ): - # Create request with one valid and one invalid key - request_data = BulkUpdateKeyRequest( - keys=[ - BulkUpdateKeyRequestItem( - key="test-key-1", - max_budget=100.0, - tags=["production"], - ), - BulkUpdateKeyRequestItem( - key="non-existent-key", - max_budget=200.0, - tags=["staging"], - ), - ] - ) - - user_api_key_dict = UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, - api_key="sk-admin", - user_id="admin-user", - ) - - # Call endpoint - response = await bulk_update_keys( - data=request_data, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=None, - ) - - # Verify response - assert response.total_requested == 2 - assert len(response.successful_updates) == 1 - assert len(response.failed_updates) == 1 - assert response.successful_updates[0].key == "test-key-1" - assert response.failed_updates[0].key == "non-existent-key" - assert "Key not found" in response.failed_updates[0].failed_reason + with patch( + "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" + ): + # Create request with one valid and one invalid key + request_data = BulkUpdateKeyRequest( + keys=[ + BulkUpdateKeyRequestItem( + key="test-key-1", + max_budget=100.0, + tags=["production"], + ), + BulkUpdateKeyRequestItem( + key="non-existent-key", + max_budget=200.0, + tags=["staging"], + ), + ] + ) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-admin", + user_id="admin-user", + ) + + # Call endpoint + response = await bulk_update_keys( + data=request_data, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + + # Verify response + assert response.total_requested == 2 + assert len(response.successful_updates) == 1 + assert len(response.failed_updates) == 1 + assert response.successful_updates[0].key == "test-key-1" + assert response.failed_updates[0].key == "non-existent-key" + assert "Key not found" in response.failed_updates[0].failed_reason @pytest.mark.parametrize( From f60e3cfd34f6a4523dbd4ba0d42c97343e0195f1 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 19 Mar 2026 16:29:51 -0700 Subject: [PATCH 57/66] remove returning key in error message --- .../proxy/management_endpoints/key_management_endpoints.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index a550af31e71..e1845963701 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -1670,7 +1670,7 @@ async def _get_and_validate_existing_key( if existing_key_row is None: raise ProxyException( - message=f"Key not found. Passed key={token}", + message=f"Key not found.", type=ProxyErrorTypes.not_found_error, param="key", code=status.HTTP_404_NOT_FOUND, @@ -4947,7 +4947,7 @@ async def block_key( ) if existing_record is None: raise ProxyException( - message=f"Key not found. Passed key={data.key}", + message=f"Key not found.", type=ProxyErrorTypes.not_found_error, param="key", code=status.HTTP_404_NOT_FOUND, @@ -5056,7 +5056,7 @@ async def unblock_key( ) if existing_record is None: raise ProxyException( - message=f"Key not found. Passed key={data.key}", + message=f"Key not found.", type=ProxyErrorTypes.not_found_error, param="key", code=status.HTTP_404_NOT_FOUND, From 7b600cdbfe333333a3d86ea835f44b8f00ace195 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 19 Mar 2026 16:31:50 -0700 Subject: [PATCH 58/66] linting --- .../proxy/management_endpoints/key_management_endpoints.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index e1845963701..831922ec3f9 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -1670,7 +1670,7 @@ async def _get_and_validate_existing_key( if existing_key_row is None: raise ProxyException( - message=f"Key not found.", + message="Key not found.", type=ProxyErrorTypes.not_found_error, param="key", code=status.HTTP_404_NOT_FOUND, @@ -4947,7 +4947,7 @@ async def block_key( ) if existing_record is None: raise ProxyException( - message=f"Key not found.", + message="Key not found.", type=ProxyErrorTypes.not_found_error, param="key", code=status.HTTP_404_NOT_FOUND, @@ -5056,7 +5056,7 @@ async def unblock_key( ) if existing_record is None: raise ProxyException( - message=f"Key not found.", + message="Key not found.", type=ProxyErrorTypes.not_found_error, param="key", code=status.HTTP_404_NOT_FOUND, From 6f1bac07e57e4fcd9e2b9e83716f28acfc36462b Mon Sep 17 00:00:00 2001 From: joereyna Date: Thu, 19 Mar 2026 17:11:04 -0700 Subject: [PATCH 59/66] chore: apply black formatting to proxy/_types.py to fix lint CI --- litellm/proxy/_types.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index f5568757a22..d62bfbb7d50 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2955,7 +2955,9 @@ class LiteLLM_ErrorLogs(LiteLLMPydanticObjectBase): endTime: Union[str, datetime, None] -AUDIT_ACTIONS = Literal["created", "updated", "deleted", "blocked", "unblocked", "rotated"] +AUDIT_ACTIONS = Literal[ + "created", "updated", "deleted", "blocked", "unblocked", "rotated" +] class LiteLLM_AuditLogs(LiteLLMPydanticObjectBase): From e668ca310dabf5c56f0792e97f166711445cb09c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 20 Mar 2026 00:28:21 +0000 Subject: [PATCH 60/66] docs: add LiteLLM license key environment variable instructions Added a new section to the config.yaml documentation explaining how to set the LITELLM_LICENSE environment variable for enterprise features. Co-authored-by: Krish Dholakia --- docs/my-website/docs/proxy/configs.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index 56a8b9566db..84a6fac1210 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -602,6 +602,22 @@ Since you shouldn't use 12.5, round down to **10** to leave a safety buffer. Thi - Total maximum connections: 8 workers × 10 connections = 80 connections - This stays safely under your database's 100 connection limit +## LiteLLM License Key (Enterprise) + +To enable [LiteLLM Enterprise features](https://docs.litellm.ai/docs/proxy/enterprise), set your license key as an environment variable: + +```bash +export LITELLM_LICENSE="eyJ..." +``` + +The license key is a JWT token provided when you purchase a LiteLLM Enterprise license. Once set, LiteLLM will automatically detect and activate enterprise features. + +You can also add it to your `.env` file: + +```env +LITELLM_LICENSE="eyJ..." +``` + ## Extras From 87b5039aaab7ad5cd2cfa72c160d49e470738a72 Mon Sep 17 00:00:00 2001 From: joereyna Date: Thu, 19 Mar 2026 18:11:40 -0700 Subject: [PATCH 61/66] chore: apply black formatting to fix lint CI (batch 3) --- litellm/integrations/langsmith.py | 18 +++-- litellm/litellm_core_utils/litellm_logging.py | 5 +- .../litellm_core_utils/streaming_handler.py | 2 +- litellm/llms/anthropic/chat/handler.py | 40 +++++------ litellm/llms/anthropic/chat/transformation.py | 67 ++++++++++++------- .../anthropic_passthrough_logging_handler.py | 28 ++++---- litellm/proxy/utils.py | 6 +- .../transformation.py | 6 +- litellm/responses/streaming_iterator.py | 12 ++-- litellm/types/llms/openai.py | 36 +++++----- 10 files changed, 121 insertions(+), 99 deletions(-) diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 479b5027ef7..9cc37359928 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -114,7 +114,9 @@ class LangsmithLogger(CustomBatchLogger): self, metadata: dict, credentials: LangsmithCredentialsObject ): return { - "project_name": metadata.get("project_name", credentials["LANGSMITH_PROJECT"]), + "project_name": metadata.get( + "project_name", credentials["LANGSMITH_PROJECT"] + ), "run_name": metadata.get("run_name", self.langsmith_default_run_name), "run_id": metadata.get("id", metadata.get("run_id", None)), "parent_run_id": metadata.get("parent_run_id", None), @@ -132,7 +134,9 @@ class LangsmithLogger(CustomBatchLogger): extra_metadata[key] = requester_metadata[key] return extra_metadata - def _build_outputs_with_usage(self, payload: StandardLoggingPayload) -> Dict[str, Any]: + def _build_outputs_with_usage( + self, payload: StandardLoggingPayload + ) -> Dict[str, Any]: response = payload["response"] outputs: Dict[str, Any] if isinstance(response, dict): @@ -171,7 +175,7 @@ class LangsmithLogger(CustomBatchLogger): try: _litellm_params = kwargs.get("litellm_params", {}) or {} metadata = _litellm_params.get("metadata", {}) or {} - + fields = self._extract_metadata_fields(metadata, credentials) verbose_logger.debug( f"Langsmith Logging - project_name: {fields['project_name']}, run_name {fields['run_name']}" @@ -202,7 +206,13 @@ class LangsmithLogger(CustomBatchLogger): if payload["error_str"] is not None and payload["status"] == "failure": data["error"] = payload["error_str"] - for key in ("id", "parent_run_id", "trace_id", "session_id", "dotted_order"): + for key in ( + "id", + "parent_run_id", + "trace_id", + "session_id", + "dotted_order", + ): field_key = "run_id" if key == "id" else key if fields[field_key]: data[key] = fields[field_key] diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index ca865d73da5..fea139a64b4 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3331,7 +3331,10 @@ class Logging(LiteLLMLoggingBaseClass): return result elif isinstance(result, TextCompletionResponse): return result - elif isinstance(result, (ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent)): + elif isinstance( + result, + (ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent), + ): ## return unified Usage object if isinstance(result.response.usage, ResponseAPIUsage): transformed_usage = ( diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 6e512a1e579..96e70845b28 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -279,7 +279,7 @@ class CustomStreamWrapper: model="", llm_provider="", ) - + def check_special_tokens(self, chunk: str, finish_reason: Optional[str]): """ Output parse / special tokens for sagemaker + hf streaming. diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index a2389f44295..9f2ddcae2c7 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -578,7 +578,9 @@ class ModelResponseIterator: speed=self.speed, ) - def _content_block_delta_helper(self, chunk: dict) -> Tuple[ + def _content_block_delta_helper( + self, chunk: dict + ) -> Tuple[ str, Optional[ChatCompletionToolCallChunk], List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]], @@ -803,9 +805,9 @@ class ModelResponseIterator: tool_input = content_block_start["content_block"].get( "input", {} ) - self._server_tool_inputs[self._current_server_tool_id] = ( - tool_input - ) + self._server_tool_inputs[ + self._current_server_tool_id + ] = tool_input # Include caller information if present (for programmatic tool calling) if "caller" in content_block_start["content_block"]: caller_data = content_block_start["content_block"]["caller"] @@ -826,9 +828,9 @@ class ModelResponseIterator: # Handle compaction blocks # The full content comes in content_block_start self.compaction_blocks.append(content_block_start["content_block"]) - provider_specific_fields["compaction_blocks"] = ( - self.compaction_blocks - ) + provider_specific_fields[ + "compaction_blocks" + ] = self.compaction_blocks provider_specific_fields["compaction_start"] = { "type": "compaction", "content": content_block_start["content_block"].get( @@ -850,9 +852,9 @@ class ModelResponseIterator: self.web_search_results.append( content_block_start["content_block"] ) - provider_specific_fields["web_search_results"] = ( - self.web_search_results - ) + provider_specific_fields[ + "web_search_results" + ] = self.web_search_results elif content_type == "web_fetch_tool_result": # Capture web_fetch_tool_result for multi-turn reconstruction # The full content comes in content_block_start, not in deltas @@ -860,18 +862,18 @@ class ModelResponseIterator: self.web_search_results.append( content_block_start["content_block"] ) - provider_specific_fields["web_search_results"] = ( - self.web_search_results - ) + provider_specific_fields[ + "web_search_results" + ] = self.web_search_results elif content_type != "tool_search_tool_result": # Handle other tool results (code execution, etc.) # Skip tool_search_tool_result as it's internal metadata self.tool_results.append(content_block_start["content_block"]) provider_specific_fields["tool_results"] = self.tool_results # Convert to provider-neutral code_interpreter_results - provider_specific_fields["code_interpreter_results"] = ( - self._build_code_interpreter_results() - ) + provider_specific_fields[ + "code_interpreter_results" + ] = self._build_code_interpreter_results() elif type_chunk == "content_block_stop": ContentBlockStop(**chunk) # type: ignore @@ -928,9 +930,9 @@ class ModelResponseIterator: ) if container_id and self.tool_results: self._container_id = container_id - provider_specific_fields["code_interpreter_results"] = ( - self._build_code_interpreter_results() - ) + provider_specific_fields[ + "code_interpreter_results" + ] = self._build_code_interpreter_results() elif type_chunk == "message_start": """ Anthropic diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 808b68fefdc..73d1b02c76d 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -964,11 +964,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if mcp_servers: optional_params["mcp_servers"] = mcp_servers elif param == "tool_choice" or param == "parallel_tool_calls": - _tool_choice: Optional[AnthropicMessagesToolChoice] = ( - self._map_tool_choice( - tool_choice=non_default_params.get("tool_choice"), - parallel_tool_use=non_default_params.get("parallel_tool_calls"), - ) + _tool_choice: Optional[ + AnthropicMessagesToolChoice + ] = self._map_tool_choice( + tool_choice=non_default_params.get("tool_choice"), + parallel_tool_use=non_default_params.get("parallel_tool_calls"), ) if _tool_choice is not None: @@ -1066,9 +1066,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self.map_openai_context_management_to_anthropic(value) ) if anthropic_context_management is not None: - optional_params["context_management"] = ( - anthropic_context_management - ) + optional_params[ + "context_management" + ] = anthropic_context_management elif param == "speed" and isinstance(value, str): # Pass through Anthropic-specific speed parameter for fast mode optional_params["speed"] = value @@ -1142,9 +1142,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): text=system_message_block["content"], ) if "cache_control" in system_message_block: - anthropic_system_message_content["cache_control"] = ( - system_message_block["cache_control"] - ) + anthropic_system_message_content[ + "cache_control" + ] = system_message_block["cache_control"] anthropic_system_message_list.append( anthropic_system_message_content ) @@ -1168,9 +1168,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) ) if "cache_control" in _content: - anthropic_system_message_content["cache_control"] = ( - _content["cache_control"] - ) + anthropic_system_message_content[ + "cache_control" + ] = _content["cache_control"] anthropic_system_message_list.append( anthropic_system_message_content @@ -1467,7 +1467,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) return _message - def extract_response_content(self, completion_response: dict) -> Tuple[ + def extract_response_content( + self, completion_response: dict + ) -> Tuple[ str, Optional[List[Any]], Optional[ @@ -1684,7 +1686,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) return usage - def _build_code_by_id_map(self, tool_calls: List[ChatCompletionToolCallChunk]) -> Dict[str, str]: + def _build_code_by_id_map( + self, tool_calls: List[ChatCompletionToolCallChunk] + ) -> Dict[str, str]: code_by_id: Dict[str, str] = {} for tc in tool_calls: try: @@ -1698,7 +1702,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return code_by_id def _build_code_interpreter_results( - self, tool_results: List[Any], code_by_id: Dict[str, str], container_id: Optional[str] + self, + tool_results: List[Any], + code_by_id: Dict[str, str], + container_id: Optional[str], ) -> List[OutputCodeInterpreterCall]: code_interpreter_results = [] for tr in tool_results: @@ -1723,7 +1730,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self, completion_response: dict, citations: Optional[List[Any]], - thinking_blocks: Optional[List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]], + thinking_blocks: Optional[ + List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] + ], web_search_results: Optional[List[Any]], tool_results: Optional[List[Any]], compaction_blocks: Optional[List[Any]], @@ -1733,14 +1744,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "citations": citations, "thinking_blocks": thinking_blocks, } - + context_management = completion_response.get("context_management") if context_management is not None: provider_specific_fields["context_management"] = context_management - + if web_search_results is not None: provider_specific_fields["web_search_results"] = web_search_results - + if tool_results is not None: provider_specific_fields["tool_results"] = tool_results container_id = ( @@ -1752,15 +1763,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): code_interpreter_results = self._build_code_interpreter_results( tool_results, code_by_id, container_id ) - provider_specific_fields["code_interpreter_results"] = code_interpreter_results - + provider_specific_fields[ + "code_interpreter_results" + ] = code_interpreter_results + container = completion_response.get("container") if container is not None: provider_specific_fields["container"] = container - + if compaction_blocks is not None: provider_specific_fields["compaction_blocks"] = compaction_blocks - + return provider_specific_fields def transform_parsed_response( @@ -1830,7 +1843,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): _message = json_mode_message model_response.choices[0].message = _message - model_response._hidden_params["original_response"] = completion_response["content"] + model_response._hidden_params["original_response"] = completion_response[ + "content" + ] model_response.choices[0].finish_reason = cast( OpenAIChatCompletionFinishReason, map_finish_reason(completion_response["stop_reason"]), diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index 3241c1ca93f..fcb1e0b2e49 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -6,23 +6,21 @@ import httpx import litellm from litellm._logging import verbose_proxy_logger -from litellm.litellm_core_utils.litellm_logging import \ - Logging as LiteLLMLoggingObj -from litellm.litellm_core_utils.litellm_logging import \ - use_custom_pricing_for_model +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model from litellm.llms.anthropic import get_anthropic_config -from litellm.llms.anthropic.chat.handler import \ - ModelResponseIterator as AnthropicModelResponseIterator +from litellm.llms.anthropic.chat.handler import ( + ModelResponseIterator as AnthropicModelResponseIterator, +) from litellm.proxy._types import PassThroughEndpointLoggingTypedDict from litellm.proxy.auth.auth_utils import get_end_user_id_from_request_body -from litellm.types.passthrough_endpoints.pass_through_endpoints import \ - PassthroughStandardLoggingPayload -from litellm.types.utils import (LiteLLMBatch, ModelResponse, - TextCompletionResponse) +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + PassthroughStandardLoggingPayload, +) +from litellm.types.utils import LiteLLMBatch, ModelResponse, TextCompletionResponse if TYPE_CHECKING: - from litellm.types.passthrough_endpoints.pass_through_endpoints import \ - EndpointType + from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType from ..success_handler import PassThroughEndpointLogging else: @@ -333,8 +331,7 @@ class AnthropicPassthroughLoggingHandler: import base64 from litellm._uuid import uuid - from litellm.llms.anthropic.batches.transformation import \ - AnthropicBatchesConfig + from litellm.llms.anthropic.batches.transformation import AnthropicBatchesConfig from litellm.types.utils import Choices, SpecialEnums try: @@ -550,8 +547,7 @@ class AnthropicPassthroughLoggingHandler: managed_files_hook, "store_unified_object_id" ): # Create a mock user API key dict for the managed object storage - from litellm.proxy._types import (LitellmUserRoles, - UserAPIKeyAuth) + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth user_api_key_dict = UserAPIKeyAuth( user_id=kwargs.get("user_id", "default-user"), diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 566e1ff5178..351bc23915a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1898,9 +1898,9 @@ class ProxyLogging: normalized_call_type = CallTypes.aembedding.value if normalized_call_type is not None: litellm_logging_obj.call_type = normalized_call_type - litellm_logging_obj.model_call_details["call_type"] = ( - normalized_call_type - ) + litellm_logging_obj.model_call_details[ + "call_type" + ] = normalized_call_type # Pass-through endpoints are logged via the callback loop's # async_post_call_failure_hook — skip pre_call and failure handlers. if litellm_logging_obj.call_type == CallTypes.pass_through.value: diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index cf18511bfa3..b6479a36998 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -2113,9 +2113,9 @@ class LiteLLMCompletionResponsesConfig: hasattr(completion_details, "reasoning_tokens") and completion_details.reasoning_tokens is not None ): - output_details_dict["reasoning_tokens"] = ( - completion_details.reasoning_tokens - ) + output_details_dict[ + "reasoning_tokens" + ] = completion_details.reasoning_tokens else: output_details_dict["reasoning_tokens"] = 0 diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 8a91368dd68..10a74a5b3c6 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -168,14 +168,10 @@ class BaseResponsesAPIStreamingIterator: # Store the completed response (also for incomplete/failed so logging still fires) _chunk_type = getattr(openai_responses_api_chunk, "type", None) - if ( - openai_responses_api_chunk - and _chunk_type - in ( - ResponsesAPIStreamEvents.RESPONSE_COMPLETED, - ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, - ResponsesAPIStreamEvents.RESPONSE_FAILED, - ) + if openai_responses_api_chunk and _chunk_type in ( + ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, + ResponsesAPIStreamEvents.RESPONSE_FAILED, ): self.completed_response = openai_responses_api_chunk # Add cost to usage object if include_cost_in_streaming_usage is True diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index a265198e6b8..5a80b40d61f 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -970,12 +970,12 @@ class OpenAIChatCompletionChunk(ChatCompletionChunk): class Hyperparameters(BaseModel): batch_size: Optional[Union[str, int]] = None # "Number of examples in each batch." - learning_rate_multiplier: Optional[Union[str, float]] = ( - None # Scaling factor for the learning rate - ) - n_epochs: Optional[Union[str, int]] = ( - None # "The number of epochs to train the model for" - ) + learning_rate_multiplier: Optional[ + Union[str, float] + ] = None # Scaling factor for the learning rate + n_epochs: Optional[ + Union[str, int] + ] = None # "The number of epochs to train the model for" model_config = {"extra": "allow"} @@ -1004,18 +1004,18 @@ class FineTuningJobCreate(BaseModel): model: str # "The name of the model to fine-tune." training_file: str # "The ID of an uploaded file that contains training data." - hyperparameters: Optional[Hyperparameters] = ( - None # "The hyperparameters used for the fine-tuning job." - ) - suffix: Optional[str] = ( - None # "A string of up to 18 characters that will be added to your fine-tuned model name." - ) - validation_file: Optional[str] = ( - None # "The ID of an uploaded file that contains validation data." - ) - integrations: Optional[List[str]] = ( - None # "A list of integrations to enable for your fine-tuning job." - ) + hyperparameters: Optional[ + Hyperparameters + ] = None # "The hyperparameters used for the fine-tuning job." + suffix: Optional[ + str + ] = None # "A string of up to 18 characters that will be added to your fine-tuned model name." + validation_file: Optional[ + str + ] = None # "The ID of an uploaded file that contains validation data." + integrations: Optional[ + List[str] + ] = None # "A list of integrations to enable for your fine-tuning job." seed: Optional[int] = None # "The seed controls the reproducibility of the job." From 3093ef844e8fc6c656777c7ff0b42cbd15c3aa45 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 19 Mar 2026 18:57:02 -0700 Subject: [PATCH 62/66] fix: document new config_settings.md --- docs/my-website/docs/proxy/config_settings.md | 4 +- ...odel_prices_and_context_window_backup.json | 152 ++++++++++++++++++ 2 files changed, 155 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 042af2bfb45..d7542fc2c3d 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -401,8 +401,10 @@ router_settings: | AUTH_STRATEGY | Strategy used for authentication (e.g., OAuth, API key) | AUTO_REDIRECT_UI_LOGIN_TO_SSO | Flag to enable automatic redirect of UI login page to SSO when SSO is configured. Default is **false** | AUDIO_SPEECH_CHUNK_SIZE | Chunk size for audio speech processing. Default is 1024 -| ANTHROPIC_API_KEY | API key for Anthropic service +| ANTHROPIC_API_KEY | API key for Anthropic service. Uses `x-api-key` header for authentication. +| ANTHROPIC_AUTH_TOKEN | Alternative auth token for Anthropic service. Uses `Authorization: Bearer` header instead of `x-api-key`. Used as fallback when `ANTHROPIC_API_KEY` is not set. | ANTHROPIC_API_BASE | Base URL for Anthropic API. Default is https://api.anthropic.com +| ANTHROPIC_BASE_URL | Alternative to `ANTHROPIC_API_BASE` for setting the Anthropic API base URL. Used as fallback when `ANTHROPIC_API_BASE` is not set. | ANTHROPIC_TOKEN_COUNTING_BETA_VERSION | Beta version header for Anthropic token counting API. Default is `token-counting-2024-11-01` | AWS_ACCESS_KEY_ID | Access Key ID for AWS services | AWS_BATCH_ROLE_ARN | ARN of the AWS IAM role for batch operations diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e7ff57f27e1..879dd42be47 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -37104,5 +37104,157 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true + }, + "volcengine/doubao-seed-2-0-pro-260215": { + "litellm_provider": "volcengine", + "max_input_tokens": 256000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://www.volcengine.com/docs/82379/1330310", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true, + "tiered_pricing": [ + { + "input_cost_per_token": 4.6e-07, + "output_cost_per_token": 2.3e-06, + "range": [ + 0, + 32000.0 + ] + }, + { + "input_cost_per_token": 7e-07, + "output_cost_per_token": 3.5e-06, + "range": [ + 32000.0, + 128000.0 + ] + }, + { + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 7e-06, + "range": [ + 128000.0, + 256000.0 + ] + } + ] + }, + "volcengine/doubao-seed-2-0-lite-260215": { + "litellm_provider": "volcengine", + "max_input_tokens": 256000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://www.volcengine.com/docs/82379/1330310", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true, + "tiered_pricing": [ + { + "input_cost_per_token": 8.7e-08, + "output_cost_per_token": 5.2e-07, + "range": [ + 0, + 32000.0 + ] + }, + { + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 7.8e-07, + "range": [ + 32000.0, + 128000.0 + ] + }, + { + "input_cost_per_token": 2.6e-07, + "output_cost_per_token": 1.6e-06, + "range": [ + 128000.0, + 256000.0 + ] + } + ] + }, + "volcengine/doubao-seed-2-0-mini-260215": { + "litellm_provider": "volcengine", + "max_input_tokens": 256000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://www.volcengine.com/docs/82379/1330310", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true, + "tiered_pricing": [ + { + "input_cost_per_token": 2.9e-08, + "output_cost_per_token": 2.9e-07, + "range": [ + 0, + 32000.0 + ] + }, + { + "input_cost_per_token": 5.8e-08, + "output_cost_per_token": 5.8e-07, + "range": [ + 32000.0, + 128000.0 + ] + }, + { + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 1.2e-06, + "range": [ + 128000.0, + 256000.0 + ] + } + ] + }, + "volcengine/doubao-seed-2-0-code-preview-260215": { + "litellm_provider": "volcengine", + "max_input_tokens": 256000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://www.volcengine.com/docs/82379/1330310", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true, + "tiered_pricing": [ + { + "input_cost_per_token": 4.6e-07, + "output_cost_per_token": 2.3e-06, + "range": [ + 0, + 32000.0 + ] + }, + { + "input_cost_per_token": 7e-07, + "output_cost_per_token": 3.5e-06, + "range": [ + 32000.0, + 128000.0 + ] + }, + { + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 7e-06, + "range": [ + 128000.0, + 256000.0 + ] + } + ] } } From e5dc912367721744b835e701118a89d41201ed7a Mon Sep 17 00:00:00 2001 From: joereyna Date: Thu, 19 Mar 2026 18:59:39 -0700 Subject: [PATCH 63/66] =?UTF-8?q?bump:=20version=201.82.4=20=E2=86=92=201.?= =?UTF-8?q?82.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3fb05cc8b34..73f495203bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.82.4" +version = "1.82.5" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -184,7 +184,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.82.4" +version = "1.82.5" version_files = [ "pyproject.toml:^version" ] From c6b5c070056d667db80259cc8503373a712157a1 Mon Sep 17 00:00:00 2001 From: joereyna Date: Thu, 19 Mar 2026 21:00:54 -0700 Subject: [PATCH 64/66] chore: apply black formatting and enable black pre-commit hook --- .pre-commit-config.yaml | 12 ++++++------ litellm/llms/anthropic/skills/transformation.py | 4 +++- .../llm_passthrough_endpoints.py | 6 +++++- litellm/utils.py | 10 ++++++++-- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9396f323e45..2bc361bc48f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,12 +14,12 @@ repos: types: [python] files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py exclude: ^litellm/__init__.py$ - # - id: black - # name: black - # entry: poetry run black - # language: system - # types: [python] - # files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py + - id: black + name: black + entry: poetry run black + language: system + types: [python] + files: (litellm/|litellm_proxy_extras/).*\.py - repo: https://github.com/pycqa/flake8 rev: 7.0.0 # The version of flake8 to use hooks: diff --git a/litellm/llms/anthropic/skills/transformation.py b/litellm/llms/anthropic/skills/transformation.py index f582aefd81d..a992d84d459 100644 --- a/litellm/llms/anthropic/skills/transformation.py +++ b/litellm/llms/anthropic/skills/transformation.py @@ -42,7 +42,9 @@ class AnthropicSkillsConfig(BaseSkillsAPIConfig): auth_header = AnthropicModelInfo.get_auth_header(api_key) if auth_header is None: - raise ValueError("ANTHROPIC_API_KEY or ANTHROPIC_AUTH_TOKEN is required for Skills API") + raise ValueError( + "ANTHROPIC_API_KEY or ANTHROPIC_AUTH_TOKEN is required for Skills API" + ) headers.update(auth_header) headers["anthropic-version"] = "2023-06-01" diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 96a7b5a27ae..534022cc133 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -586,7 +586,11 @@ async def anthropic_proxy_route( """ [Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion) """ - base_target_url = os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com" + base_target_url = ( + os.getenv("ANTHROPIC_API_BASE") + or os.getenv("ANTHROPIC_BASE_URL") + or "https://api.anthropic.com" + ) encoded_endpoint = httpx.URL(endpoint).path # Ensure endpoint starts with '/' for proper URL construction diff --git a/litellm/utils.py b/litellm/utils.py index 77625809bf0..c8272586dad 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6160,7 +6160,10 @@ def validate_environment( # noqa: PLR0915 ["AZURE_API_BASE", "AZURE_API_VERSION", "AZURE_API_KEY"] ) elif custom_llm_provider == "anthropic": - if "ANTHROPIC_API_KEY" in os.environ or "ANTHROPIC_AUTH_TOKEN" in os.environ: + if ( + "ANTHROPIC_API_KEY" in os.environ + or "ANTHROPIC_AUTH_TOKEN" in os.environ + ): keys_in_environment = True else: missing_keys.append("ANTHROPIC_API_KEY") @@ -6399,7 +6402,10 @@ def validate_environment( # noqa: PLR0915 missing_keys.append("OPENAI_API_KEY") ## anthropic elif model in litellm.anthropic_models: - if "ANTHROPIC_API_KEY" in os.environ or "ANTHROPIC_AUTH_TOKEN" in os.environ: + if ( + "ANTHROPIC_API_KEY" in os.environ + or "ANTHROPIC_AUTH_TOKEN" in os.environ + ): keys_in_environment = True else: missing_keys.append("ANTHROPIC_API_KEY") From ff6faacc641f1d1568ef649e6dc1438c83d4a36f Mon Sep 17 00:00:00 2001 From: joereyna Date: Thu, 19 Mar 2026 21:20:12 -0700 Subject: [PATCH 65/66] fix: resolve mypy union-attr error in anthropic messages transformation --- .../experimental_pass_through/messages/transformation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 6f1a7718fc5..2e0b58bc49a 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -126,7 +126,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): litellm_params: dict, stream: Optional[bool] = None, ) -> str: - api_base = AnthropicModelInfo.get_api_base(api_base) + api_base = AnthropicModelInfo.get_api_base(api_base) or "https://api.anthropic.com" if not api_base.endswith("/v1/messages"): api_base = f"{api_base}/v1/messages" return api_base From 6f6e23a7b0b634bb5304c2ca2fc35441fba5f905 Mon Sep 17 00:00:00 2001 From: joereyna Date: Thu, 19 Mar 2026 21:25:02 -0700 Subject: [PATCH 66/66] chore: apply black formatting to experimental_pass_through transformation --- .../experimental_pass_through/messages/transformation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 2e0b58bc49a..9b60a58260b 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -126,7 +126,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): litellm_params: dict, stream: Optional[bool] = None, ) -> str: - api_base = AnthropicModelInfo.get_api_base(api_base) or "https://api.anthropic.com" + api_base = ( + AnthropicModelInfo.get_api_base(api_base) or "https://api.anthropic.com" + ) if not api_base.endswith("/v1/messages"): api_base = f"{api_base}/v1/messages" return api_base