scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

fix: preserve API key metadata in streaming pass-through endpoint logs

Browse source 
When Anthropic and Vertex pass-through endpoints handled streaming responses, they passed empty kwargs={} to the logging payload creator, discarding user API key metadata (hash, alias, team_id, etc.). This prevented Langfuse traces from including key identification data for streaming requests.

Fixed by retrieving litellm_params and passthrough_logging_payload from model_call_details (populated during request setup), following the existing OpenAI handler pattern. This ensures metadata reaches Langfuse callbacks for both streaming and non-streaming requests.

Added tests to verify litellm_params with user_api_key metadata and passthrough_logging_payload are preserved in streaming kwargs.

Affects: Anthropic and Vertex pass-through streaming endpoints, Langfuse logging integration

Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
Harshit28j 2026-03-03 01:55:28 +05:30
parent 552066e56c
commit 7bf2e25f7e
3 changed files with 181 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py
View file

@ -209,10 +209,24 @@ class AnthropicPassthroughLoggingHandler:
"result": None,
"kwargs": {},
}
# Preserve existing litellm_params to maintain metadata
# (user_api_key_hash, user_api_key_alias, team_id, etc.)
existing_litellm_params = litellm_logging_obj.model_call_details.get(
"litellm_params", {}
) or {}
initial_kwargs: dict = {
"litellm_params": existing_litellm_params.copy(),
}
passthrough_logging_payload = litellm_logging_obj.model_call_details.get(
"passthrough_logging_payload"
)
if passthrough_logging_payload is not None:
initial_kwargs["passthrough_logging_payload"] = passthrough_logging_payload
kwargs = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload(
litellm_model_response=complete_streaming_response,
model=model,
kwargs={},
kwargs=initial_kwargs,
start_time=start_time,
end_time=end_time,
logging_obj=litellm_logging_obj,

9
litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py
View file

@ -341,7 +341,14 @@ class VertexPassthroughLoggingHandler:
- Creates standard logging object
- Logs in litellm callbacks
"""
kwargs: Dict[str, Any] = {}
# Preserve existing litellm_params to maintain metadata
# (user_api_key_hash, user_api_key_alias, team_id, etc.)
existing_litellm_params = litellm_logging_obj.model_call_details.get(
"litellm_params", {}
) or {}
kwargs: Dict[str, Any] = {
"litellm_params": existing_litellm_params.copy(),
}
model = model or VertexPassthroughLoggingHandler.extract_model_from_url(
url_route
)

159
tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py
View file

@ -614,4 +614,161 @@ class TestAnthropicBatchPassthroughCostTracking:
)
# Verify managed files hook was called
mock_proxy_logging_obj.get_proxy_hook.assert_called_once_with("managed_files")
mock_proxy_logging_obj.get_proxy_hook.assert_called_once_with("managed_files")
class TestAnthropicStreamingMetadataPreservation:
"""Test that metadata (user_api_key_hash, team_id, etc.) is preserved in streaming kwargs."""
def setup_method(self):
self.start_time = datetime.now()
self.end_time = datetime.now()
self.mock_chunks = [
'{"type": "message_start", "message": {"id": "msg_123", "model": "claude-3-sonnet-20240229"}}',
'{"type": "content_block_delta", "delta": {"text": "Hello"}}',
'{"type": "message_stop"}',
]
self.mock_metadata = {
"user_api_key_hash": "sk-test-hash-1234",
"user_api_key_alias": "my-test-key",
"user_api_key_team_id": "team-abc",
"user_api_key_org_id": "org-xyz",
"user_api_key_user_id": "user-123",
}
def _create_mock_logging_obj_with_metadata(self):
mock_logging_obj = MagicMock()
mock_logging_obj.model_call_details = {
"model": "claude-3-sonnet-20240229",
"litellm_params": {
"metadata": self.mock_metadata.copy(),
},
"passthrough_logging_payload": {
"url": "https://api.anthropic.com/v1/messages",
"request_body": {
"model": "claude-3-sonnet-20240229",
"messages": [{"role": "user", "content": "Hello"}],
},
},
}
mock_logging_obj.litellm_call_id = "test-call-id"
return mock_logging_obj
@patch.object(
AnthropicPassthroughLoggingHandler, "_build_complete_streaming_response"
)
@patch.object(
AnthropicPassthroughLoggingHandler,
"_create_anthropic_response_logging_payload",
)
def test_streaming_handler_passes_litellm_params_in_kwargs(
self, mock_create_payload, mock_build_response
):
"""Verify that litellm_params with metadata is passed to _create_anthropic_response_logging_payload."""
mock_build_response.return_value = MagicMock()
mock_create_payload.return_value = {
"response_cost": 0.001,
"model": "claude-3-sonnet-20240229",
"litellm_params": {"metadata": self.mock_metadata.copy()},
}
logging_obj = self._create_mock_logging_obj_with_metadata()
result = AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks(
litellm_logging_obj=logging_obj,
passthrough_success_handler_obj=MagicMock(),
url_route="/anthropic/v1/messages",
request_body={"model": "claude-3-sonnet-20240229"},
endpoint_type="messages",
start_time=self.start_time,
all_chunks=self.mock_chunks,
end_time=self.end_time,
)
# Verify _create_anthropic_response_logging_payload was called with kwargs containing litellm_params
mock_create_payload.assert_called_once()
call_kwargs = mock_create_payload.call_args[1]["kwargs"]
assert "litellm_params" in call_kwargs
assert "metadata" in call_kwargs["litellm_params"]
assert (
call_kwargs["litellm_params"]["metadata"]["user_api_key_hash"]
== "sk-test-hash-1234"
)
assert (
call_kwargs["litellm_params"]["metadata"]["user_api_key_alias"]
== "my-test-key"
)
assert (
call_kwargs["litellm_params"]["metadata"]["user_api_key_team_id"]
== "team-abc"
)
@patch.object(
AnthropicPassthroughLoggingHandler, "_build_complete_streaming_response"
)
@patch.object(
AnthropicPassthroughLoggingHandler,
"_create_anthropic_response_logging_payload",
)
def test_streaming_handler_passes_passthrough_logging_payload(
self, mock_create_payload, mock_build_response
):
"""Verify that passthrough_logging_payload is included in kwargs when present."""
mock_build_response.return_value = MagicMock()
mock_create_payload.return_value = {"response_cost": 0.001}
logging_obj = self._create_mock_logging_obj_with_metadata()
AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks(
litellm_logging_obj=logging_obj,
passthrough_success_handler_obj=MagicMock(),
url_route="/anthropic/v1/messages",
request_body={"model": "claude-3-sonnet-20240229"},
endpoint_type="messages",
start_time=self.start_time,
all_chunks=self.mock_chunks,
end_time=self.end_time,
)
call_kwargs = mock_create_payload.call_args[1]["kwargs"]
assert "passthrough_logging_payload" in call_kwargs
assert (
call_kwargs["passthrough_logging_payload"]["url"]
== "https://api.anthropic.com/v1/messages"
)
@patch.object(
AnthropicPassthroughLoggingHandler, "_build_complete_streaming_response"
)
@patch.object(
AnthropicPassthroughLoggingHandler,
"_create_anthropic_response_logging_payload",
)
def test_streaming_handler_without_passthrough_logging_payload(
self, mock_create_payload, mock_build_response
):
"""Verify kwargs still contain litellm_params even when passthrough_logging_payload is absent."""
mock_build_response.return_value = MagicMock()
mock_create_payload.return_value = {"response_cost": 0.001}
logging_obj = MagicMock()
logging_obj.model_call_details = {
"model": "claude-3-sonnet-20240229",
"litellm_params": {
"metadata": self.mock_metadata.copy(),
},
}
logging_obj.litellm_call_id = "test-call-id"
AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks(
litellm_logging_obj=logging_obj,
passthrough_success_handler_obj=MagicMock(),
url_route="/anthropic/v1/messages",
request_body={"model": "claude-3-sonnet-20240229"},
endpoint_type="messages",
start_time=self.start_time,
all_chunks=self.mock_chunks,
end_time=self.end_time,
)
call_kwargs = mock_create_payload.call_args[1]["kwargs"]
assert "litellm_params" in call_kwargs
assert "passthrough_logging_payload" not in call_kwargs