mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
fix: Set standard_logging_object for pass-through endpoints (#19887)
Pass-through endpoints (like vLLM classify) were not setting
standard_logging_object because _get_assembled_streaming_response
returns None for non-ModelResponse results.
This caused model_max_budget_limiter.async_log_success_event to raise
ValueError('standard_logging_payload is required').
The fix adds an elif branch in async_success_handler that mirrors the
non-pass-through code path.
Co-authored-by: openhands <openhands@all-hands.dev>
Co-authored-by: Krish Dholakia <krrishdholakia@gmail.com>
This commit is contained in:
Graham Neubig
GitHub
parent
8d1e579965
commit
22dd7c224e
2 changed files with 247 additions and 0 deletions
|
|
@ -2435,6 +2435,36 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
standard_built_in_tools_params=self.standard_built_in_tools_params,
|
||||
)
|
||||
|
||||
# print standard logging payload
|
||||
if (
|
||||
standard_logging_payload := self.model_call_details.get(
|
||||
"standard_logging_object"
|
||||
)
|
||||
) is not None:
|
||||
emit_standard_logging_payload(standard_logging_payload)
|
||||
elif self.call_type == "pass_through_endpoint":
|
||||
print_verbose(
|
||||
"Async success callbacks: Got a pass-through endpoint response"
|
||||
)
|
||||
|
||||
self.model_call_details["async_complete_streaming_response"] = result
|
||||
|
||||
# cost calculation not possible for pass-through
|
||||
self.model_call_details["response_cost"] = None
|
||||
|
||||
## STANDARDIZED LOGGING PAYLOAD
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = get_standard_logging_object_payload(
|
||||
kwargs=self.model_call_details,
|
||||
init_response_obj=result,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
logging_obj=self,
|
||||
status="success",
|
||||
standard_built_in_tools_params=self.standard_built_in_tools_params,
|
||||
)
|
||||
|
||||
# print standard logging payload
|
||||
if (
|
||||
standard_logging_payload := self.model_call_details.get(
|
||||
|
|
|
|||
|
|
@ -1062,6 +1062,223 @@ def test_append_system_prompt_messages():
|
|||
assert result == messages
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_success_handler_sets_standard_logging_object_for_pass_through_endpoints():
|
||||
"""
|
||||
Test that async_success_handler sets standard_logging_object for pass-through endpoints
|
||||
even when complete_streaming_response is None.
|
||||
|
||||
This is a regression test for the bug where pass-through endpoints (like vLLM classify)
|
||||
would not set standard_logging_object, causing model_max_budget_limiter to raise
|
||||
ValueError("standard_logging_payload is required").
|
||||
|
||||
The fix adds an elif branch in async_success_handler to set standard_logging_object
|
||||
for pass-through endpoints when complete_streaming_response is None.
|
||||
"""
|
||||
from datetime import datetime
|
||||
from unittest.mock import patch
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.utils import StandardPassThroughResponseObject
|
||||
|
||||
# Create a logging object for a pass-through endpoint
|
||||
logging_obj = LiteLLMLoggingObj(
|
||||
model="unknown",
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
stream=False,
|
||||
call_type="pass_through_endpoint",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="test-call-id",
|
||||
function_id="test-function-id",
|
||||
)
|
||||
|
||||
# Set up model_call_details with required fields
|
||||
logging_obj.model_call_details = {
|
||||
"litellm_params": {
|
||||
"metadata": {},
|
||||
"proxy_server_request": {},
|
||||
},
|
||||
"litellm_call_id": "test-call-id",
|
||||
}
|
||||
|
||||
# Create a pass-through response object (not a ModelResponse)
|
||||
result = StandardPassThroughResponseObject(response='{"status": "success"}')
|
||||
|
||||
start_time = datetime.now()
|
||||
end_time = datetime.now()
|
||||
|
||||
# Mock the callbacks to avoid actual logging
|
||||
with patch.object(logging_obj, "get_combined_callback_list", return_value=[]):
|
||||
# Call async_success_handler
|
||||
await logging_obj.async_success_handler(
|
||||
result=result,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
cache_hit=False,
|
||||
)
|
||||
|
||||
# Verify that standard_logging_object was set
|
||||
assert "standard_logging_object" in logging_obj.model_call_details, (
|
||||
"standard_logging_object should be set for pass-through endpoints "
|
||||
"even when complete_streaming_response is None"
|
||||
)
|
||||
assert logging_obj.model_call_details["standard_logging_object"] is not None, (
|
||||
"standard_logging_object should not be None for pass-through endpoints"
|
||||
)
|
||||
|
||||
# Verify that async_complete_streaming_response was set to prevent re-processing
|
||||
# This is consistent with the existing code pattern for regular streaming
|
||||
assert "async_complete_streaming_response" in logging_obj.model_call_details, (
|
||||
"async_complete_streaming_response should be set to prevent re-processing, "
|
||||
"consistent with the existing code pattern"
|
||||
)
|
||||
assert logging_obj.model_call_details["async_complete_streaming_response"] is result, (
|
||||
"async_complete_streaming_response should be set to the result"
|
||||
)
|
||||
|
||||
# Verify that response_cost is set to None (cost calculation not possible for pass-through)
|
||||
# This is consistent with the error handling in the non-pass-through code path
|
||||
assert "response_cost" in logging_obj.model_call_details, (
|
||||
"response_cost should be set for pass-through endpoints"
|
||||
)
|
||||
assert logging_obj.model_call_details["response_cost"] is None, (
|
||||
"response_cost should be None for pass-through endpoints since "
|
||||
"StandardPassThroughResponseObject doesn't have standard usage info"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_success_handler_prevents_reprocessing_for_pass_through_endpoints():
|
||||
"""
|
||||
Test that async_success_handler prevents re-processing for pass-through endpoints
|
||||
by setting async_complete_streaming_response, consistent with the existing code pattern.
|
||||
|
||||
This ensures that if async_success_handler is called multiple times (e.g., during
|
||||
streaming), it won't re-process the response after the first complete call.
|
||||
"""
|
||||
from datetime import datetime
|
||||
from unittest.mock import patch
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.utils import StandardPassThroughResponseObject
|
||||
|
||||
# Create a logging object for a pass-through endpoint
|
||||
logging_obj = LiteLLMLoggingObj(
|
||||
model="unknown",
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
stream=False,
|
||||
call_type="pass_through_endpoint",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="test-call-id-reprocess",
|
||||
function_id="test-function-id-reprocess",
|
||||
)
|
||||
|
||||
# Set up model_call_details with required fields
|
||||
logging_obj.model_call_details = {
|
||||
"litellm_params": {
|
||||
"metadata": {},
|
||||
"proxy_server_request": {},
|
||||
},
|
||||
"litellm_call_id": "test-call-id-reprocess",
|
||||
}
|
||||
|
||||
result = StandardPassThroughResponseObject(response='{"status": "success"}')
|
||||
start_time = datetime.now()
|
||||
end_time = datetime.now()
|
||||
|
||||
# Mock the callbacks to avoid actual logging
|
||||
with patch.object(logging_obj, "get_combined_callback_list", return_value=[]):
|
||||
# First call - should process and set standard_logging_object
|
||||
await logging_obj.async_success_handler(
|
||||
result=result,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
cache_hit=False,
|
||||
)
|
||||
|
||||
# Verify first call set the values
|
||||
assert "standard_logging_object" in logging_obj.model_call_details
|
||||
assert "async_complete_streaming_response" in logging_obj.model_call_details
|
||||
first_standard_logging_object = logging_obj.model_call_details["standard_logging_object"]
|
||||
|
||||
# Second call - should return early due to async_complete_streaming_response guard
|
||||
with patch.object(logging_obj, "get_combined_callback_list", return_value=[]) as mock_callbacks:
|
||||
await logging_obj.async_success_handler(
|
||||
result=result,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
cache_hit=False,
|
||||
)
|
||||
# The guard should cause early return, so get_combined_callback_list should not be called
|
||||
mock_callbacks.assert_not_called()
|
||||
|
||||
# Verify standard_logging_object wasn't modified by second call
|
||||
assert logging_obj.model_call_details["standard_logging_object"] is first_standard_logging_object, (
|
||||
"standard_logging_object should not be modified on re-processing"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_success_handler_sets_standard_logging_object_for_streaming_pass_through():
|
||||
"""
|
||||
Test that async_success_handler sets standard_logging_object for streaming
|
||||
pass-through endpoints when the response cannot be parsed into a ModelResponse.
|
||||
|
||||
This covers the case where streaming pass-through endpoints for unknown providers
|
||||
return a StandardPassThroughResponseObject instead of a ModelResponse.
|
||||
"""
|
||||
from datetime import datetime
|
||||
from unittest.mock import patch
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.utils import StandardPassThroughResponseObject
|
||||
|
||||
# Create a logging object for a streaming pass-through endpoint
|
||||
logging_obj = LiteLLMLoggingObj(
|
||||
model="unknown",
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
stream=True, # Streaming request
|
||||
call_type="pass_through_endpoint",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="test-call-id-streaming",
|
||||
function_id="test-function-id-streaming",
|
||||
)
|
||||
|
||||
# Set up model_call_details with required fields
|
||||
logging_obj.model_call_details = {
|
||||
"litellm_params": {
|
||||
"metadata": {},
|
||||
"proxy_server_request": {},
|
||||
},
|
||||
"litellm_call_id": "test-call-id-streaming",
|
||||
}
|
||||
|
||||
# Create a pass-through response object (simulating unparseable streaming response)
|
||||
result = StandardPassThroughResponseObject(
|
||||
response='data: {"chunk": 1}\ndata: {"chunk": 2}\ndata: [DONE]'
|
||||
)
|
||||
|
||||
start_time = datetime.now()
|
||||
end_time = datetime.now()
|
||||
|
||||
# Mock the callbacks to avoid actual logging
|
||||
with patch.object(logging_obj, "get_combined_callback_list", return_value=[]):
|
||||
# Call async_success_handler
|
||||
await logging_obj.async_success_handler(
|
||||
result=result,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
cache_hit=False,
|
||||
)
|
||||
|
||||
# Verify that standard_logging_object was set
|
||||
assert "standard_logging_object" in logging_obj.model_call_details, (
|
||||
"standard_logging_object should be set for streaming pass-through endpoints "
|
||||
"even when the response cannot be parsed into a ModelResponse"
|
||||
)
|
||||
assert logging_obj.model_call_details["standard_logging_object"] is not None, (
|
||||
"standard_logging_object should not be None for streaming pass-through endpoints"
|
||||
)
|
||||
def test_get_error_information_error_code_priority():
|
||||
"""
|
||||
Test get_error_information prioritizes 'code' attribute over 'status_code' attribute
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue