diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 200af3ac460..14015225f38 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2435,6 +2435,36 @@ class Logging(LiteLLMLoggingBaseClass): standard_built_in_tools_params=self.standard_built_in_tools_params, ) + # print standard logging payload + if ( + standard_logging_payload := self.model_call_details.get( + "standard_logging_object" + ) + ) is not None: + emit_standard_logging_payload(standard_logging_payload) + elif self.call_type == "pass_through_endpoint": + print_verbose( + "Async success callbacks: Got a pass-through endpoint response" + ) + + self.model_call_details["async_complete_streaming_response"] = result + + # cost calculation not possible for pass-through + self.model_call_details["response_cost"] = None + + ## STANDARDIZED LOGGING PAYLOAD + self.model_call_details[ + "standard_logging_object" + ] = get_standard_logging_object_payload( + kwargs=self.model_call_details, + init_response_obj=result, + start_time=start_time, + end_time=end_time, + logging_obj=self, + status="success", + standard_built_in_tools_params=self.standard_built_in_tools_params, + ) + # print standard logging payload if ( standard_logging_payload := self.model_call_details.get( diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 316bd49cf89..734d52918ba 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -1062,6 +1062,223 @@ def test_append_system_prompt_messages(): assert result == messages +@pytest.mark.asyncio +async def test_async_success_handler_sets_standard_logging_object_for_pass_through_endpoints(): + """ + Test that async_success_handler sets standard_logging_object for pass-through endpoints + even when complete_streaming_response is None. + + This is a regression test for the bug where pass-through endpoints (like vLLM classify) + would not set standard_logging_object, causing model_max_budget_limiter to raise + ValueError("standard_logging_payload is required"). + + The fix adds an elif branch in async_success_handler to set standard_logging_object + for pass-through endpoints when complete_streaming_response is None. + """ + from datetime import datetime + from unittest.mock import patch + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.utils import StandardPassThroughResponseObject + + # Create a logging object for a pass-through endpoint + logging_obj = LiteLLMLoggingObj( + model="unknown", + messages=[{"role": "user", "content": "test"}], + stream=False, + call_type="pass_through_endpoint", + start_time=datetime.now(), + litellm_call_id="test-call-id", + function_id="test-function-id", + ) + + # Set up model_call_details with required fields + logging_obj.model_call_details = { + "litellm_params": { + "metadata": {}, + "proxy_server_request": {}, + }, + "litellm_call_id": "test-call-id", + } + + # Create a pass-through response object (not a ModelResponse) + result = StandardPassThroughResponseObject(response='{"status": "success"}') + + start_time = datetime.now() + end_time = datetime.now() + + # Mock the callbacks to avoid actual logging + with patch.object(logging_obj, "get_combined_callback_list", return_value=[]): + # Call async_success_handler + await logging_obj.async_success_handler( + result=result, + start_time=start_time, + end_time=end_time, + cache_hit=False, + ) + + # Verify that standard_logging_object was set + assert "standard_logging_object" in logging_obj.model_call_details, ( + "standard_logging_object should be set for pass-through endpoints " + "even when complete_streaming_response is None" + ) + assert logging_obj.model_call_details["standard_logging_object"] is not None, ( + "standard_logging_object should not be None for pass-through endpoints" + ) + + # Verify that async_complete_streaming_response was set to prevent re-processing + # This is consistent with the existing code pattern for regular streaming + assert "async_complete_streaming_response" in logging_obj.model_call_details, ( + "async_complete_streaming_response should be set to prevent re-processing, " + "consistent with the existing code pattern" + ) + assert logging_obj.model_call_details["async_complete_streaming_response"] is result, ( + "async_complete_streaming_response should be set to the result" + ) + + # Verify that response_cost is set to None (cost calculation not possible for pass-through) + # This is consistent with the error handling in the non-pass-through code path + assert "response_cost" in logging_obj.model_call_details, ( + "response_cost should be set for pass-through endpoints" + ) + assert logging_obj.model_call_details["response_cost"] is None, ( + "response_cost should be None for pass-through endpoints since " + "StandardPassThroughResponseObject doesn't have standard usage info" + ) + + +@pytest.mark.asyncio +async def test_async_success_handler_prevents_reprocessing_for_pass_through_endpoints(): + """ + Test that async_success_handler prevents re-processing for pass-through endpoints + by setting async_complete_streaming_response, consistent with the existing code pattern. + + This ensures that if async_success_handler is called multiple times (e.g., during + streaming), it won't re-process the response after the first complete call. + """ + from datetime import datetime + from unittest.mock import patch + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.utils import StandardPassThroughResponseObject + + # Create a logging object for a pass-through endpoint + logging_obj = LiteLLMLoggingObj( + model="unknown", + messages=[{"role": "user", "content": "test"}], + stream=False, + call_type="pass_through_endpoint", + start_time=datetime.now(), + litellm_call_id="test-call-id-reprocess", + function_id="test-function-id-reprocess", + ) + + # Set up model_call_details with required fields + logging_obj.model_call_details = { + "litellm_params": { + "metadata": {}, + "proxy_server_request": {}, + }, + "litellm_call_id": "test-call-id-reprocess", + } + + result = StandardPassThroughResponseObject(response='{"status": "success"}') + start_time = datetime.now() + end_time = datetime.now() + + # Mock the callbacks to avoid actual logging + with patch.object(logging_obj, "get_combined_callback_list", return_value=[]): + # First call - should process and set standard_logging_object + await logging_obj.async_success_handler( + result=result, + start_time=start_time, + end_time=end_time, + cache_hit=False, + ) + + # Verify first call set the values + assert "standard_logging_object" in logging_obj.model_call_details + assert "async_complete_streaming_response" in logging_obj.model_call_details + first_standard_logging_object = logging_obj.model_call_details["standard_logging_object"] + + # Second call - should return early due to async_complete_streaming_response guard + with patch.object(logging_obj, "get_combined_callback_list", return_value=[]) as mock_callbacks: + await logging_obj.async_success_handler( + result=result, + start_time=start_time, + end_time=end_time, + cache_hit=False, + ) + # The guard should cause early return, so get_combined_callback_list should not be called + mock_callbacks.assert_not_called() + + # Verify standard_logging_object wasn't modified by second call + assert logging_obj.model_call_details["standard_logging_object"] is first_standard_logging_object, ( + "standard_logging_object should not be modified on re-processing" + ) + + +@pytest.mark.asyncio +async def test_async_success_handler_sets_standard_logging_object_for_streaming_pass_through(): + """ + Test that async_success_handler sets standard_logging_object for streaming + pass-through endpoints when the response cannot be parsed into a ModelResponse. + + This covers the case where streaming pass-through endpoints for unknown providers + return a StandardPassThroughResponseObject instead of a ModelResponse. + """ + from datetime import datetime + from unittest.mock import patch + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.utils import StandardPassThroughResponseObject + + # Create a logging object for a streaming pass-through endpoint + logging_obj = LiteLLMLoggingObj( + model="unknown", + messages=[{"role": "user", "content": "test"}], + stream=True, # Streaming request + call_type="pass_through_endpoint", + start_time=datetime.now(), + litellm_call_id="test-call-id-streaming", + function_id="test-function-id-streaming", + ) + + # Set up model_call_details with required fields + logging_obj.model_call_details = { + "litellm_params": { + "metadata": {}, + "proxy_server_request": {}, + }, + "litellm_call_id": "test-call-id-streaming", + } + + # Create a pass-through response object (simulating unparseable streaming response) + result = StandardPassThroughResponseObject( + response='data: {"chunk": 1}\ndata: {"chunk": 2}\ndata: [DONE]' + ) + + start_time = datetime.now() + end_time = datetime.now() + + # Mock the callbacks to avoid actual logging + with patch.object(logging_obj, "get_combined_callback_list", return_value=[]): + # Call async_success_handler + await logging_obj.async_success_handler( + result=result, + start_time=start_time, + end_time=end_time, + cache_hit=False, + ) + + # Verify that standard_logging_object was set + assert "standard_logging_object" in logging_obj.model_call_details, ( + "standard_logging_object should be set for streaming pass-through endpoints " + "even when the response cannot be parsed into a ModelResponse" + ) + assert logging_obj.model_call_details["standard_logging_object"] is not None, ( + "standard_logging_object should not be None for streaming pass-through endpoints" + ) def test_get_error_information_error_code_priority(): """ Test get_error_information prioritizes 'code' attribute over 'status_code' attribute