diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index cff8ced87af..68ec1587acb 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -76,6 +76,7 @@ from litellm.types.llms.openai import ( HttpxBinaryResponseContent, OpenAIFileObject, OpenAIModerationResponse, + ResponseAPIUsage, ResponseCompletedEvent, ResponsesAPIResponse, ) @@ -700,8 +701,13 @@ class Logging(LiteLLMLoggingBaseClass): vector_store_custom_logger.__class__.__name__ ) # Add to global callbacks so post-call hooks are invoked - if vector_store_custom_logger and vector_store_custom_logger not in litellm.callbacks: - litellm.logging_callback_manager.add_litellm_callback(vector_store_custom_logger) + if ( + vector_store_custom_logger + and vector_store_custom_logger not in litellm.callbacks + ): + litellm.logging_callback_manager.add_litellm_callback( + vector_store_custom_logger + ) return vector_store_custom_logger return None @@ -1206,8 +1212,6 @@ class Logging(LiteLLMLoggingBaseClass): if discount_amount is not None: self.cost_breakdown["discount_amount"] = discount_amount - - def _response_cost_calculator( self, result: Union[ @@ -1304,6 +1308,7 @@ class Logging(LiteLLMLoggingBaseClass): response_cost = litellm.response_cost_calculator( **response_cost_calculator_kwargs ) + verbose_logger.debug(f"response_cost: {response_cost}") return response_cost except Exception as e: # error calculating cost @@ -2985,6 +2990,17 @@ class Logging(LiteLLMLoggingBaseClass): elif isinstance(result, TextCompletionResponse): return result elif isinstance(result, ResponseCompletedEvent): + ## return unified Usage object + if isinstance(result.response.usage, ResponseAPIUsage): + setattr( + result.response, + "usage", + ( + ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + result.response.usage + ) + ), + ) return result.response else: return None @@ -3103,7 +3119,7 @@ def _get_masked_values( ( v[: unmasked_length // 2] + "*" * number_of_asterisks - + v[-unmasked_length // 2:] + + v[-unmasked_length // 2 :] ) if ( isinstance(v, str) @@ -3114,7 +3130,7 @@ def _get_masked_values( ( v[: unmasked_length // 2] + "*" * (len(v) - unmasked_length) - + v[-unmasked_length // 2:] + + v[-unmasked_length // 2 :] ) if (isinstance(v, str) and len(v) > unmasked_length) else ("*****" if isinstance(v, str) else v) @@ -4464,11 +4480,10 @@ class StandardLoggingPayloadSetup: return request_tags - def _get_status_fields( status: StandardLoggingPayloadStatus, guardrail_information: Optional[dict], - error_str: Optional[str] + error_str: Optional[str], ) -> "StandardLoggingPayloadStatusFields": """ Determine status fields based on request status and guardrail information. @@ -4488,13 +4503,12 @@ def _get_status_fields( "guardrail_intervened": "guardrail_intervened", # direct "failure": "guardrail_failed_to_respond", # legacy "guardrail_failed_to_respond": "guardrail_failed_to_respond", # direct - "not_run": "not_run" + "not_run": "not_run", } # Set LLM API status llm_api_status: StandardLoggingPayloadStatus = status - ######################################################### # Map - guardrail_information.guardrail_status to guardrail_status ######################################################### @@ -4504,8 +4518,7 @@ def _get_status_fields( guardrail_status = GUARDRAIL_STATUS_MAP.get(raw_status, "not_run") return StandardLoggingPayloadStatusFields( - llm_api_status=llm_api_status, - guardrail_status=guardrail_status + llm_api_status=llm_api_status, guardrail_status=guardrail_status ) @@ -4675,8 +4688,10 @@ def get_standard_logging_object_payload( status=status, status_fields=_get_status_fields( status=status, - guardrail_information=metadata.get("standard_logging_guardrail_information", None), - error_str=error_str + guardrail_information=metadata.get( + "standard_logging_guardrail_information", None + ), + error_str=error_str, ), custom_llm_provider=cast(Optional[str], kwargs.get("custom_llm_provider")), saved_cache_cost=saved_cache_cost, diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index b2d170514af..7db7e9e7894 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -674,7 +674,9 @@ class OpenAIChatCompletionAssistantMessage(TypedDict, total=False): class ChatCompletionAssistantMessage(OpenAIChatCompletionAssistantMessage, total=False): cache_control: ChatCompletionCachedContent - thinking_blocks: Optional[List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]] + thinking_blocks: Optional[ + List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] + ] class ChatCompletionToolMessage(TypedDict): @@ -1410,9 +1412,9 @@ class ImageGenerationPartialImageEvent(BaseLiteLLMOpenAIResponseObject): class ErrorEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.ERROR] - code: Optional[str] - message: str - param: Optional[str] + code: Optional[str] = None + message: Optional[str] = None + param: Optional[str] = None class GenericEvent(BaseLiteLLMOpenAIResponseObject): diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index b3947b26789..9b4ff462531 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -24,11 +24,13 @@ from litellm.types.llms.openai import ( from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from base_responses_api import BaseResponsesAPITest, validate_responses_api_response + class TestOpenAIResponsesAPITest(BaseResponsesAPITest): def get_base_completion_call_args(self): return { "model": "openai/gpt-4o", } + def get_base_completion_reasoning_call_args(self): return { "model": "openai/gpt-5-mini", @@ -590,8 +592,6 @@ async def test_openai_responses_litellm_router_no_metadata(): request_body = mock_post.call_args.kwargs["json"] print("Request body:", json.dumps(request_body, indent=4)) - - # Assert metadata is not in the request assert ( "metadata" not in request_body @@ -1063,6 +1063,7 @@ def test_basic_computer_use_preview_tool_call(): "user": None, "metadata": {}, } + class MockResponse: def __init__(self, json_data, status_code): self._json_data = json_data @@ -1082,21 +1083,23 @@ def test_basic_computer_use_preview_tool_call(): # Call the responses API with computer_use_preview tool response = litellm.responses( model="openai/computer-use-preview", - tools=[{ - "type": "computer_use_preview", - "display_width": 1024, - "display_height": 768, - "environment": "linux" # other possible values: "mac", "windows", "ubuntu" - }], + tools=[ + { + "type": "computer_use_preview", + "display_width": 1024, + "display_height": 768, + "environment": "linux", # other possible values: "mac", "windows", "ubuntu" + } + ], input="Check the latest OpenAI news on bing.com.", reasoning={"summary": "concise"}, - truncation="auto" + truncation="auto", ) # Verify the request was made correctly mock_post.assert_called_once() request_body = mock_post.call_args.kwargs["json"] - + # Validate the request structure assert request_body["model"] == "computer-use-preview" assert len(request_body["tools"]) == 1 @@ -1104,15 +1107,14 @@ def test_basic_computer_use_preview_tool_call(): assert request_body["tools"][0]["display_width"] == 1024 assert request_body["tools"][0]["display_height"] == 768 assert request_body["tools"][0]["environment"] == "linux" - + # Check that reasoning was passed correctly assert request_body["reasoning"]["summary"] == "concise" assert request_body["truncation"] == "auto" - + # Validate the input format assert isinstance(request_body["input"], str) assert request_body["input"] == "Check the latest OpenAI news on bing.com." - def test_mcp_tools_with_responses_api(): @@ -1124,19 +1126,15 @@ def test_mcp_tools_with_responses_api(): "server_url": "https://mcp.zapier.com/api/mcp/mcp", "headers": { "Authorization": f"Bearer {os.getenv('ZAPIER_CI_CD_MCP_TOKEN')}" - } + }, } ] MODEL = "openai/gpt-4.1" USER_QUERY = "how does tiktoken work?" ######################################################### - # Step 1: OpenAI will use MCP LIST, and return a list of MCP calls for our approval + # Step 1: OpenAI will use MCP LIST, and return a list of MCP calls for our approval try: - response = litellm.responses( - model=MODEL, - tools=MCP_TOOLS, - input=USER_QUERY - ) + response = litellm.responses(model=MODEL, tools=MCP_TOOLS, input=USER_QUERY) print(response) response = cast(ResponsesAPIResponse, response) @@ -1156,20 +1154,26 @@ def test_mcp_tools_with_responses_api(): { "type": "mcp_approval_response", "approve": True, - "approval_request_id": mcp_approval_id + "approval_request_id": mcp_approval_id, } ], previous_response_id=response.id, ) print(response_with_mcp_call) except litellm.APIError as e: - if "424" in str(e) or "Failed Dependency" in str(e) or "external_connector_error" in str(e): + if ( + "424" in str(e) + or "Failed Dependency" in str(e) + or "external_connector_error" in str(e) + ): pytest.skip(f"Skipping test due to external MCP server error: {e}") else: raise e except litellm.InternalServerError as e: if "500" in str(e) or "server_error" in str(e): - pytest.skip(f"Skipping test due to OpenAI server error (likely MCP server unavailable): {e}") + pytest.skip( + f"Skipping test due to OpenAI server error (likely MCP server unavailable): {e}" + ) else: raise e @@ -1179,29 +1183,28 @@ async def test_openai_responses_api_field_types(): """Test that specific fields in the response have the correct types""" litellm._turn_on_debug() litellm.set_verbose = True - + # Test with store=True response = await litellm.aresponses( model="gpt-4o", input="hi", ) - + # Verify created_at is an integer assert isinstance(response.created_at, int), "created_at should be an integer" - + # Verify store field is present and matches input assert hasattr(response, "store"), "store field should be present" assert response.store is True, "store field should match input value" - + # Test without store parameter - response_without_store = await litellm.aresponses( - model="gpt-4o", - input="hi" - ) - + response_without_store = await litellm.aresponses(model="gpt-4o", input="hi") + # Verify created_at is still an integer - assert isinstance(response_without_store.created_at, int), "created_at should be an integer" - + assert isinstance( + response_without_store.created_at, int + ), "created_at should be an integer" + # Verify store field is present but None when not specified assert hasattr(response_without_store, "store"), "store field should be present" @@ -1210,7 +1213,7 @@ async def test_openai_responses_api_field_types(): async def test_store_field_transformation(): """Test store field transformation with mocked API responses""" config = OpenAIResponsesAPIConfig() - + # Initialize logging object with required parameters logging_obj = LiteLLMLoggingObj( model="gpt-4o", @@ -1219,7 +1222,7 @@ async def test_store_field_transformation(): call_type="aresponses", start_time=time.time(), litellm_call_id="test-call-id", - function_id="test-function-id" + function_id="test-function-id", ) # Base response data with all required fields @@ -1228,7 +1231,17 @@ async def test_store_field_transformation(): "created_at": 1751443898, "model": "gpt-4o", "object": "response", - "output": [{"type": "message", "id": "msg_1", "status": "completed", "role": "assistant", "content": [{"type": "output_text", "text": "Hello", "annotations": []}]}], + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [ + {"type": "output_text", "text": "Hello", "annotations": []} + ], + } + ], "parallel_tool_calls": True, "tool_choice": "auto", "tools": [], @@ -1245,70 +1258,70 @@ async def test_store_field_transformation(): "text": None, "truncation": "auto", "usage": {"input_tokens": 10, "output_tokens": 20, "total_tokens": 30}, - "user": "test_user" + "user": "test_user", } # Test case 1: API returns store=True mock_response_store_true = httpx.Response( - status_code=200, - content=json.dumps({**base_response, "store": True}).encode() + status_code=200, content=json.dumps({**base_response, "store": True}).encode() ) # Test case 2: API returns store=False mock_response_store_false = httpx.Response( - status_code=200, - content=json.dumps({**base_response, "store": False}).encode() + status_code=200, content=json.dumps({**base_response, "store": False}).encode() ) # Test case 3: API returns store=null mock_response_store_null = httpx.Response( - status_code=200, - content=json.dumps({**base_response, "store": None}).encode() + status_code=200, content=json.dumps({**base_response, "store": None}).encode() ) # Test case 4: API omits store field mock_response_no_store = httpx.Response( - status_code=200, - content=json.dumps(base_response).encode() + status_code=200, content=json.dumps(base_response).encode() ) # Test when store=True in request logging_obj.optional_params = {"store": True} response = config.transform_response_api_response( - model="gpt-4o", - raw_response=mock_response_store_true, - logging_obj=logging_obj + model="gpt-4o", raw_response=mock_response_store_true, logging_obj=logging_obj ) - assert response.store is True, "store should be True when specified in request and API returns True" + assert ( + response.store is True + ), "store should be True when specified in request and API returns True" # Test when store=False in request logging_obj.optional_params = {"store": False} response = config.transform_response_api_response( - model="gpt-4o", - raw_response=mock_response_store_false, - logging_obj=logging_obj + model="gpt-4o", raw_response=mock_response_store_false, logging_obj=logging_obj ) - assert response.store is False, "store should be False when specified in request and API returns False" + assert ( + response.store is False + ), "store should be False when specified in request and API returns False" # Test when store not in request but API returns null response = config.transform_response_api_response( - model="gpt-4o", - raw_response=mock_response_store_null, - logging_obj=logging_obj + model="gpt-4o", raw_response=mock_response_store_null, logging_obj=logging_obj ) - assert response.store is None, "store should be None when not specified in request and API returns null" + assert ( + response.store is None + ), "store should be None when not specified in request and API returns null" # Test when store not in request and API omits store field response = config.transform_response_api_response( - model="gpt-4o", - raw_response=mock_response_no_store, - logging_obj=logging_obj + model="gpt-4o", raw_response=mock_response_no_store, logging_obj=logging_obj ) - assert response.store is None, "store should be None when not specified in request and API omits store" + assert ( + response.store is None + ), "store should be None when not specified in request and API omits store" # Verify created_at is always converted to integer - assert isinstance(response.created_at, int), "created_at should always be converted to integer" - assert response.created_at == 1751443898, "created_at should maintain the same value after conversion" + assert isinstance( + response.created_at, int + ), "created_at should always be converted to integer" + assert ( + response.created_at == 1751443898 + ), "created_at should maintain the same value after conversion" @pytest.mark.asyncio @@ -1386,10 +1399,14 @@ async def test_aresponses_service_tier_and_safety_identifier(): mock_post.assert_called_once() request_body = mock_post.call_args.kwargs["json"] print("request_body=", json.dumps(request_body, indent=4, default=str)) - + # Validate that both parameters are present in the request body - assert request_body["service_tier"] == "flex", "service_tier should be 'flex' in request body" - assert request_body["safety_identifier"] == "123", "safety_identifier should be '123' in request body" + assert ( + request_body["service_tier"] == "flex" + ), "service_tier should be 'flex' in request body" + assert ( + request_body["safety_identifier"] == "123" + ), "safety_identifier should be '123' in request body" assert request_body["model"] == "gpt-4o" assert request_body["input"] == "Test with service tier and safety identifier" @@ -1400,11 +1417,11 @@ async def test_aresponses_service_tier_and_safety_identifier(): @pytest.mark.asyncio async def test_openai_gpt5_reasoning_effort_parameter(): """Test that reasoning_effort parameter is properly sent in the HTTP request for GPT-5 models.""" - + # Mock response for GPT-5 responses API (correct format) mock_response = { "id": "resp_01ABC123", - "object": "response", + "object": "response", "created_at": 1729621667, "status": "completed", "model": "gpt-5-mini", @@ -1412,10 +1429,14 @@ async def test_openai_gpt5_reasoning_effort_parameter(): { "type": "message", "id": "msg_123", - "status": "completed", + "status": "completed", "role": "assistant", "content": [ - {"type": "output_text", "text": "The capital of France is Paris.", "annotations": []} + { + "type": "output_text", + "text": "The capital of France is Paris.", + "annotations": [], + } ], } ], @@ -1475,8 +1496,12 @@ async def test_openai_gpt5_reasoning_effort_parameter(): print("request_body=", json.dumps(request_body, indent=4, default=str)) print("reasoning=", request_body["reasoning"]) # Validate that reasoning_effort is present in the request body - assert "reasoning" in request_body, "reasoning should be present in request body" - assert request_body["reasoning"]["effort"] == "minimal", "reasoning_effort should be 'minimal' in request body" + assert ( + "reasoning" in request_body + ), "reasoning should be present in request body" + assert ( + request_body["reasoning"]["effort"] == "minimal" + ), "reasoning_effort should be 'minimal' in request body" assert request_body["model"] == "gpt-5-mini" assert request_body["input"] == "What is the capital of France?" @@ -1484,9 +1509,6 @@ async def test_openai_gpt5_reasoning_effort_parameter(): print("Response:", json.dumps(response, indent=4, default=str)) - - - @pytest.mark.asyncio @pytest.mark.parametrize("stream", [True, False]) async def test_basic_openai_responses_with_websearch(stream): @@ -1496,15 +1518,52 @@ async def test_basic_openai_responses_with_websearch(stream): model=request_model, stream=stream, input="hi", - tools=[ - { - "type": "web_search", - "search_context_size": "low" - } - ] + tools=[{"type": "web_search", "search_context_size": "low"}], ) if stream: async for chunk in response: print("chunk=", json.dumps(chunk, indent=4, default=str)) else: print("response=", json.dumps(response, indent=4, default=str)) + + +@pytest.mark.asyncio +async def test_openai_streaming_logging(): + """Test that hard_limit parameter is properly sent in the HTTP request for GPT-5 models.""" + litellm._turn_on_debug() + from litellm.integrations.custom_logger import CustomLogger + from litellm.types.utils import Usage + + class TestCustomLogger(CustomLogger): + validate_usage = False + + def __init__(self): + self.standard_logging_object: Optional[StandardLoggingPayload] = None + + async def async_log_success_event( + self, kwargs, response_obj, start_time, end_time + ): + print(f"response_obj: {response_obj.usage}") + assert isinstance( + response_obj.usage, Usage + ), f"Expected response_obj.usage to be of type Usage, but got {type(response_obj.usage)}" + print("\n\nVALIDATED USAGE\n\n") + self.validate_usage = True + + tcl = TestCustomLogger() + litellm.callbacks = [tcl] + request_model = "gpt-5-mini" + response = await litellm.aresponses( + model=request_model, + input="What is the capital of France?", + stream=True, + ) + print("response=", json.dumps(response, indent=4, default=str)) + + async for event in response: + if event.type == "response.completed": + final_response = event + print("litellm response=", json.dumps(event, indent=4, default=str)) + + await asyncio.sleep(2) + assert tcl.validate_usage, "Usage should be validated"