scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

fix(litellm_logging.py): for responses api - return a unified usage object for logging

Browse source 
ensures logging integrations all pull the right usage information
This commit is contained in:
Krrish Dholakia 2025-10-22 13:33:52 -07:00
parent bea8e13a94
commit 09fc9deac8
3 changed files with 176 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
litellm/litellm_core_utils/litellm_logging.py
View file

@ -76,6 +76,7 @@ from litellm.types.llms.openai import (
HttpxBinaryResponseContent,
OpenAIFileObject,
OpenAIModerationResponse,
ResponseAPIUsage,
ResponseCompletedEvent,
ResponsesAPIResponse,
)
@ -700,8 +701,13 @@ class Logging(LiteLLMLoggingBaseClass):
vector_store_custom_logger.__class__.__name__
)
# Add to global callbacks so post-call hooks are invoked
if vector_store_custom_logger and vector_store_custom_logger not in litellm.callbacks:
litellm.logging_callback_manager.add_litellm_callback(vector_store_custom_logger)
if (
vector_store_custom_logger
and vector_store_custom_logger not in litellm.callbacks
):
litellm.logging_callback_manager.add_litellm_callback(
vector_store_custom_logger
)
return vector_store_custom_logger
return None
@ -1206,8 +1212,6 @@ class Logging(LiteLLMLoggingBaseClass):
if discount_amount is not None:
self.cost_breakdown["discount_amount"] = discount_amount
def _response_cost_calculator(
self,
result: Union[
@ -1304,6 +1308,7 @@ class Logging(LiteLLMLoggingBaseClass):
response_cost = litellm.response_cost_calculator(
**response_cost_calculator_kwargs
)
verbose_logger.debug(f"response_cost: {response_cost}")
return response_cost
except Exception as e: # error calculating cost
@ -2985,6 +2990,17 @@ class Logging(LiteLLMLoggingBaseClass):
elif isinstance(result, TextCompletionResponse):
return result
elif isinstance(result, ResponseCompletedEvent):
## return unified Usage object
if isinstance(result.response.usage, ResponseAPIUsage):
setattr(
result.response,
"usage",
(
ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
result.response.usage
)
),
)
return result.response
else:
return None
@ -3103,7 +3119,7 @@ def _get_masked_values(
(
v[: unmasked_length // 2]
+ "*" * number_of_asterisks
+ v[-unmasked_length // 2:]
+ v[-unmasked_length // 2 :]
)
if (
isinstance(v, str)
@ -3114,7 +3130,7 @@ def _get_masked_values(
(
v[: unmasked_length // 2]
+ "*" * (len(v) - unmasked_length)
+ v[-unmasked_length // 2:]
+ v[-unmasked_length // 2 :]
)
if (isinstance(v, str) and len(v) > unmasked_length)
else ("*****" if isinstance(v, str) else v)
@ -4464,11 +4480,10 @@ class StandardLoggingPayloadSetup:
return request_tags
def _get_status_fields(
status: StandardLoggingPayloadStatus,
guardrail_information: Optional[dict],
error_str: Optional[str]
error_str: Optional[str],
) -> "StandardLoggingPayloadStatusFields":
"""
Determine status fields based on request status and guardrail information.
@ -4488,13 +4503,12 @@ def _get_status_fields(
"guardrail_intervened": "guardrail_intervened", # direct
"failure": "guardrail_failed_to_respond", # legacy
"guardrail_failed_to_respond": "guardrail_failed_to_respond", # direct
"not_run": "not_run"
"not_run": "not_run",
}
# Set LLM API status
llm_api_status: StandardLoggingPayloadStatus = status
#########################################################
# Map - guardrail_information.guardrail_status to guardrail_status
#########################################################
@ -4504,8 +4518,7 @@ def _get_status_fields(
guardrail_status = GUARDRAIL_STATUS_MAP.get(raw_status, "not_run")
return StandardLoggingPayloadStatusFields(
llm_api_status=llm_api_status,
guardrail_status=guardrail_status
llm_api_status=llm_api_status, guardrail_status=guardrail_status
)
@ -4675,8 +4688,10 @@ def get_standard_logging_object_payload(
status=status,
status_fields=_get_status_fields(
status=status,
guardrail_information=metadata.get("standard_logging_guardrail_information", None),
error_str=error_str
guardrail_information=metadata.get(
"standard_logging_guardrail_information", None
),
error_str=error_str,
),
custom_llm_provider=cast(Optional[str], kwargs.get("custom_llm_provider")),
saved_cache_cost=saved_cache_cost,

10
litellm/types/llms/openai.py
View file

@ -674,7 +674,9 @@ class OpenAIChatCompletionAssistantMessage(TypedDict, total=False):
class ChatCompletionAssistantMessage(OpenAIChatCompletionAssistantMessage, total=False):
cache_control: ChatCompletionCachedContent
thinking_blocks: Optional[List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]]
thinking_blocks: Optional[
List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]
]
class ChatCompletionToolMessage(TypedDict):
@ -1410,9 +1412,9 @@ class ImageGenerationPartialImageEvent(BaseLiteLLMOpenAIResponseObject):
class ErrorEvent(BaseLiteLLMOpenAIResponseObject):
type: Literal[ResponsesAPIStreamEvents.ERROR]
code: Optional[str]
message: str
param: Optional[str]
code: Optional[str] = None
message: Optional[str] = None
param: Optional[str] = None
class GenericEvent(BaseLiteLLMOpenAIResponseObject):

223
tests/llm_responses_api_testing/test_openai_responses_api.py
View file

@ -24,11 +24,13 @@ from litellm.types.llms.openai import (
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from base_responses_api import BaseResponsesAPITest, validate_responses_api_response
class TestOpenAIResponsesAPITest(BaseResponsesAPITest):
def get_base_completion_call_args(self):
return {
"model": "openai/gpt-4o",
}
def get_base_completion_reasoning_call_args(self):
return {
"model": "openai/gpt-5-mini",
@ -590,8 +592,6 @@ async def test_openai_responses_litellm_router_no_metadata():
request_body = mock_post.call_args.kwargs["json"]
print("Request body:", json.dumps(request_body, indent=4))
# Assert metadata is not in the request
assert (
"metadata" not in request_body
@ -1063,6 +1063,7 @@ def test_basic_computer_use_preview_tool_call():
"user": None,
"metadata": {},
}
class MockResponse:
def __init__(self, json_data, status_code):
self._json_data = json_data
@ -1082,21 +1083,23 @@ def test_basic_computer_use_preview_tool_call():
# Call the responses API with computer_use_preview tool
response = litellm.responses(
model="openai/computer-use-preview",
tools=[{
"type": "computer_use_preview",
"display_width": 1024,
"display_height": 768,
"environment": "linux" # other possible values: "mac", "windows", "ubuntu"
}],
tools=[
{
"type": "computer_use_preview",
"display_width": 1024,
"display_height": 768,
"environment": "linux", # other possible values: "mac", "windows", "ubuntu"
}
],
input="Check the latest OpenAI news on bing.com.",
reasoning={"summary": "concise"},
truncation="auto"
truncation="auto",
)
# Verify the request was made correctly
mock_post.assert_called_once()
request_body = mock_post.call_args.kwargs["json"]
# Validate the request structure
assert request_body["model"] == "computer-use-preview"
assert len(request_body["tools"]) == 1
@ -1104,15 +1107,14 @@ def test_basic_computer_use_preview_tool_call():
assert request_body["tools"][0]["display_width"] == 1024
assert request_body["tools"][0]["display_height"] == 768
assert request_body["tools"][0]["environment"] == "linux"
# Check that reasoning was passed correctly
assert request_body["reasoning"]["summary"] == "concise"
assert request_body["truncation"] == "auto"
# Validate the input format
assert isinstance(request_body["input"], str)
assert request_body["input"] == "Check the latest OpenAI news on bing.com."
def test_mcp_tools_with_responses_api():
@ -1124,19 +1126,15 @@ def test_mcp_tools_with_responses_api():
"server_url": "https://mcp.zapier.com/api/mcp/mcp",
"headers": {
"Authorization": f"Bearer {os.getenv('ZAPIER_CI_CD_MCP_TOKEN')}"
}
},
}
]
MODEL = "openai/gpt-4.1"
USER_QUERY = "how does tiktoken work?"
#########################################################
# Step 1: OpenAI will use MCP LIST, and return a list of MCP calls for our approval
# Step 1: OpenAI will use MCP LIST, and return a list of MCP calls for our approval
try:
response = litellm.responses(
model=MODEL,
tools=MCP_TOOLS,
input=USER_QUERY
)
response = litellm.responses(model=MODEL, tools=MCP_TOOLS, input=USER_QUERY)
print(response)
response = cast(ResponsesAPIResponse, response)
@ -1156,20 +1154,26 @@ def test_mcp_tools_with_responses_api():
{
"type": "mcp_approval_response",
"approve": True,
"approval_request_id": mcp_approval_id
"approval_request_id": mcp_approval_id,
}
],
previous_response_id=response.id,
)
print(response_with_mcp_call)
except litellm.APIError as e:
if "424" in str(e) or "Failed Dependency" in str(e) or "external_connector_error" in str(e):
if (
"424" in str(e)
or "Failed Dependency" in str(e)
or "external_connector_error" in str(e)
):
pytest.skip(f"Skipping test due to external MCP server error: {e}")
else:
raise e
except litellm.InternalServerError as e:
if "500" in str(e) or "server_error" in str(e):
pytest.skip(f"Skipping test due to OpenAI server error (likely MCP server unavailable): {e}")
pytest.skip(
f"Skipping test due to OpenAI server error (likely MCP server unavailable): {e}"
)
else:
raise e
@ -1179,29 +1183,28 @@ async def test_openai_responses_api_field_types():
"""Test that specific fields in the response have the correct types"""
litellm._turn_on_debug()
litellm.set_verbose = True
# Test with store=True
response = await litellm.aresponses(
model="gpt-4o",
input="hi",
)
# Verify created_at is an integer
assert isinstance(response.created_at, int), "created_at should be an integer"
# Verify store field is present and matches input
assert hasattr(response, "store"), "store field should be present"
assert response.store is True, "store field should match input value"
# Test without store parameter
response_without_store = await litellm.aresponses(
model="gpt-4o",
input="hi"
)
response_without_store = await litellm.aresponses(model="gpt-4o", input="hi")
# Verify created_at is still an integer
assert isinstance(response_without_store.created_at, int), "created_at should be an integer"
assert isinstance(
response_without_store.created_at, int
), "created_at should be an integer"
# Verify store field is present but None when not specified
assert hasattr(response_without_store, "store"), "store field should be present"
@ -1210,7 +1213,7 @@ async def test_openai_responses_api_field_types():
async def test_store_field_transformation():
"""Test store field transformation with mocked API responses"""
config = OpenAIResponsesAPIConfig()
# Initialize logging object with required parameters
logging_obj = LiteLLMLoggingObj(
model="gpt-4o",
@ -1219,7 +1222,7 @@ async def test_store_field_transformation():
call_type="aresponses",
start_time=time.time(),
litellm_call_id="test-call-id",
function_id="test-function-id"
function_id="test-function-id",
)
# Base response data with all required fields
@ -1228,7 +1231,17 @@ async def test_store_field_transformation():
"created_at": 1751443898,
"model": "gpt-4o",
"object": "response",
"output": [{"type": "message", "id": "msg_1", "status": "completed", "role": "assistant", "content": [{"type": "output_text", "text": "Hello", "annotations": []}]}],
"output": [
{
"type": "message",
"id": "msg_1",
"status": "completed",
"role": "assistant",
"content": [
{"type": "output_text", "text": "Hello", "annotations": []}
],
}
],
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
@ -1245,70 +1258,70 @@ async def test_store_field_transformation():
"text": None,
"truncation": "auto",
"usage": {"input_tokens": 10, "output_tokens": 20, "total_tokens": 30},
"user": "test_user"
"user": "test_user",
}
# Test case 1: API returns store=True
mock_response_store_true = httpx.Response(
status_code=200,
content=json.dumps({**base_response, "store": True}).encode()
status_code=200, content=json.dumps({**base_response, "store": True}).encode()
)
# Test case 2: API returns store=False
mock_response_store_false = httpx.Response(
status_code=200,
content=json.dumps({**base_response, "store": False}).encode()
status_code=200, content=json.dumps({**base_response, "store": False}).encode()
)
# Test case 3: API returns store=null
mock_response_store_null = httpx.Response(
status_code=200,
content=json.dumps({**base_response, "store": None}).encode()
status_code=200, content=json.dumps({**base_response, "store": None}).encode()
)
# Test case 4: API omits store field
mock_response_no_store = httpx.Response(
status_code=200,
content=json.dumps(base_response).encode()
status_code=200, content=json.dumps(base_response).encode()
)
# Test when store=True in request
logging_obj.optional_params = {"store": True}
response = config.transform_response_api_response(
model="gpt-4o",
raw_response=mock_response_store_true,
logging_obj=logging_obj
model="gpt-4o", raw_response=mock_response_store_true, logging_obj=logging_obj
)
assert response.store is True, "store should be True when specified in request and API returns True"
assert (
response.store is True
), "store should be True when specified in request and API returns True"
# Test when store=False in request
logging_obj.optional_params = {"store": False}
response = config.transform_response_api_response(
model="gpt-4o",
raw_response=mock_response_store_false,
logging_obj=logging_obj
model="gpt-4o", raw_response=mock_response_store_false, logging_obj=logging_obj
)
assert response.store is False, "store should be False when specified in request and API returns False"
assert (
response.store is False
), "store should be False when specified in request and API returns False"
# Test when store not in request but API returns null
response = config.transform_response_api_response(
model="gpt-4o",
raw_response=mock_response_store_null,
logging_obj=logging_obj
model="gpt-4o", raw_response=mock_response_store_null, logging_obj=logging_obj
)
assert response.store is None, "store should be None when not specified in request and API returns null"
assert (
response.store is None
), "store should be None when not specified in request and API returns null"
# Test when store not in request and API omits store field
response = config.transform_response_api_response(
model="gpt-4o",
raw_response=mock_response_no_store,
logging_obj=logging_obj
model="gpt-4o", raw_response=mock_response_no_store, logging_obj=logging_obj
)
assert response.store is None, "store should be None when not specified in request and API omits store"
assert (
response.store is None
), "store should be None when not specified in request and API omits store"
# Verify created_at is always converted to integer
assert isinstance(response.created_at, int), "created_at should always be converted to integer"
assert response.created_at == 1751443898, "created_at should maintain the same value after conversion"
assert isinstance(
response.created_at, int
), "created_at should always be converted to integer"
assert (
response.created_at == 1751443898
), "created_at should maintain the same value after conversion"
@pytest.mark.asyncio
@ -1386,10 +1399,14 @@ async def test_aresponses_service_tier_and_safety_identifier():
mock_post.assert_called_once()
request_body = mock_post.call_args.kwargs["json"]
print("request_body=", json.dumps(request_body, indent=4, default=str))
# Validate that both parameters are present in the request body
assert request_body["service_tier"] == "flex", "service_tier should be 'flex' in request body"
assert request_body["safety_identifier"] == "123", "safety_identifier should be '123' in request body"
assert (
request_body["service_tier"] == "flex"
), "service_tier should be 'flex' in request body"
assert (
request_body["safety_identifier"] == "123"
), "safety_identifier should be '123' in request body"
assert request_body["model"] == "gpt-4o"
assert request_body["input"] == "Test with service tier and safety identifier"
@ -1400,11 +1417,11 @@ async def test_aresponses_service_tier_and_safety_identifier():
@pytest.mark.asyncio
async def test_openai_gpt5_reasoning_effort_parameter():
"""Test that reasoning_effort parameter is properly sent in the HTTP request for GPT-5 models."""
# Mock response for GPT-5 responses API (correct format)
mock_response = {
"id": "resp_01ABC123",
"object": "response",
"object": "response",
"created_at": 1729621667,
"status": "completed",
"model": "gpt-5-mini",
@ -1412,10 +1429,14 @@ async def test_openai_gpt5_reasoning_effort_parameter():
{
"type": "message",
"id": "msg_123",
"status": "completed",
"status": "completed",
"role": "assistant",
"content": [
{"type": "output_text", "text": "The capital of France is Paris.", "annotations": []}
{
"type": "output_text",
"text": "The capital of France is Paris.",
"annotations": [],
}
],
}
],
@ -1475,8 +1496,12 @@ async def test_openai_gpt5_reasoning_effort_parameter():
print("request_body=", json.dumps(request_body, indent=4, default=str))
print("reasoning=", request_body["reasoning"])
# Validate that reasoning_effort is present in the request body
assert "reasoning" in request_body, "reasoning should be present in request body"
assert request_body["reasoning"]["effort"] == "minimal", "reasoning_effort should be 'minimal' in request body"
assert (
"reasoning" in request_body
), "reasoning should be present in request body"
assert (
request_body["reasoning"]["effort"] == "minimal"
), "reasoning_effort should be 'minimal' in request body"
assert request_body["model"] == "gpt-5-mini"
assert request_body["input"] == "What is the capital of France?"
@ -1484,9 +1509,6 @@ async def test_openai_gpt5_reasoning_effort_parameter():
print("Response:", json.dumps(response, indent=4, default=str))
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [True, False])
async def test_basic_openai_responses_with_websearch(stream):
@ -1496,15 +1518,52 @@ async def test_basic_openai_responses_with_websearch(stream):
model=request_model,
stream=stream,
input="hi",
tools=[
{
"type": "web_search",
"search_context_size": "low"
}
]
tools=[{"type": "web_search", "search_context_size": "low"}],
)
if stream:
async for chunk in response:
print("chunk=", json.dumps(chunk, indent=4, default=str))
else:
print("response=", json.dumps(response, indent=4, default=str))
@pytest.mark.asyncio
async def test_openai_streaming_logging():
"""Test that hard_limit parameter is properly sent in the HTTP request for GPT-5 models."""
litellm._turn_on_debug()
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.utils import Usage
class TestCustomLogger(CustomLogger):
validate_usage = False
def __init__(self):
self.standard_logging_object: Optional[StandardLoggingPayload] = None
async def async_log_success_event(
self, kwargs, response_obj, start_time, end_time
):
print(f"response_obj: {response_obj.usage}")
assert isinstance(
response_obj.usage, Usage
), f"Expected response_obj.usage to be of type Usage, but got {type(response_obj.usage)}"
print("\n\nVALIDATED USAGE\n\n")
self.validate_usage = True
tcl = TestCustomLogger()
litellm.callbacks = [tcl]
request_model = "gpt-5-mini"
response = await litellm.aresponses(
model=request_model,
input="What is the capital of France?",
stream=True,
)
print("response=", json.dumps(response, indent=4, default=str))
async for event in response:
if event.type == "response.completed":
final_response = event
print("litellm response=", json.dumps(event, indent=4, default=str))
await asyncio.sleep(2)
assert tcl.validate_usage, "Usage should be validated"