mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-06 02:48:13 +00:00
fix: report real token usage on guardrail-blocked /v1/responses replies
## TLDR Signed-off-by: Ishaan <ishaangupta0408@gmail.com>
This commit is contained in:
parent
423b791ee0
commit
81c27fc4a0
2 changed files with 50 additions and 1 deletions
|
|
@ -164,6 +164,13 @@ async def _resolve_cursor_model_variant_before_auth(request: Request) -> None:
|
|||
_safe_set_request_parsed_body(request=request, parsed_body=resolved)
|
||||
|
||||
|
||||
def _blocked_responses_api_usage(original_response: Any) -> ResponseAPIUsage:
|
||||
usage: Final = getattr(original_response, "usage", None) if original_response is not None else None
|
||||
if isinstance(usage, ResponseAPIUsage):
|
||||
return usage
|
||||
return ResponseAPIUsage(input_tokens=0, output_tokens=0, total_tokens=0)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/responses",
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
|
|
@ -415,7 +422,7 @@ async def responses_api(
|
|||
model=e.model or data.get("model"),
|
||||
output=cast(Any, [{"content": [{"type": "text", "text": violation_text}]}]),
|
||||
status="completed",
|
||||
usage=ResponseAPIUsage(input_tokens=0, output_tokens=0, total_tokens=0),
|
||||
usage=_blocked_responses_api_usage(e.original_response),
|
||||
)
|
||||
return response_obj
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import pytest
|
|||
|
||||
import litellm
|
||||
from litellm.proxy.proxy_server import _blocked_response_usage
|
||||
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
|
||||
|
||||
|
||||
def test_uses_original_response_usage():
|
||||
|
|
@ -82,3 +83,44 @@ async def test_success_hook_attaches_original_response_on_block():
|
|||
)
|
||||
|
||||
assert excinfo.value.original_response is response
|
||||
|
||||
|
||||
def test_responses_api_blocked_reply_carries_real_usage():
|
||||
"""Regression: /v1/responses blocked reply must carry the real upstream token counts.
|
||||
|
||||
The ModifyResponseException handler in responses_api used to hardcode usage to zeros.
|
||||
"""
|
||||
import time
|
||||
|
||||
from litellm.proxy.response_api_endpoints.endpoints import (
|
||||
_blocked_responses_api_usage,
|
||||
)
|
||||
|
||||
original_response = ResponsesAPIResponse(
|
||||
id="resp_orig",
|
||||
object="response",
|
||||
created_at=int(time.time()),
|
||||
model="gpt-4o-mini",
|
||||
output=[],
|
||||
status="completed",
|
||||
usage=ResponseAPIUsage(input_tokens=14, output_tokens=20, total_tokens=34),
|
||||
)
|
||||
|
||||
usage = _blocked_responses_api_usage(original_response)
|
||||
|
||||
assert usage.input_tokens == 14
|
||||
assert usage.output_tokens == 20
|
||||
assert usage.total_tokens == 34
|
||||
|
||||
|
||||
def test_responses_api_blocked_reply_zero_usage_when_no_original_response():
|
||||
"""Pre-call block has no original_response, so usage must be zero."""
|
||||
from litellm.proxy.response_api_endpoints.endpoints import (
|
||||
_blocked_responses_api_usage,
|
||||
)
|
||||
|
||||
usage = _blocked_responses_api_usage(None)
|
||||
|
||||
assert usage.input_tokens == 0
|
||||
assert usage.output_tokens == 0
|
||||
assert usage.total_tokens == 0
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue