Merge pull request #38997 from BerriAI/litellm_add_responses_input_tokens_endpoint

feat(proxy): add /v1/responses/input_tokens token counting endpoint
This commit is contained in:
Mateo Wang 2026-08-31 14:30:44 -07:00 committed by GitHub
commit b518be45fb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 965 additions and 72 deletions

View file

@ -693,6 +693,26 @@ def _count_document_tokens(
)
def _count_file_tokens(
file_value: object,
count_function: TokenCounterFunction,
use_default_image_token_count: bool,
) -> int:
"""An OpenAI `file` block is the chat-completions spelling of a document, so it prices like one."""
if not isinstance(file_value, Mapping):
return 0
filename: Final = file_value.get("filename")
file_data: Final = file_value.get("file_data")
name_tokens: Final = count_function(filename) if isinstance(filename, str) and filename else 0
if not isinstance(file_data, str) or not file_data:
return name_tokens
return name_tokens + calculate_img_tokens(
data=file_data,
mode="auto",
use_default_image_token_count=use_default_image_token_count,
)
def _count_anthropic_content(
content: Mapping[str, Any],
count_function: TokenCounterFunction,
@ -778,6 +798,12 @@ def _count_content_list(
use_default_image_token_count,
default_token_count,
)
elif c["type"] == "file":
num_tokens += _count_file_tokens(
c.get("file"),
count_function,
use_default_image_token_count,
)
elif c["type"] in ("tool_use", "tool_result"):
num_tokens += _count_anthropic_content(
c,
@ -807,7 +833,7 @@ def _count_content_list(
raise ValueError(
f"Invalid content item type: {content_type}. "
f"Expected str or dict with 'type' field "
f"(text, image_url, image, document, tool_use, tool_result, thinking, tool_reference)."
f"(text, image_url, image, document, file, tool_use, tool_result, thinking, tool_reference)."
)
return num_tokens
except Exception as e:

View file

@ -4,7 +4,100 @@ OpenAI Responses API token counting transformation logic.
This module handles the transformation of requests to OpenAI's /v1/responses/input_tokens endpoint.
"""
from typing import Any, Final
from collections.abc import Mapping, Sequence
from typing import Any, Final, Literal
from typing_extensions import ReadOnly, TypedDict
class ResponsesInputTextPart(TypedDict):
type: ReadOnly[Literal["input_text"]]
text: ReadOnly[str]
class ResponsesInputImagePart(TypedDict):
type: ReadOnly[Literal["input_image"]]
image_url: ReadOnly[str]
detail: ReadOnly[str]
class ResponsesInputFilePart(TypedDict):
type: ReadOnly[Literal["input_file"]]
filename: ReadOnly[str]
file_data: ReadOnly[str]
ResponsesInputPart = ResponsesInputTextPart | ResponsesInputImagePart | ResponsesInputFilePart
ResponsesContentRole = Literal["user", "assistant"]
def _chat_image_block_to_responses_part(image_url: object) -> ResponsesInputImagePart | None:
url: Final = image_url.get("url") if isinstance(image_url, Mapping) else image_url
if not isinstance(url, str) or not url:
return None
detail: Final = image_url.get("detail") if isinstance(image_url, Mapping) else None
part: Final[ResponsesInputImagePart] = {
"type": "input_image",
"image_url": url,
"detail": detail if isinstance(detail, str) and detail else "auto",
}
return part
def _chat_file_block_to_responses_part(file_value: object) -> ResponsesInputFilePart | None:
"""Only an inline file round trips: OpenAI rejects `file_data` without the `filename` beside it."""
if not isinstance(file_value, Mapping):
return None
filename: Final = file_value.get("filename")
file_data: Final = file_value.get("file_data")
if not isinstance(filename, str) or not filename or not isinstance(file_data, str) or not file_data:
return None
part: Final[ResponsesInputFilePart] = {
"type": "input_file",
"filename": filename,
"file_data": file_data,
}
return part
def _chat_block_to_responses_part(block: object, role: ResponsesContentRole) -> ResponsesInputPart | None:
if isinstance(block, str):
bare: Final[ResponsesInputTextPart] = {"type": "input_text", "text": block}
return bare
if not isinstance(block, Mapping):
return None
match block.get("type"):
case "text":
text_value: Final = block.get("text")
text: Final[ResponsesInputTextPart] = {
"type": "input_text",
"text": text_value if isinstance(text_value, str) else "",
}
return text
case "image_url" if role == "user":
return _chat_image_block_to_responses_part(block.get("image_url"))
case "file" if role == "user":
return _chat_file_block_to_responses_part(block.get("file"))
case _:
return None
def chat_content_blocks_to_responses_content(
content: Sequence[object],
role: ResponsesContentRole,
) -> str | tuple[ResponsesInputPart, ...]:
"""Text-only content collapses to a joined string, which every role accepts and counts identically.
Only a user turn may carry an image or file part: the Responses API rejects any part but
output_text and refusal inside an assistant turn.
"""
parts: Final = tuple(
part for part in (_chat_block_to_responses_part(block, role) for block in content) if part is not None
)
if any(part["type"] != "input_text" for part in parts):
return parts
return "\n".join(part["text"] for part in parts if part["type"] == "input_text")
class OpenAICountTokensConfig:
@ -120,18 +213,13 @@ class OpenAICountTokensConfig:
instructions_parts.append("\n".join(text_parts))
elif role == "user":
if isinstance(content, list):
# Extract text from content blocks for Responses API
text_parts = []
for block in content:
if isinstance(block, dict) and block.get("type") == "text":
text_parts.append(block.get("text", ""))
elif isinstance(block, str):
text_parts.append(block)
content = "\n".join(text_parts)
content = chat_content_blocks_to_responses_content(content, "user")
input_items.append({"role": "user", "content": content})
elif role == "assistant":
# Map tool_calls to Responses API function_call items
tool_calls = msg.get("tool_calls")
if isinstance(content, list):
content = chat_content_blocks_to_responses_content(content, "assistant")
if content:
input_items.append({"role": "assistant", "content": content})
if tool_calls:

View file

@ -422,6 +422,9 @@ class LiteLLMRoutes(enum.Enum):
"/responses/{response_id}/cancel",
"/v1/responses/{response_id}/cancel",
"/openai/v1/responses/{response_id}/cancel",
"/responses/input_tokens",
"/v1/responses/input_tokens",
"/openai/v1/responses/input_tokens",
# vector stores
"/vector_stores",
"/v1/vector_stores",

View file

@ -1,14 +1,18 @@
import asyncio
import json
import time
from collections.abc import AsyncIterator, Mapping
from collections.abc import AsyncIterator, Awaitable, Mapping
from enum import Enum
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, NamedTuple, cast, get_args
from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, cast, get_args
from uuid import uuid4
import fastapi
from fastapi import APIRouter, Depends, HTTPException, Request, Response
from fastapi.responses import JSONResponse
from openai.types.responses.response_create_params import ResponseInputParam
from starlette.websockets import WebSocket, WebSocketDisconnect
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_guardrail import ModifyResponseException
@ -26,8 +30,13 @@ from litellm.proxy.common_utils.http_parsing_utils import (
_read_request_body,
_safe_set_request_parsed_body,
)
from litellm.types.llms.openai import REASONING_EFFORT, ResponsesAPIResponse
from litellm.types.llms.openai import (
REASONING_EFFORT,
ResponsesAPIOptionalRequestParams,
ResponsesAPIResponse,
)
from litellm.types.responses.main import DeleteResponseResult
from litellm.types.utils import TokenCountResponse
if TYPE_CHECKING:
from litellm.router import Router
@ -35,7 +44,7 @@ if TYPE_CHECKING:
router: Final = APIRouter()
_user_api_key_auth_dep: Final = Depends(user_api_key_auth)
_RESPONSES_TAGS: Final = ["responses"] # mutable-ok: fastapi's route signature requires List[str] tags
_RESPONSES_TAGS: Final[list[str | Enum]] = ["responses"] # mutable-ok: fastapi's route signature requires list tags
_TOOL_PAYLOAD_KEYS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
{
@ -1017,6 +1026,152 @@ async def compact_response(
)
class _ResponsesApiErrorDetail(TypedDict):
message: ReadOnly[str]
type: ReadOnly[str]
param: ReadOnly[str | None]
code: ReadOnly[str | None]
class _ResponsesApiErrorBody(TypedDict):
error: ReadOnly[_ResponsesApiErrorDetail]
class _ResponsesInputTokensResult(TypedDict):
object: ReadOnly[str]
input_tokens: ReadOnly[int]
class _TokenCountPayload(TypedDict):
model: ReadOnly[str]
messages: ReadOnly[tuple[Mapping[str, object], ...]]
tools: ReadOnly[object]
class _TokenCounter(Protocol):
def __call__(self, request: TokenCountRequest, call_endpoint: bool) -> Awaitable[TokenCountResponse]: ...
def _proxy_token_counter() -> _TokenCounter:
from litellm.proxy.proxy_server import token_counter
return token_counter
_token_counter_dep: Final = Depends(_proxy_token_counter)
def _responses_invalid_request_response(message: str, param: str | None, code: str | None) -> JSONResponse:
body: Final[_ResponsesApiErrorBody] = {
"error": {
"message": message,
"type": "invalid_request_error",
"param": param,
"code": code,
}
}
return JSONResponse(status_code=400, content=body)
def _missing_responses_param_response(param: str) -> JSONResponse:
return _responses_invalid_request_response(
message=f"Missing required parameter: '{param}'.",
param=param,
code="missing_required_parameter",
)
def _responses_input_as_token_count_messages(
input_value: str | ResponseInputParam,
instructions: str | None,
) -> tuple[Mapping[str, object], ...]:
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
request_params: Final[ResponsesAPIOptionalRequestParams] = {"instructions": instructions}
transformed: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=input_value,
responses_api_request=request_params,
)
return tuple(
message if isinstance(message, dict) else message.model_dump(exclude_none=True) for message in transformed
)
@router.post(
"/v1/responses/input_tokens",
dependencies=(_user_api_key_auth_dep,),
tags=_RESPONSES_TAGS,
)
@router.post(
"/responses/input_tokens",
dependencies=(_user_api_key_auth_dep,),
tags=_RESPONSES_TAGS,
)
@router.post(
"/openai/v1/responses/input_tokens",
dependencies=(_user_api_key_auth_dep,),
tags=_RESPONSES_TAGS,
)
async def responses_input_tokens(
request: Request,
token_counter: _TokenCounter = _token_counter_dep,
):
"""
Count the input tokens of a Responses API request without calling the model.
Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens
```bash
curl -X POST http://localhost:4000/v1/responses/input_tokens \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-4o",
"input": "Hello, how are you?"
}'
```
Returns: `{"object": "response.input_tokens", "input_tokens": <count>}`
"""
data: Final = await _read_request_body(request=request)
model_name: Final = data.get("model")
input_value: Final = data.get("input")
if not isinstance(model_name, str) or not model_name:
return _missing_responses_param_response("model")
if input_value is None:
return _missing_responses_param_response("input")
if isinstance(input_value, (str, list)) and not input_value:
return _responses_invalid_request_response(
message="""One of "input" or "previous_response_id" or 'prompt' or 'conversation' must be provided.""",
param=None,
code="missing_required_parameter",
)
try:
payload: Final[_TokenCountPayload] = {
"model": model_name,
"messages": _responses_input_as_token_count_messages(
input_value=input_value,
instructions=data.get("instructions"),
),
"tools": data.get("tools"),
}
token_request: Final = TokenCountRequest.model_validate(payload)
except Exception as e:
return _responses_invalid_request_response(
message=f"Invalid request for token counting: {e}", param=None, code=None
)
token_response: Final = await token_counter(request=token_request, call_endpoint=True)
result: Final[_ResponsesInputTokensResult] = {
"object": "response.input_tokens",
"input_tokens": token_response.total_tokens,
}
return result
@router.post(
"/v1/responses/{response_id}/cancel",
dependencies=[Depends(user_api_key_auth)],

View file

@ -172,7 +172,14 @@ async def reserve_budget_for_request(
) -> dict | None:
if valid_token is None or not RouteChecks.is_llm_api_route(route=route):
return None
if route in {"/models", "/v1/models", "/utils/token_counter"}:
if route in {
"/models",
"/v1/models",
"/utils/token_counter",
"/responses/input_tokens",
"/v1/responses/input_tokens",
"/openai/v1/responses/input_tokens",
}:
return None
if get_model_from_request(request_body, route, llm_router=llm_router) is None:
return None

View file

@ -1629,6 +1629,8 @@ class LiteLLMCompletionResponsesConfig:
file_dict["file_id"] = file_id
if item.get("file_data"):
file_dict["file_data"] = item["file_data"]
if item.get("filename"):
file_dict["filename"] = item["filename"]
new_item: Final[dict[str, object]] = {"type": "file", "file": file_dict}
if "cache_control" in item:

View file

@ -1377,3 +1377,38 @@ def test_anthropic_document_title_and_context_add_their_tokens():
{"type": "document", "source": source},
]
)
def test_openai_file_block_prices_like_the_equivalent_anthropic_document():
"""An inline `file` is a `document` in the chat-completions dialect, so it must price identically, not raise.
Before the fix `file` was missing from the content-block match even though `ChatCompletionFileObject`
is in the union this counter accepts, so every local count of a Responses `input_file` raised
`Invalid content item type: file` and surfaced as a 500 on /v1/responses/input_tokens.
"""
prompt = {"type": "text", "text": "Summarize this file."}
inline_file = {
"type": "file",
"file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0xLjQK"},
}
document = {
"type": "document",
"title": "report.pdf",
"source": {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0xLjQK"},
}
assert _count_user_content([prompt, inline_file]) == _count_user_content([prompt, document])
assert _count_user_content([prompt, inline_file]) > _count_user_content([prompt])
def test_openai_file_block_without_inline_bytes_counts_what_it_carries():
"""A `file` block naming an uploaded file has no bytes to price, so it adds only the filename's tokens."""
prompt = {"type": "text", "text": "Summarize this file."}
by_id = {"type": "file", "file": {"file_id": "file-abc123"}}
assert _count_user_content([prompt, by_id]) == _count_user_content([prompt])
named = {"type": "file", "file": {"file_id": "file-abc123", "filename": "report.pdf"}}
assert _count_user_content([prompt, named]) == _count_user_content(
[prompt, {"type": "text", "text": "report.pdf"}]
)

View file

@ -163,6 +163,240 @@ def test_messages_to_responses_input_with_tool():
}
def test_messages_to_responses_input_preserves_images():
"""An image block must survive the round trip, or OpenAI counts only the text.
A 256x256 image is worth 255 tokens to OpenAI's counting API; dropping it
turned a 268-token request into a 13-token one.
"""
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "What is in this image?"},
{
"type": "image_url",
"image_url": {"url": "data:image/png;base64,iVBORw0KGgo=", "detail": "high"},
},
],
}
]
input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(messages)
assert instructions is None
assert input_items == [
{
"role": "user",
"content": (
{"type": "input_text", "text": "What is in this image?"},
{
"type": "input_image",
"image_url": "data:image/png;base64,iVBORw0KGgo=",
"detail": "high",
},
),
}
]
def test_messages_to_responses_input_image_without_detail_defaults_to_auto():
messages = [
{
"role": "user",
"content": [{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}],
}
]
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
assert input_items[0]["content"] == (
{"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"},
)
def test_messages_to_responses_input_bare_string_image_url_is_preserved():
messages = [{"role": "user", "content": [{"type": "image_url", "image_url": "https://example.com/cat.png"}]}]
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
assert input_items[0]["content"] == (
{"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"},
)
def test_messages_to_responses_input_text_only_blocks_stay_a_joined_string():
"""Text-only content must keep collapsing to a string so existing counts do not shift."""
messages = [
{
"role": "user",
"content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}],
}
]
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
assert input_items == [{"role": "user", "content": "first\nsecond"}]
def test_messages_to_responses_input_drops_unmappable_blocks():
"""A block with no Responses API equivalent is skipped, never forwarded verbatim."""
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "hi"},
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
{"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}},
],
}
]
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
assert input_items[0]["content"] == (
{"type": "input_text", "text": "hi"},
{"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"},
)
def test_messages_to_responses_input_assistant_blocks_collapse_to_a_string():
"""An assistant turn must never forward chat `text` blocks.
The Responses API only accepts output_text and refusal inside an assistant turn, so
forwarding them 400s the whole request and silently drops the count back to the local
tokenizer, which is exactly what defeats the image fix above.
"""
messages = [
{"role": "user", "content": [{"type": "text", "text": "What is the capital of France?"}]},
{"role": "assistant", "content": [{"type": "text", "text": "Paris."}]},
]
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
assert input_items == [
{"role": "user", "content": "What is the capital of France?"},
{"role": "assistant", "content": "Paris."},
]
def test_messages_to_responses_input_assistant_image_block_is_dropped():
"""An image part is illegal inside an assistant turn, so it must not reach the provider."""
messages = [
{
"role": "assistant",
"content": [
{"type": "text", "text": "Here it is"},
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
],
}
]
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
assert input_items == [{"role": "assistant", "content": "Here it is"}]
def test_messages_to_responses_input_keeps_user_image_alongside_an_assistant_turn():
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "What is in this image?"},
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
],
},
{"role": "assistant", "content": [{"type": "text", "text": "A cat."}]},
]
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
assert input_items == [
{
"role": "user",
"content": (
{"type": "input_text", "text": "What is in this image?"},
{"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"},
),
},
{"role": "assistant", "content": "A cat."},
]
def test_messages_to_responses_input_preserves_inline_files():
"""An inline file must survive the round trip, or the count silently drops the file.
A small PDF is worth 36 tokens to OpenAI's counting API; dropping it left the same
request counting 13, the text-only total.
"""
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "Summarize this file."},
{
"type": "file",
"file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0="},
},
],
}
]
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
assert input_items == [
{
"role": "user",
"content": (
{"type": "input_text", "text": "Summarize this file."},
{
"type": "input_file",
"filename": "report.pdf",
"file_data": "data:application/pdf;base64,JVBERi0=",
},
),
}
]
def test_messages_to_responses_input_drops_a_file_with_no_inline_data():
"""OpenAI rejects `file_data` without a `filename`, and a rejected request loses the whole count."""
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "Summarize this file."},
{"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0="}},
{"type": "file", "file": {"file_id": "file-abc123"}},
],
}
]
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
assert input_items == [{"role": "user", "content": "Summarize this file."}]
def test_messages_to_responses_input_assistant_file_block_is_dropped():
"""A file part is illegal inside an assistant turn, so it must not reach the provider."""
messages = [
{
"role": "assistant",
"content": [
{"type": "text", "text": "Here it is"},
{
"type": "file",
"file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0="},
},
],
}
]
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
assert input_items == [{"role": "assistant", "content": "Here it is"}]
def test_validate_request_valid():
"""Test that valid requests pass validation."""
config = OpenAICountTokensConfig()

View file

@ -3,10 +3,12 @@ Test for response_api_endpoints/endpoints.py
"""
import unittest
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi.testclient import TestClient
from httpx import Response
import litellm
from litellm.proxy.proxy_server import app
@ -82,11 +84,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase):
ResponseOutputMessage(
type="message",
role="assistant",
content=[
ResponseOutputText(
type="output_text", text="Hello from Cursor!"
)
],
content=[ResponseOutputText(type="output_text", text="Hello from Cursor!")],
)
],
)
@ -121,9 +119,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase):
@pytest.mark.asyncio
@patch("litellm.proxy.proxy_server.llm_router")
@patch("litellm.proxy.proxy_server.user_api_key_auth")
async def test_responses_api_key_spend_header_includes_response_cost(
self, mock_auth, mock_router
):
async def test_responses_api_key_spend_header_includes_response_cost(self, mock_auth, mock_router):
"""
Test that x-litellm-key-spend header includes the current request's response_cost
for /v1/responses endpoint.
@ -159,9 +155,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase):
ResponseOutputMessage(
type="message",
role="assistant",
content=[
ResponseOutputText(type="output_text", text="Test response")
],
content=[ResponseOutputText(type="output_text", text="Test response")],
)
],
)
@ -356,6 +350,7 @@ class TestWSModelExtraction:
from litellm.proxy.response_api_endpoints.endpoints import (
_extract_model_from_first_ws_event,
)
event = {"type": "response.create", "model": "gpt-4o", "input": "hello"}
assert _extract_model_from_first_ws_event(event) == "gpt-4o"
@ -363,6 +358,7 @@ class TestWSModelExtraction:
from litellm.proxy.response_api_endpoints.endpoints import (
_extract_model_from_first_ws_event,
)
event = {"type": "response.create", "response": {"model": "gpt-4o", "input": "hello"}}
assert _extract_model_from_first_ws_event(event) == "gpt-4o"
@ -370,6 +366,7 @@ class TestWSModelExtraction:
from litellm.proxy.response_api_endpoints.endpoints import (
_extract_model_from_first_ws_event,
)
event = {
"type": "response.create",
"model": "flat-model",
@ -381,6 +378,7 @@ class TestWSModelExtraction:
from litellm.proxy.response_api_endpoints.endpoints import (
_extract_model_from_first_ws_event,
)
event = {"type": "response.create", "input": "hello"}
assert _extract_model_from_first_ws_event(event) is None
@ -400,9 +398,7 @@ class TestResponsesWSFirstFrameValidation:
)
ws = MagicMock()
ws.receive_text = AsyncMock(
return_value=json.dumps({"type": "session.update", "model": "gpt-4o"})
)
ws.receive_text = AsyncMock(return_value=json.dumps({"type": "session.update", "model": "gpt-4o"}))
ws.send_text = AsyncMock()
ws.close = AsyncMock()
@ -412,10 +408,7 @@ class TestResponsesWSFirstFrameValidation:
ws.send_text.assert_awaited_once()
ws.close.assert_awaited_once_with(code=1008, reason="Invalid first message")
error_payload = json.loads(ws.send_text.await_args.args[0])
assert (
error_payload["error"]["message"]
== "First message must be a response.create JSON object."
)
assert error_payload["error"]["message"] == "First message must be a response.create JSON object."
@pytest.mark.asyncio
async def test_rejects_non_object_json_first_frame(self):
@ -484,16 +477,12 @@ class TestResponsesWSFirstFrameModelAuth:
ws.url = "ws://testserver/v1/responses"
ws.accept = AsyncMock()
ws.receive_text = AsyncMock(
return_value=json.dumps(
{"type": "response.create", "model": "gpt-4o-mini", "input": []}
)
return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []})
)
ws.close = AsyncMock()
processor = MagicMock()
processor.common_processing_pre_call_logic = AsyncMock(
return_value=({"model": "gpt-4o-mini"}, MagicMock())
)
processor.common_processing_pre_call_logic = AsyncMock(return_value=({"model": "gpt-4o-mini"}, MagicMock()))
async def fake_llm_call():
return None
@ -529,9 +518,7 @@ class TestResponsesWSFirstFrameModelAuth:
_enforce_responses_ws_first_frame_model_auth,
)
request = Request(
{"type": "http", "method": "POST", "path": "/v1/responses", "headers": []}
)
request = Request({"type": "http", "method": "POST", "path": "/v1/responses", "headers": []})
user_api_key_dict = MagicMock()
llm_router = MagicMock()
@ -593,9 +580,7 @@ class TestReadWSModelFromFirstFrameErrors:
assert result is None
ws.send_text.assert_not_awaited()
ws.close.assert_awaited_once_with(
code=1008, reason="Timed out waiting for first message"
)
ws.close.assert_awaited_once_with(code=1008, reason="Timed out waiting for first message")
@pytest.mark.asyncio
async def test_invalid_json_sends_error_and_closes(self):
@ -613,9 +598,7 @@ class TestReadWSModelFromFirstFrameErrors:
assert result is None
payload = json.loads(ws.send_text.await_args.args[0])
assert payload["error"]["message"] == "First message is not valid JSON."
ws.close.assert_awaited_once_with(
code=1008, reason="Invalid JSON in first message"
)
ws.close.assert_awaited_once_with(code=1008, reason="Invalid JSON in first message")
@pytest.mark.asyncio
async def test_missing_model_sends_error_and_closes(self):
@ -624,9 +607,7 @@ class TestReadWSModelFromFirstFrameErrors:
)
ws = MagicMock()
ws.receive_text = AsyncMock(
return_value=json.dumps({"type": "response.create", "input": []})
)
ws.receive_text = AsyncMock(return_value=json.dumps({"type": "response.create", "input": []}))
ws.send_text = AsyncMock()
ws.close = AsyncMock()
@ -679,10 +660,7 @@ class TestManagedResponsesSameProvider:
assert self._handler("gpt-4o")._same_provider("gpt-4o-mini") is True
def test_different_provider_is_not_same(self):
assert (
self._handler("gpt-4o")._same_provider("vertex_ai/gemini-2.0-flash")
is False
)
assert self._handler("gpt-4o")._same_provider("vertex_ai/gemini-2.0-flash") is False
def test_inject_credentials_keeps_provider_for_same_provider_model(self):
handler = self._handler("gpt-4o", custom_llm_provider="openai")
@ -697,18 +675,14 @@ class TestManagedResponsesSameProvider:
assert "custom_llm_provider" not in call_kwargs
def test_unresolvable_connection_model_falls_back_to_custom_provider(self):
handler = self._handler(
"my-custom-deployment", custom_llm_provider="openai"
)
handler = self._handler("my-custom-deployment", custom_llm_provider="openai")
assert handler._same_provider("gpt-4o-mini") is True
call_kwargs: dict = {}
handler._inject_credentials(call_kwargs, model="gpt-4o-mini")
assert call_kwargs["custom_llm_provider"] == "openai"
def test_unresolvable_connection_model_still_drops_cross_provider(self):
handler = self._handler(
"my-custom-deployment", custom_llm_provider="openai"
)
handler = self._handler("my-custom-deployment", custom_llm_provider="openai")
call_kwargs: dict = {}
handler._inject_credentials(call_kwargs, model="vertex_ai/gemini-2.0-flash")
assert "custom_llm_provider" not in call_kwargs
@ -840,9 +814,7 @@ def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_s
type="message",
role="assistant",
status="completed",
content=[
ResponseOutputText(type="output_text", text="agent reply", annotations=[])
],
content=[ResponseOutputText(type="output_text", text="agent reply", annotations=[])],
)
],
)
@ -851,9 +823,12 @@ def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_s
app.dependency_overrides[user_api_key_auth] = _auth_override
try:
with patch.object(ps, "llm_router", mock_router), patch(
"litellm.proxy.response_api_endpoints.endpoints._read_request_body",
side_effect=capturing_read_request_body,
with (
patch.object(ps, "llm_router", mock_router),
patch(
"litellm.proxy.response_api_endpoints.endpoints._read_request_body",
side_effect=capturing_read_request_body,
),
):
client = TestClient(app)
response = client.post(
@ -1488,8 +1463,8 @@ def _router_serving_only(base_model: str) -> MagicMock:
mock_router.router_general_settings.pass_through_all_models = False
mock_router.default_deployment = None
mock_router.pattern_router.patterns = {base_model: ["anthropic/*"]}
mock_router.pattern_router.get_pattern.side_effect = (
lambda model: [{"model_name": "anthropic/*"}] if model == base_model else None
mock_router.pattern_router.get_pattern.side_effect = lambda model: (
[{"model_name": "anthropic/*"}] if model == base_model else None
)
return mock_router
@ -1739,9 +1714,7 @@ class TestCursorGateRecognizesRoutingGroups:
from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant
router = Router(
model_list=[
{"model_name": "member-fast", "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}}
],
model_list=[{"model_name": "member-fast", "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}}],
routing_groups=[
{"group_name": "grouped-thinking-high", "models": ["member-fast"], "routing_strategy": "simple-shuffle"}
],
@ -1836,3 +1809,153 @@ class TestGuardrailBlockedResponsesUsage:
assert usage["input_tokens"] == 0
assert usage["output_tokens"] == 0
assert usage["total_tokens"] == 0
class TestResponsesInputTokens:
"""Regression tests for POST /v1/responses/input_tokens.
The docs promise OpenAI-format token counting on the proxy, but the route was
never registered, so the POST fell through to the GET/DELETE-only
/v1/responses/{response_id} route and returned 405."""
def _post_input_tokens(
self,
body: dict[str, Any],
path: str = "/v1/responses/input_tokens",
counter: AsyncMock | None = None,
) -> tuple[Response, AsyncMock]:
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.response_api_endpoints.endpoints import _proxy_token_counter
from litellm.types.utils import TokenCountResponse
token_counter_mock = (
counter
if counter is not None
else AsyncMock(
return_value=TokenCountResponse(
total_tokens=13,
request_model=body.get("model", ""),
model_used=body.get("model", ""),
tokenizer_type="openai_api",
)
)
)
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-test", request_route=path)
app.dependency_overrides[_proxy_token_counter] = lambda: token_counter_mock
try:
client = TestClient(app)
response = client.post(path, json=body, headers={"Authorization": "Bearer sk-1234"})
return response, token_counter_mock
finally:
app.dependency_overrides.pop(user_api_key_auth, None)
app.dependency_overrides.pop(_proxy_token_counter, None)
def test_string_input_returns_openai_input_tokens_shape(self):
response, counter = self._post_input_tokens({"model": "gpt-4o", "input": "Hello, how are you?"})
assert response.status_code == 200, response.text
assert response.json() == {"object": "response.input_tokens", "input_tokens": 13}
counter.assert_awaited_once()
assert counter.call_args.kwargs["call_endpoint"] is True
token_request = counter.call_args.kwargs["request"]
assert token_request.model == "gpt-4o"
assert token_request.messages == [{"role": "user", "content": "Hello, how are you?"}]
def test_every_route_alias_is_registered(self):
for path in ("/v1/responses/input_tokens", "/responses/input_tokens", "/openai/v1/responses/input_tokens"):
response, _ = self._post_input_tokens({"model": "gpt-4o", "input": "hi"}, path=path)
assert response.status_code == 200, f"{path}: {response.status_code} {response.text}"
def test_input_items_instructions_and_tools_are_forwarded(self):
tools = [
{
"type": "function",
"name": "get_weather",
"description": "Get weather for a city",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
}
]
response, counter = self._post_input_tokens(
{
"model": "gpt-4o",
"input": [{"role": "user", "content": "What is the weather in Paris?"}],
"instructions": "You are terse.",
"tools": tools,
}
)
assert response.status_code == 200, response.text
token_request = counter.call_args.kwargs["request"]
assert token_request.messages == [
{"role": "system", "content": "You are terse."},
{"role": "user", "content": "What is the weather in Paris?"},
]
assert token_request.tools == tools
def test_missing_model_returns_openai_400(self):
response, counter = self._post_input_tokens({"input": "Hello"})
assert response.status_code == 400, response.text
assert response.json() == {
"error": {
"message": "Missing required parameter: 'model'.",
"type": "invalid_request_error",
"param": "model",
"code": "missing_required_parameter",
}
}
counter.assert_not_awaited()
def test_missing_input_returns_openai_400(self):
response, counter = self._post_input_tokens({"model": "gpt-4o"})
assert response.status_code == 400, response.text
assert response.json() == {
"error": {
"message": "Missing required parameter: 'input'.",
"type": "invalid_request_error",
"param": "input",
"code": "missing_required_parameter",
}
}
counter.assert_not_awaited()
@pytest.mark.parametrize("empty_input", ["", []])
def test_empty_input_returns_openai_400(self, empty_input):
response, counter = self._post_input_tokens({"model": "gpt-4o", "input": empty_input})
assert response.status_code == 400, response.text
assert response.json() == {
"error": {
"message": """One of "input" or "previous_response_id" or 'prompt' or 'conversation' must be provided.""",
"type": "invalid_request_error",
"param": None,
"code": "missing_required_parameter",
}
}
counter.assert_not_awaited()
def test_invalid_tools_returns_openai_400(self):
response, counter = self._post_input_tokens({"model": "gpt-4o", "input": "hi", "tools": "not-a-list"})
assert response.status_code == 400, response.text
error = response.json()["error"]
assert error["type"] == "invalid_request_error"
counter.assert_not_awaited()
def test_provider_error_maps_status_code(self):
from litellm.proxy._types import ProxyException
failing_counter = AsyncMock(
side_effect=ProxyException(
message="rate limited",
type="token_counting_error",
param="model",
code="429",
)
)
response, _ = self._post_input_tokens({"model": "gpt-4o", "input": "hi"}, counter=failing_counter)
assert response.status_code == 429, response.text
assert response.json()["error"]["message"] == "rate limited"

View file

@ -0,0 +1,48 @@
from typing import Final
import pytest
from litellm.caching import DualCache
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.spend_tracking.budget_reservation import reserve_budget_for_request
from litellm.proxy.utils import ProxyLogging
TOKEN_COUNTING_ROUTES: Final = (
"/responses/input_tokens",
"/v1/responses/input_tokens",
"/openai/v1/responses/input_tokens",
"/utils/token_counter",
)
def _budgeted_token() -> UserAPIKeyAuth:
return UserAPIKeyAuth(api_key="sk-test", token="hashed-token", max_budget=100.0, spend=0.0)
async def _reserve(route: str) -> dict | None:
return await reserve_budget_for_request(
request_body={"model": "gpt-4o", "input": "hello"},
route=route,
llm_router=None,
valid_token=_budgeted_token(),
team_object=None,
user_object=None,
prisma_client=None,
user_api_key_cache=UserApiKeyCache(),
proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()),
)
@pytest.mark.asyncio
@pytest.mark.parametrize("route", TOKEN_COUNTING_ROUTES)
async def test_token_counting_routes_are_exempt_from_budget_reservation(route):
assert await _reserve(route) is None
@pytest.mark.asyncio
async def test_non_exempt_llm_route_still_reserves_budget():
reservation: Final = await _reserve("/v1/responses")
assert reservation is not None
assert reservation["reserved_cost"] > 0

View file

@ -124,6 +124,25 @@ class TestLiteLLMCompletionResponsesConfig:
assert "extra_field" not in result["file"]
assert "another_field" not in result["file"]
def test_transform_input_file_item_to_file_item_keeps_filename(self):
"""OpenAI rejects file_data with no filename beside it, so dropping it 400s the request"""
result = (
LiteLLMCompletionResponsesConfig._transform_input_file_item_to_file_item(
{
"type": "input_file",
"filename": "report.pdf",
"file_data": "data:application/pdf;base64,JVBERi0=",
}
)
)
assert result == {
"type": "file",
"file": {
"file_data": "data:application/pdf;base64,JVBERi0=",
"filename": "report.pdf",
},
}
def test_transform_input_file_item_to_file_item_with_file_url(self):
"""file_url should be mapped to file_id for downstream URL handling"""
result = (

View file

@ -9700,6 +9700,37 @@ export interface paths {
patch?: never;
trace?: never;
};
"/openai/v1/responses/input_tokens": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
/**
* Responses Input Tokens
* @description Count the input tokens of a Responses API request without calling the model.
*
* Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens
*
* ```bash
* curl -X POST http://localhost:4000/v1/responses/input_tokens -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{
* "model": "gpt-4o",
* "input": "Hello, how are you?"
* }'
* ```
*
* Returns: `{"object": "response.input_tokens", "input_tokens": <count>}`
*/
post: operations["responses_input_tokens_openai_v1_responses_input_tokens_post"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/openai/v1/responses/{response_id}": {
parameters: {
query?: never;
@ -12619,6 +12650,37 @@ export interface paths {
patch?: never;
trace?: never;
};
"/responses/input_tokens": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
/**
* Responses Input Tokens
* @description Count the input tokens of a Responses API request without calling the model.
*
* Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens
*
* ```bash
* curl -X POST http://localhost:4000/v1/responses/input_tokens -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{
* "model": "gpt-4o",
* "input": "Hello, how are you?"
* }'
* ```
*
* Returns: `{"object": "response.input_tokens", "input_tokens": <count>}`
*/
post: operations["responses_input_tokens_responses_input_tokens_post"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/responses/{response_id}": {
parameters: {
query?: never;
@ -19194,6 +19256,37 @@ export interface paths {
patch?: never;
trace?: never;
};
"/v1/responses/input_tokens": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
/**
* Responses Input Tokens
* @description Count the input tokens of a Responses API request without calling the model.
*
* Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens
*
* ```bash
* curl -X POST http://localhost:4000/v1/responses/input_tokens -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{
* "model": "gpt-4o",
* "input": "Hello, how are you?"
* }'
* ```
*
* Returns: `{"object": "response.input_tokens", "input_tokens": <count>}`
*/
post: operations["responses_input_tokens_v1_responses_input_tokens_post"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/v1/responses/{response_id}": {
parameters: {
query?: never;
@ -51496,6 +51589,26 @@ export interface operations {
};
};
};
responses_input_tokens_openai_v1_responses_input_tokens_post: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
};
};
get_response_openai_v1_responses__response_id__get: {
parameters: {
query?: never;
@ -54460,6 +54573,26 @@ export interface operations {
};
};
};
responses_input_tokens_responses_input_tokens_post: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
};
};
get_response_responses__response_id__get: {
parameters: {
query?: never;
@ -62882,6 +63015,26 @@ export interface operations {
};
};
};
responses_input_tokens_v1_responses_input_tokens_post: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
};
};
get_response_v1_responses__response_id__get: {
parameters: {
query?: never;