mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
Merge pull request #38997 from BerriAI/litellm_add_responses_input_tokens_endpoint
feat(proxy): add /v1/responses/input_tokens token counting endpoint
This commit is contained in:
commit
b518be45fb
12 changed files with 965 additions and 72 deletions
|
|
@ -693,6 +693,26 @@ def _count_document_tokens(
|
|||
)
|
||||
|
||||
|
||||
def _count_file_tokens(
|
||||
file_value: object,
|
||||
count_function: TokenCounterFunction,
|
||||
use_default_image_token_count: bool,
|
||||
) -> int:
|
||||
"""An OpenAI `file` block is the chat-completions spelling of a document, so it prices like one."""
|
||||
if not isinstance(file_value, Mapping):
|
||||
return 0
|
||||
filename: Final = file_value.get("filename")
|
||||
file_data: Final = file_value.get("file_data")
|
||||
name_tokens: Final = count_function(filename) if isinstance(filename, str) and filename else 0
|
||||
if not isinstance(file_data, str) or not file_data:
|
||||
return name_tokens
|
||||
return name_tokens + calculate_img_tokens(
|
||||
data=file_data,
|
||||
mode="auto",
|
||||
use_default_image_token_count=use_default_image_token_count,
|
||||
)
|
||||
|
||||
|
||||
def _count_anthropic_content(
|
||||
content: Mapping[str, Any],
|
||||
count_function: TokenCounterFunction,
|
||||
|
|
@ -778,6 +798,12 @@ def _count_content_list(
|
|||
use_default_image_token_count,
|
||||
default_token_count,
|
||||
)
|
||||
elif c["type"] == "file":
|
||||
num_tokens += _count_file_tokens(
|
||||
c.get("file"),
|
||||
count_function,
|
||||
use_default_image_token_count,
|
||||
)
|
||||
elif c["type"] in ("tool_use", "tool_result"):
|
||||
num_tokens += _count_anthropic_content(
|
||||
c,
|
||||
|
|
@ -807,7 +833,7 @@ def _count_content_list(
|
|||
raise ValueError(
|
||||
f"Invalid content item type: {content_type}. "
|
||||
f"Expected str or dict with 'type' field "
|
||||
f"(text, image_url, image, document, tool_use, tool_result, thinking, tool_reference)."
|
||||
f"(text, image_url, image, document, file, tool_use, tool_result, thinking, tool_reference)."
|
||||
)
|
||||
return num_tokens
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -4,7 +4,100 @@ OpenAI Responses API token counting transformation logic.
|
|||
This module handles the transformation of requests to OpenAI's /v1/responses/input_tokens endpoint.
|
||||
"""
|
||||
|
||||
from typing import Any, Final
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, Final, Literal
|
||||
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
|
||||
class ResponsesInputTextPart(TypedDict):
|
||||
type: ReadOnly[Literal["input_text"]]
|
||||
text: ReadOnly[str]
|
||||
|
||||
|
||||
class ResponsesInputImagePart(TypedDict):
|
||||
type: ReadOnly[Literal["input_image"]]
|
||||
image_url: ReadOnly[str]
|
||||
detail: ReadOnly[str]
|
||||
|
||||
|
||||
class ResponsesInputFilePart(TypedDict):
|
||||
type: ReadOnly[Literal["input_file"]]
|
||||
filename: ReadOnly[str]
|
||||
file_data: ReadOnly[str]
|
||||
|
||||
|
||||
ResponsesInputPart = ResponsesInputTextPart | ResponsesInputImagePart | ResponsesInputFilePart
|
||||
|
||||
ResponsesContentRole = Literal["user", "assistant"]
|
||||
|
||||
|
||||
def _chat_image_block_to_responses_part(image_url: object) -> ResponsesInputImagePart | None:
|
||||
url: Final = image_url.get("url") if isinstance(image_url, Mapping) else image_url
|
||||
if not isinstance(url, str) or not url:
|
||||
return None
|
||||
detail: Final = image_url.get("detail") if isinstance(image_url, Mapping) else None
|
||||
part: Final[ResponsesInputImagePart] = {
|
||||
"type": "input_image",
|
||||
"image_url": url,
|
||||
"detail": detail if isinstance(detail, str) and detail else "auto",
|
||||
}
|
||||
return part
|
||||
|
||||
|
||||
def _chat_file_block_to_responses_part(file_value: object) -> ResponsesInputFilePart | None:
|
||||
"""Only an inline file round trips: OpenAI rejects `file_data` without the `filename` beside it."""
|
||||
if not isinstance(file_value, Mapping):
|
||||
return None
|
||||
filename: Final = file_value.get("filename")
|
||||
file_data: Final = file_value.get("file_data")
|
||||
if not isinstance(filename, str) or not filename or not isinstance(file_data, str) or not file_data:
|
||||
return None
|
||||
part: Final[ResponsesInputFilePart] = {
|
||||
"type": "input_file",
|
||||
"filename": filename,
|
||||
"file_data": file_data,
|
||||
}
|
||||
return part
|
||||
|
||||
|
||||
def _chat_block_to_responses_part(block: object, role: ResponsesContentRole) -> ResponsesInputPart | None:
|
||||
if isinstance(block, str):
|
||||
bare: Final[ResponsesInputTextPart] = {"type": "input_text", "text": block}
|
||||
return bare
|
||||
if not isinstance(block, Mapping):
|
||||
return None
|
||||
match block.get("type"):
|
||||
case "text":
|
||||
text_value: Final = block.get("text")
|
||||
text: Final[ResponsesInputTextPart] = {
|
||||
"type": "input_text",
|
||||
"text": text_value if isinstance(text_value, str) else "",
|
||||
}
|
||||
return text
|
||||
case "image_url" if role == "user":
|
||||
return _chat_image_block_to_responses_part(block.get("image_url"))
|
||||
case "file" if role == "user":
|
||||
return _chat_file_block_to_responses_part(block.get("file"))
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def chat_content_blocks_to_responses_content(
|
||||
content: Sequence[object],
|
||||
role: ResponsesContentRole,
|
||||
) -> str | tuple[ResponsesInputPart, ...]:
|
||||
"""Text-only content collapses to a joined string, which every role accepts and counts identically.
|
||||
|
||||
Only a user turn may carry an image or file part: the Responses API rejects any part but
|
||||
output_text and refusal inside an assistant turn.
|
||||
"""
|
||||
parts: Final = tuple(
|
||||
part for part in (_chat_block_to_responses_part(block, role) for block in content) if part is not None
|
||||
)
|
||||
if any(part["type"] != "input_text" for part in parts):
|
||||
return parts
|
||||
return "\n".join(part["text"] for part in parts if part["type"] == "input_text")
|
||||
|
||||
|
||||
class OpenAICountTokensConfig:
|
||||
|
|
@ -120,18 +213,13 @@ class OpenAICountTokensConfig:
|
|||
instructions_parts.append("\n".join(text_parts))
|
||||
elif role == "user":
|
||||
if isinstance(content, list):
|
||||
# Extract text from content blocks for Responses API
|
||||
text_parts = []
|
||||
for block in content:
|
||||
if isinstance(block, dict) and block.get("type") == "text":
|
||||
text_parts.append(block.get("text", ""))
|
||||
elif isinstance(block, str):
|
||||
text_parts.append(block)
|
||||
content = "\n".join(text_parts)
|
||||
content = chat_content_blocks_to_responses_content(content, "user")
|
||||
input_items.append({"role": "user", "content": content})
|
||||
elif role == "assistant":
|
||||
# Map tool_calls to Responses API function_call items
|
||||
tool_calls = msg.get("tool_calls")
|
||||
if isinstance(content, list):
|
||||
content = chat_content_blocks_to_responses_content(content, "assistant")
|
||||
if content:
|
||||
input_items.append({"role": "assistant", "content": content})
|
||||
if tool_calls:
|
||||
|
|
|
|||
|
|
@ -422,6 +422,9 @@ class LiteLLMRoutes(enum.Enum):
|
|||
"/responses/{response_id}/cancel",
|
||||
"/v1/responses/{response_id}/cancel",
|
||||
"/openai/v1/responses/{response_id}/cancel",
|
||||
"/responses/input_tokens",
|
||||
"/v1/responses/input_tokens",
|
||||
"/openai/v1/responses/input_tokens",
|
||||
# vector stores
|
||||
"/vector_stores",
|
||||
"/v1/vector_stores",
|
||||
|
|
|
|||
|
|
@ -1,14 +1,18 @@
|
|||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from collections.abc import AsyncIterator, Mapping
|
||||
from collections.abc import AsyncIterator, Awaitable, Mapping
|
||||
from enum import Enum
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, NamedTuple, cast, get_args
|
||||
from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, cast, get_args
|
||||
from uuid import uuid4
|
||||
|
||||
import fastapi
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response
|
||||
from fastapi.responses import JSONResponse
|
||||
from openai.types.responses.response_create_params import ResponseInputParam
|
||||
from starlette.websockets import WebSocket, WebSocketDisconnect
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_guardrail import ModifyResponseException
|
||||
|
|
@ -26,8 +30,13 @@ from litellm.proxy.common_utils.http_parsing_utils import (
|
|||
_read_request_body,
|
||||
_safe_set_request_parsed_body,
|
||||
)
|
||||
from litellm.types.llms.openai import REASONING_EFFORT, ResponsesAPIResponse
|
||||
from litellm.types.llms.openai import (
|
||||
REASONING_EFFORT,
|
||||
ResponsesAPIOptionalRequestParams,
|
||||
ResponsesAPIResponse,
|
||||
)
|
||||
from litellm.types.responses.main import DeleteResponseResult
|
||||
from litellm.types.utils import TokenCountResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
|
@ -35,7 +44,7 @@ if TYPE_CHECKING:
|
|||
router: Final = APIRouter()
|
||||
|
||||
_user_api_key_auth_dep: Final = Depends(user_api_key_auth)
|
||||
_RESPONSES_TAGS: Final = ["responses"] # mutable-ok: fastapi's route signature requires List[str] tags
|
||||
_RESPONSES_TAGS: Final[list[str | Enum]] = ["responses"] # mutable-ok: fastapi's route signature requires list tags
|
||||
|
||||
_TOOL_PAYLOAD_KEYS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
|
||||
{
|
||||
|
|
@ -1017,6 +1026,152 @@ async def compact_response(
|
|||
)
|
||||
|
||||
|
||||
class _ResponsesApiErrorDetail(TypedDict):
|
||||
message: ReadOnly[str]
|
||||
type: ReadOnly[str]
|
||||
param: ReadOnly[str | None]
|
||||
code: ReadOnly[str | None]
|
||||
|
||||
|
||||
class _ResponsesApiErrorBody(TypedDict):
|
||||
error: ReadOnly[_ResponsesApiErrorDetail]
|
||||
|
||||
|
||||
class _ResponsesInputTokensResult(TypedDict):
|
||||
object: ReadOnly[str]
|
||||
input_tokens: ReadOnly[int]
|
||||
|
||||
|
||||
class _TokenCountPayload(TypedDict):
|
||||
model: ReadOnly[str]
|
||||
messages: ReadOnly[tuple[Mapping[str, object], ...]]
|
||||
tools: ReadOnly[object]
|
||||
|
||||
|
||||
class _TokenCounter(Protocol):
|
||||
def __call__(self, request: TokenCountRequest, call_endpoint: bool) -> Awaitable[TokenCountResponse]: ...
|
||||
|
||||
|
||||
def _proxy_token_counter() -> _TokenCounter:
|
||||
from litellm.proxy.proxy_server import token_counter
|
||||
|
||||
return token_counter
|
||||
|
||||
|
||||
_token_counter_dep: Final = Depends(_proxy_token_counter)
|
||||
|
||||
|
||||
def _responses_invalid_request_response(message: str, param: str | None, code: str | None) -> JSONResponse:
|
||||
body: Final[_ResponsesApiErrorBody] = {
|
||||
"error": {
|
||||
"message": message,
|
||||
"type": "invalid_request_error",
|
||||
"param": param,
|
||||
"code": code,
|
||||
}
|
||||
}
|
||||
return JSONResponse(status_code=400, content=body)
|
||||
|
||||
|
||||
def _missing_responses_param_response(param: str) -> JSONResponse:
|
||||
return _responses_invalid_request_response(
|
||||
message=f"Missing required parameter: '{param}'.",
|
||||
param=param,
|
||||
code="missing_required_parameter",
|
||||
)
|
||||
|
||||
|
||||
def _responses_input_as_token_count_messages(
|
||||
input_value: str | ResponseInputParam,
|
||||
instructions: str | None,
|
||||
) -> tuple[Mapping[str, object], ...]:
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
LiteLLMCompletionResponsesConfig,
|
||||
)
|
||||
|
||||
request_params: Final[ResponsesAPIOptionalRequestParams] = {"instructions": instructions}
|
||||
transformed: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=input_value,
|
||||
responses_api_request=request_params,
|
||||
)
|
||||
return tuple(
|
||||
message if isinstance(message, dict) else message.model_dump(exclude_none=True) for message in transformed
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/responses/input_tokens",
|
||||
dependencies=(_user_api_key_auth_dep,),
|
||||
tags=_RESPONSES_TAGS,
|
||||
)
|
||||
@router.post(
|
||||
"/responses/input_tokens",
|
||||
dependencies=(_user_api_key_auth_dep,),
|
||||
tags=_RESPONSES_TAGS,
|
||||
)
|
||||
@router.post(
|
||||
"/openai/v1/responses/input_tokens",
|
||||
dependencies=(_user_api_key_auth_dep,),
|
||||
tags=_RESPONSES_TAGS,
|
||||
)
|
||||
async def responses_input_tokens(
|
||||
request: Request,
|
||||
token_counter: _TokenCounter = _token_counter_dep,
|
||||
):
|
||||
"""
|
||||
Count the input tokens of a Responses API request without calling the model.
|
||||
|
||||
Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:4000/v1/responses/input_tokens \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer sk-1234" \
|
||||
-d '{
|
||||
"model": "gpt-4o",
|
||||
"input": "Hello, how are you?"
|
||||
}'
|
||||
```
|
||||
|
||||
Returns: `{"object": "response.input_tokens", "input_tokens": <count>}`
|
||||
"""
|
||||
data: Final = await _read_request_body(request=request)
|
||||
model_name: Final = data.get("model")
|
||||
input_value: Final = data.get("input")
|
||||
if not isinstance(model_name, str) or not model_name:
|
||||
return _missing_responses_param_response("model")
|
||||
if input_value is None:
|
||||
return _missing_responses_param_response("input")
|
||||
if isinstance(input_value, (str, list)) and not input_value:
|
||||
return _responses_invalid_request_response(
|
||||
message="""One of "input" or "previous_response_id" or 'prompt' or 'conversation' must be provided.""",
|
||||
param=None,
|
||||
code="missing_required_parameter",
|
||||
)
|
||||
|
||||
try:
|
||||
payload: Final[_TokenCountPayload] = {
|
||||
"model": model_name,
|
||||
"messages": _responses_input_as_token_count_messages(
|
||||
input_value=input_value,
|
||||
instructions=data.get("instructions"),
|
||||
),
|
||||
"tools": data.get("tools"),
|
||||
}
|
||||
token_request: Final = TokenCountRequest.model_validate(payload)
|
||||
except Exception as e:
|
||||
return _responses_invalid_request_response(
|
||||
message=f"Invalid request for token counting: {e}", param=None, code=None
|
||||
)
|
||||
|
||||
token_response: Final = await token_counter(request=token_request, call_endpoint=True)
|
||||
result: Final[_ResponsesInputTokensResult] = {
|
||||
"object": "response.input_tokens",
|
||||
"input_tokens": token_response.total_tokens,
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/responses/{response_id}/cancel",
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
|
|
|
|||
|
|
@ -172,7 +172,14 @@ async def reserve_budget_for_request(
|
|||
) -> dict | None:
|
||||
if valid_token is None or not RouteChecks.is_llm_api_route(route=route):
|
||||
return None
|
||||
if route in {"/models", "/v1/models", "/utils/token_counter"}:
|
||||
if route in {
|
||||
"/models",
|
||||
"/v1/models",
|
||||
"/utils/token_counter",
|
||||
"/responses/input_tokens",
|
||||
"/v1/responses/input_tokens",
|
||||
"/openai/v1/responses/input_tokens",
|
||||
}:
|
||||
return None
|
||||
if get_model_from_request(request_body, route, llm_router=llm_router) is None:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -1629,6 +1629,8 @@ class LiteLLMCompletionResponsesConfig:
|
|||
file_dict["file_id"] = file_id
|
||||
if item.get("file_data"):
|
||||
file_dict["file_data"] = item["file_data"]
|
||||
if item.get("filename"):
|
||||
file_dict["filename"] = item["filename"]
|
||||
|
||||
new_item: Final[dict[str, object]] = {"type": "file", "file": file_dict}
|
||||
if "cache_control" in item:
|
||||
|
|
|
|||
|
|
@ -1377,3 +1377,38 @@ def test_anthropic_document_title_and_context_add_their_tokens():
|
|||
{"type": "document", "source": source},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_openai_file_block_prices_like_the_equivalent_anthropic_document():
|
||||
"""An inline `file` is a `document` in the chat-completions dialect, so it must price identically, not raise.
|
||||
|
||||
Before the fix `file` was missing from the content-block match even though `ChatCompletionFileObject`
|
||||
is in the union this counter accepts, so every local count of a Responses `input_file` raised
|
||||
`Invalid content item type: file` and surfaced as a 500 on /v1/responses/input_tokens.
|
||||
"""
|
||||
prompt = {"type": "text", "text": "Summarize this file."}
|
||||
inline_file = {
|
||||
"type": "file",
|
||||
"file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0xLjQK"},
|
||||
}
|
||||
document = {
|
||||
"type": "document",
|
||||
"title": "report.pdf",
|
||||
"source": {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0xLjQK"},
|
||||
}
|
||||
|
||||
assert _count_user_content([prompt, inline_file]) == _count_user_content([prompt, document])
|
||||
assert _count_user_content([prompt, inline_file]) > _count_user_content([prompt])
|
||||
|
||||
|
||||
def test_openai_file_block_without_inline_bytes_counts_what_it_carries():
|
||||
"""A `file` block naming an uploaded file has no bytes to price, so it adds only the filename's tokens."""
|
||||
prompt = {"type": "text", "text": "Summarize this file."}
|
||||
|
||||
by_id = {"type": "file", "file": {"file_id": "file-abc123"}}
|
||||
assert _count_user_content([prompt, by_id]) == _count_user_content([prompt])
|
||||
|
||||
named = {"type": "file", "file": {"file_id": "file-abc123", "filename": "report.pdf"}}
|
||||
assert _count_user_content([prompt, named]) == _count_user_content(
|
||||
[prompt, {"type": "text", "text": "report.pdf"}]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -163,6 +163,240 @@ def test_messages_to_responses_input_with_tool():
|
|||
}
|
||||
|
||||
|
||||
def test_messages_to_responses_input_preserves_images():
|
||||
"""An image block must survive the round trip, or OpenAI counts only the text.
|
||||
|
||||
A 256x256 image is worth 255 tokens to OpenAI's counting API; dropping it
|
||||
turned a 268-token request into a 13-token one.
|
||||
"""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is in this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,iVBORw0KGgo=", "detail": "high"},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert instructions is None
|
||||
assert input_items == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
{"type": "input_text", "text": "What is in this image?"},
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/png;base64,iVBORw0KGgo=",
|
||||
"detail": "high",
|
||||
},
|
||||
),
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_messages_to_responses_input_image_without_detail_defaults_to_auto():
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}],
|
||||
}
|
||||
]
|
||||
|
||||
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert input_items[0]["content"] == (
|
||||
{"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"},
|
||||
)
|
||||
|
||||
|
||||
def test_messages_to_responses_input_bare_string_image_url_is_preserved():
|
||||
messages = [{"role": "user", "content": [{"type": "image_url", "image_url": "https://example.com/cat.png"}]}]
|
||||
|
||||
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert input_items[0]["content"] == (
|
||||
{"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"},
|
||||
)
|
||||
|
||||
|
||||
def test_messages_to_responses_input_text_only_blocks_stay_a_joined_string():
|
||||
"""Text-only content must keep collapsing to a string so existing counts do not shift."""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}],
|
||||
}
|
||||
]
|
||||
|
||||
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert input_items == [{"role": "user", "content": "first\nsecond"}]
|
||||
|
||||
|
||||
def test_messages_to_responses_input_drops_unmappable_blocks():
|
||||
"""A block with no Responses API equivalent is skipped, never forwarded verbatim."""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "hi"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
|
||||
{"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert input_items[0]["content"] == (
|
||||
{"type": "input_text", "text": "hi"},
|
||||
{"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"},
|
||||
)
|
||||
|
||||
|
||||
def test_messages_to_responses_input_assistant_blocks_collapse_to_a_string():
|
||||
"""An assistant turn must never forward chat `text` blocks.
|
||||
|
||||
The Responses API only accepts output_text and refusal inside an assistant turn, so
|
||||
forwarding them 400s the whole request and silently drops the count back to the local
|
||||
tokenizer, which is exactly what defeats the image fix above.
|
||||
"""
|
||||
messages = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "What is the capital of France?"}]},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "Paris."}]},
|
||||
]
|
||||
|
||||
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert input_items == [
|
||||
{"role": "user", "content": "What is the capital of France?"},
|
||||
{"role": "assistant", "content": "Paris."},
|
||||
]
|
||||
|
||||
|
||||
def test_messages_to_responses_input_assistant_image_block_is_dropped():
|
||||
"""An image part is illegal inside an assistant turn, so it must not reach the provider."""
|
||||
messages = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "Here it is"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert input_items == [{"role": "assistant", "content": "Here it is"}]
|
||||
|
||||
|
||||
def test_messages_to_responses_input_keeps_user_image_alongside_an_assistant_turn():
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is in this image?"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
|
||||
],
|
||||
},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "A cat."}]},
|
||||
]
|
||||
|
||||
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert input_items == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
{"type": "input_text", "text": "What is in this image?"},
|
||||
{"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"},
|
||||
),
|
||||
},
|
||||
{"role": "assistant", "content": "A cat."},
|
||||
]
|
||||
|
||||
|
||||
def test_messages_to_responses_input_preserves_inline_files():
|
||||
"""An inline file must survive the round trip, or the count silently drops the file.
|
||||
|
||||
A small PDF is worth 36 tokens to OpenAI's counting API; dropping it left the same
|
||||
request counting 13, the text-only total.
|
||||
"""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Summarize this file."},
|
||||
{
|
||||
"type": "file",
|
||||
"file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0="},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert input_items == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
{"type": "input_text", "text": "Summarize this file."},
|
||||
{
|
||||
"type": "input_file",
|
||||
"filename": "report.pdf",
|
||||
"file_data": "data:application/pdf;base64,JVBERi0=",
|
||||
},
|
||||
),
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_messages_to_responses_input_drops_a_file_with_no_inline_data():
|
||||
"""OpenAI rejects `file_data` without a `filename`, and a rejected request loses the whole count."""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Summarize this file."},
|
||||
{"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0="}},
|
||||
{"type": "file", "file": {"file_id": "file-abc123"}},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert input_items == [{"role": "user", "content": "Summarize this file."}]
|
||||
|
||||
|
||||
def test_messages_to_responses_input_assistant_file_block_is_dropped():
|
||||
"""A file part is illegal inside an assistant turn, so it must not reach the provider."""
|
||||
messages = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "Here it is"},
|
||||
{
|
||||
"type": "file",
|
||||
"file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0="},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert input_items == [{"role": "assistant", "content": "Here it is"}]
|
||||
|
||||
|
||||
def test_validate_request_valid():
|
||||
"""Test that valid requests pass validation."""
|
||||
config = OpenAICountTokensConfig()
|
||||
|
|
|
|||
|
|
@ -3,10 +3,12 @@ Test for response_api_endpoints/endpoints.py
|
|||
"""
|
||||
|
||||
import unittest
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from httpx import Response
|
||||
|
||||
import litellm
|
||||
from litellm.proxy.proxy_server import app
|
||||
|
|
@ -82,11 +84,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase):
|
|||
ResponseOutputMessage(
|
||||
type="message",
|
||||
role="assistant",
|
||||
content=[
|
||||
ResponseOutputText(
|
||||
type="output_text", text="Hello from Cursor!"
|
||||
)
|
||||
],
|
||||
content=[ResponseOutputText(type="output_text", text="Hello from Cursor!")],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
|
@ -121,9 +119,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase):
|
|||
@pytest.mark.asyncio
|
||||
@patch("litellm.proxy.proxy_server.llm_router")
|
||||
@patch("litellm.proxy.proxy_server.user_api_key_auth")
|
||||
async def test_responses_api_key_spend_header_includes_response_cost(
|
||||
self, mock_auth, mock_router
|
||||
):
|
||||
async def test_responses_api_key_spend_header_includes_response_cost(self, mock_auth, mock_router):
|
||||
"""
|
||||
Test that x-litellm-key-spend header includes the current request's response_cost
|
||||
for /v1/responses endpoint.
|
||||
|
|
@ -159,9 +155,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase):
|
|||
ResponseOutputMessage(
|
||||
type="message",
|
||||
role="assistant",
|
||||
content=[
|
||||
ResponseOutputText(type="output_text", text="Test response")
|
||||
],
|
||||
content=[ResponseOutputText(type="output_text", text="Test response")],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
|
@ -356,6 +350,7 @@ class TestWSModelExtraction:
|
|||
from litellm.proxy.response_api_endpoints.endpoints import (
|
||||
_extract_model_from_first_ws_event,
|
||||
)
|
||||
|
||||
event = {"type": "response.create", "model": "gpt-4o", "input": "hello"}
|
||||
assert _extract_model_from_first_ws_event(event) == "gpt-4o"
|
||||
|
||||
|
|
@ -363,6 +358,7 @@ class TestWSModelExtraction:
|
|||
from litellm.proxy.response_api_endpoints.endpoints import (
|
||||
_extract_model_from_first_ws_event,
|
||||
)
|
||||
|
||||
event = {"type": "response.create", "response": {"model": "gpt-4o", "input": "hello"}}
|
||||
assert _extract_model_from_first_ws_event(event) == "gpt-4o"
|
||||
|
||||
|
|
@ -370,6 +366,7 @@ class TestWSModelExtraction:
|
|||
from litellm.proxy.response_api_endpoints.endpoints import (
|
||||
_extract_model_from_first_ws_event,
|
||||
)
|
||||
|
||||
event = {
|
||||
"type": "response.create",
|
||||
"model": "flat-model",
|
||||
|
|
@ -381,6 +378,7 @@ class TestWSModelExtraction:
|
|||
from litellm.proxy.response_api_endpoints.endpoints import (
|
||||
_extract_model_from_first_ws_event,
|
||||
)
|
||||
|
||||
event = {"type": "response.create", "input": "hello"}
|
||||
assert _extract_model_from_first_ws_event(event) is None
|
||||
|
||||
|
|
@ -400,9 +398,7 @@ class TestResponsesWSFirstFrameValidation:
|
|||
)
|
||||
|
||||
ws = MagicMock()
|
||||
ws.receive_text = AsyncMock(
|
||||
return_value=json.dumps({"type": "session.update", "model": "gpt-4o"})
|
||||
)
|
||||
ws.receive_text = AsyncMock(return_value=json.dumps({"type": "session.update", "model": "gpt-4o"}))
|
||||
ws.send_text = AsyncMock()
|
||||
ws.close = AsyncMock()
|
||||
|
||||
|
|
@ -412,10 +408,7 @@ class TestResponsesWSFirstFrameValidation:
|
|||
ws.send_text.assert_awaited_once()
|
||||
ws.close.assert_awaited_once_with(code=1008, reason="Invalid first message")
|
||||
error_payload = json.loads(ws.send_text.await_args.args[0])
|
||||
assert (
|
||||
error_payload["error"]["message"]
|
||||
== "First message must be a response.create JSON object."
|
||||
)
|
||||
assert error_payload["error"]["message"] == "First message must be a response.create JSON object."
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rejects_non_object_json_first_frame(self):
|
||||
|
|
@ -484,16 +477,12 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
ws.url = "ws://testserver/v1/responses"
|
||||
ws.accept = AsyncMock()
|
||||
ws.receive_text = AsyncMock(
|
||||
return_value=json.dumps(
|
||||
{"type": "response.create", "model": "gpt-4o-mini", "input": []}
|
||||
)
|
||||
return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []})
|
||||
)
|
||||
ws.close = AsyncMock()
|
||||
|
||||
processor = MagicMock()
|
||||
processor.common_processing_pre_call_logic = AsyncMock(
|
||||
return_value=({"model": "gpt-4o-mini"}, MagicMock())
|
||||
)
|
||||
processor.common_processing_pre_call_logic = AsyncMock(return_value=({"model": "gpt-4o-mini"}, MagicMock()))
|
||||
|
||||
async def fake_llm_call():
|
||||
return None
|
||||
|
|
@ -529,9 +518,7 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
_enforce_responses_ws_first_frame_model_auth,
|
||||
)
|
||||
|
||||
request = Request(
|
||||
{"type": "http", "method": "POST", "path": "/v1/responses", "headers": []}
|
||||
)
|
||||
request = Request({"type": "http", "method": "POST", "path": "/v1/responses", "headers": []})
|
||||
user_api_key_dict = MagicMock()
|
||||
llm_router = MagicMock()
|
||||
|
||||
|
|
@ -593,9 +580,7 @@ class TestReadWSModelFromFirstFrameErrors:
|
|||
|
||||
assert result is None
|
||||
ws.send_text.assert_not_awaited()
|
||||
ws.close.assert_awaited_once_with(
|
||||
code=1008, reason="Timed out waiting for first message"
|
||||
)
|
||||
ws.close.assert_awaited_once_with(code=1008, reason="Timed out waiting for first message")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_json_sends_error_and_closes(self):
|
||||
|
|
@ -613,9 +598,7 @@ class TestReadWSModelFromFirstFrameErrors:
|
|||
assert result is None
|
||||
payload = json.loads(ws.send_text.await_args.args[0])
|
||||
assert payload["error"]["message"] == "First message is not valid JSON."
|
||||
ws.close.assert_awaited_once_with(
|
||||
code=1008, reason="Invalid JSON in first message"
|
||||
)
|
||||
ws.close.assert_awaited_once_with(code=1008, reason="Invalid JSON in first message")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_model_sends_error_and_closes(self):
|
||||
|
|
@ -624,9 +607,7 @@ class TestReadWSModelFromFirstFrameErrors:
|
|||
)
|
||||
|
||||
ws = MagicMock()
|
||||
ws.receive_text = AsyncMock(
|
||||
return_value=json.dumps({"type": "response.create", "input": []})
|
||||
)
|
||||
ws.receive_text = AsyncMock(return_value=json.dumps({"type": "response.create", "input": []}))
|
||||
ws.send_text = AsyncMock()
|
||||
ws.close = AsyncMock()
|
||||
|
||||
|
|
@ -679,10 +660,7 @@ class TestManagedResponsesSameProvider:
|
|||
assert self._handler("gpt-4o")._same_provider("gpt-4o-mini") is True
|
||||
|
||||
def test_different_provider_is_not_same(self):
|
||||
assert (
|
||||
self._handler("gpt-4o")._same_provider("vertex_ai/gemini-2.0-flash")
|
||||
is False
|
||||
)
|
||||
assert self._handler("gpt-4o")._same_provider("vertex_ai/gemini-2.0-flash") is False
|
||||
|
||||
def test_inject_credentials_keeps_provider_for_same_provider_model(self):
|
||||
handler = self._handler("gpt-4o", custom_llm_provider="openai")
|
||||
|
|
@ -697,18 +675,14 @@ class TestManagedResponsesSameProvider:
|
|||
assert "custom_llm_provider" not in call_kwargs
|
||||
|
||||
def test_unresolvable_connection_model_falls_back_to_custom_provider(self):
|
||||
handler = self._handler(
|
||||
"my-custom-deployment", custom_llm_provider="openai"
|
||||
)
|
||||
handler = self._handler("my-custom-deployment", custom_llm_provider="openai")
|
||||
assert handler._same_provider("gpt-4o-mini") is True
|
||||
call_kwargs: dict = {}
|
||||
handler._inject_credentials(call_kwargs, model="gpt-4o-mini")
|
||||
assert call_kwargs["custom_llm_provider"] == "openai"
|
||||
|
||||
def test_unresolvable_connection_model_still_drops_cross_provider(self):
|
||||
handler = self._handler(
|
||||
"my-custom-deployment", custom_llm_provider="openai"
|
||||
)
|
||||
handler = self._handler("my-custom-deployment", custom_llm_provider="openai")
|
||||
call_kwargs: dict = {}
|
||||
handler._inject_credentials(call_kwargs, model="vertex_ai/gemini-2.0-flash")
|
||||
assert "custom_llm_provider" not in call_kwargs
|
||||
|
|
@ -840,9 +814,7 @@ def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_s
|
|||
type="message",
|
||||
role="assistant",
|
||||
status="completed",
|
||||
content=[
|
||||
ResponseOutputText(type="output_text", text="agent reply", annotations=[])
|
||||
],
|
||||
content=[ResponseOutputText(type="output_text", text="agent reply", annotations=[])],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
|
@ -851,9 +823,12 @@ def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_s
|
|||
|
||||
app.dependency_overrides[user_api_key_auth] = _auth_override
|
||||
try:
|
||||
with patch.object(ps, "llm_router", mock_router), patch(
|
||||
"litellm.proxy.response_api_endpoints.endpoints._read_request_body",
|
||||
side_effect=capturing_read_request_body,
|
||||
with (
|
||||
patch.object(ps, "llm_router", mock_router),
|
||||
patch(
|
||||
"litellm.proxy.response_api_endpoints.endpoints._read_request_body",
|
||||
side_effect=capturing_read_request_body,
|
||||
),
|
||||
):
|
||||
client = TestClient(app)
|
||||
response = client.post(
|
||||
|
|
@ -1488,8 +1463,8 @@ def _router_serving_only(base_model: str) -> MagicMock:
|
|||
mock_router.router_general_settings.pass_through_all_models = False
|
||||
mock_router.default_deployment = None
|
||||
mock_router.pattern_router.patterns = {base_model: ["anthropic/*"]}
|
||||
mock_router.pattern_router.get_pattern.side_effect = (
|
||||
lambda model: [{"model_name": "anthropic/*"}] if model == base_model else None
|
||||
mock_router.pattern_router.get_pattern.side_effect = lambda model: (
|
||||
[{"model_name": "anthropic/*"}] if model == base_model else None
|
||||
)
|
||||
return mock_router
|
||||
|
||||
|
|
@ -1739,9 +1714,7 @@ class TestCursorGateRecognizesRoutingGroups:
|
|||
from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{"model_name": "member-fast", "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}}
|
||||
],
|
||||
model_list=[{"model_name": "member-fast", "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}}],
|
||||
routing_groups=[
|
||||
{"group_name": "grouped-thinking-high", "models": ["member-fast"], "routing_strategy": "simple-shuffle"}
|
||||
],
|
||||
|
|
@ -1836,3 +1809,153 @@ class TestGuardrailBlockedResponsesUsage:
|
|||
assert usage["input_tokens"] == 0
|
||||
assert usage["output_tokens"] == 0
|
||||
assert usage["total_tokens"] == 0
|
||||
|
||||
|
||||
class TestResponsesInputTokens:
|
||||
"""Regression tests for POST /v1/responses/input_tokens.
|
||||
|
||||
The docs promise OpenAI-format token counting on the proxy, but the route was
|
||||
never registered, so the POST fell through to the GET/DELETE-only
|
||||
/v1/responses/{response_id} route and returned 405."""
|
||||
|
||||
def _post_input_tokens(
|
||||
self,
|
||||
body: dict[str, Any],
|
||||
path: str = "/v1/responses/input_tokens",
|
||||
counter: AsyncMock | None = None,
|
||||
) -> tuple[Response, AsyncMock]:
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _proxy_token_counter
|
||||
from litellm.types.utils import TokenCountResponse
|
||||
|
||||
token_counter_mock = (
|
||||
counter
|
||||
if counter is not None
|
||||
else AsyncMock(
|
||||
return_value=TokenCountResponse(
|
||||
total_tokens=13,
|
||||
request_model=body.get("model", ""),
|
||||
model_used=body.get("model", ""),
|
||||
tokenizer_type="openai_api",
|
||||
)
|
||||
)
|
||||
)
|
||||
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-test", request_route=path)
|
||||
app.dependency_overrides[_proxy_token_counter] = lambda: token_counter_mock
|
||||
try:
|
||||
client = TestClient(app)
|
||||
response = client.post(path, json=body, headers={"Authorization": "Bearer sk-1234"})
|
||||
return response, token_counter_mock
|
||||
finally:
|
||||
app.dependency_overrides.pop(user_api_key_auth, None)
|
||||
app.dependency_overrides.pop(_proxy_token_counter, None)
|
||||
|
||||
def test_string_input_returns_openai_input_tokens_shape(self):
|
||||
response, counter = self._post_input_tokens({"model": "gpt-4o", "input": "Hello, how are you?"})
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == {"object": "response.input_tokens", "input_tokens": 13}
|
||||
counter.assert_awaited_once()
|
||||
assert counter.call_args.kwargs["call_endpoint"] is True
|
||||
token_request = counter.call_args.kwargs["request"]
|
||||
assert token_request.model == "gpt-4o"
|
||||
assert token_request.messages == [{"role": "user", "content": "Hello, how are you?"}]
|
||||
|
||||
def test_every_route_alias_is_registered(self):
|
||||
for path in ("/v1/responses/input_tokens", "/responses/input_tokens", "/openai/v1/responses/input_tokens"):
|
||||
response, _ = self._post_input_tokens({"model": "gpt-4o", "input": "hi"}, path=path)
|
||||
assert response.status_code == 200, f"{path}: {response.status_code} {response.text}"
|
||||
|
||||
def test_input_items_instructions_and_tools_are_forwarded(self):
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"description": "Get weather for a city",
|
||||
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
|
||||
}
|
||||
]
|
||||
response, counter = self._post_input_tokens(
|
||||
{
|
||||
"model": "gpt-4o",
|
||||
"input": [{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
"instructions": "You are terse.",
|
||||
"tools": tools,
|
||||
}
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
token_request = counter.call_args.kwargs["request"]
|
||||
assert token_request.messages == [
|
||||
{"role": "system", "content": "You are terse."},
|
||||
{"role": "user", "content": "What is the weather in Paris?"},
|
||||
]
|
||||
assert token_request.tools == tools
|
||||
|
||||
def test_missing_model_returns_openai_400(self):
|
||||
response, counter = self._post_input_tokens({"input": "Hello"})
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
assert response.json() == {
|
||||
"error": {
|
||||
"message": "Missing required parameter: 'model'.",
|
||||
"type": "invalid_request_error",
|
||||
"param": "model",
|
||||
"code": "missing_required_parameter",
|
||||
}
|
||||
}
|
||||
counter.assert_not_awaited()
|
||||
|
||||
def test_missing_input_returns_openai_400(self):
|
||||
response, counter = self._post_input_tokens({"model": "gpt-4o"})
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
assert response.json() == {
|
||||
"error": {
|
||||
"message": "Missing required parameter: 'input'.",
|
||||
"type": "invalid_request_error",
|
||||
"param": "input",
|
||||
"code": "missing_required_parameter",
|
||||
}
|
||||
}
|
||||
counter.assert_not_awaited()
|
||||
|
||||
@pytest.mark.parametrize("empty_input", ["", []])
|
||||
def test_empty_input_returns_openai_400(self, empty_input):
|
||||
response, counter = self._post_input_tokens({"model": "gpt-4o", "input": empty_input})
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
assert response.json() == {
|
||||
"error": {
|
||||
"message": """One of "input" or "previous_response_id" or 'prompt' or 'conversation' must be provided.""",
|
||||
"type": "invalid_request_error",
|
||||
"param": None,
|
||||
"code": "missing_required_parameter",
|
||||
}
|
||||
}
|
||||
counter.assert_not_awaited()
|
||||
|
||||
def test_invalid_tools_returns_openai_400(self):
|
||||
response, counter = self._post_input_tokens({"model": "gpt-4o", "input": "hi", "tools": "not-a-list"})
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
error = response.json()["error"]
|
||||
assert error["type"] == "invalid_request_error"
|
||||
counter.assert_not_awaited()
|
||||
|
||||
def test_provider_error_maps_status_code(self):
|
||||
from litellm.proxy._types import ProxyException
|
||||
|
||||
failing_counter = AsyncMock(
|
||||
side_effect=ProxyException(
|
||||
message="rate limited",
|
||||
type="token_counting_error",
|
||||
param="model",
|
||||
code="429",
|
||||
)
|
||||
)
|
||||
response, _ = self._post_input_tokens({"model": "gpt-4o", "input": "hi"}, counter=failing_counter)
|
||||
|
||||
assert response.status_code == 429, response.text
|
||||
assert response.json()["error"]["message"] == "rate limited"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.caching import DualCache
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.spend_tracking.budget_reservation import reserve_budget_for_request
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
TOKEN_COUNTING_ROUTES: Final = (
|
||||
"/responses/input_tokens",
|
||||
"/v1/responses/input_tokens",
|
||||
"/openai/v1/responses/input_tokens",
|
||||
"/utils/token_counter",
|
||||
)
|
||||
|
||||
|
||||
def _budgeted_token() -> UserAPIKeyAuth:
|
||||
return UserAPIKeyAuth(api_key="sk-test", token="hashed-token", max_budget=100.0, spend=0.0)
|
||||
|
||||
|
||||
async def _reserve(route: str) -> dict | None:
|
||||
return await reserve_budget_for_request(
|
||||
request_body={"model": "gpt-4o", "input": "hello"},
|
||||
route=route,
|
||||
llm_router=None,
|
||||
valid_token=_budgeted_token(),
|
||||
team_object=None,
|
||||
user_object=None,
|
||||
prisma_client=None,
|
||||
user_api_key_cache=UserApiKeyCache(),
|
||||
proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("route", TOKEN_COUNTING_ROUTES)
|
||||
async def test_token_counting_routes_are_exempt_from_budget_reservation(route):
|
||||
assert await _reserve(route) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_exempt_llm_route_still_reserves_budget():
|
||||
reservation: Final = await _reserve("/v1/responses")
|
||||
|
||||
assert reservation is not None
|
||||
assert reservation["reserved_cost"] > 0
|
||||
|
|
@ -124,6 +124,25 @@ class TestLiteLLMCompletionResponsesConfig:
|
|||
assert "extra_field" not in result["file"]
|
||||
assert "another_field" not in result["file"]
|
||||
|
||||
def test_transform_input_file_item_to_file_item_keeps_filename(self):
|
||||
"""OpenAI rejects file_data with no filename beside it, so dropping it 400s the request"""
|
||||
result = (
|
||||
LiteLLMCompletionResponsesConfig._transform_input_file_item_to_file_item(
|
||||
{
|
||||
"type": "input_file",
|
||||
"filename": "report.pdf",
|
||||
"file_data": "data:application/pdf;base64,JVBERi0=",
|
||||
}
|
||||
)
|
||||
)
|
||||
assert result == {
|
||||
"type": "file",
|
||||
"file": {
|
||||
"file_data": "data:application/pdf;base64,JVBERi0=",
|
||||
"filename": "report.pdf",
|
||||
},
|
||||
}
|
||||
|
||||
def test_transform_input_file_item_to_file_item_with_file_url(self):
|
||||
"""file_url should be mapped to file_id for downstream URL handling"""
|
||||
result = (
|
||||
|
|
|
|||
153
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
153
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -9700,6 +9700,37 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/openai/v1/responses/input_tokens": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/**
|
||||
* Responses Input Tokens
|
||||
* @description Count the input tokens of a Responses API request without calling the model.
|
||||
*
|
||||
* Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens
|
||||
*
|
||||
* ```bash
|
||||
* curl -X POST http://localhost:4000/v1/responses/input_tokens -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{
|
||||
* "model": "gpt-4o",
|
||||
* "input": "Hello, how are you?"
|
||||
* }'
|
||||
* ```
|
||||
*
|
||||
* Returns: `{"object": "response.input_tokens", "input_tokens": <count>}`
|
||||
*/
|
||||
post: operations["responses_input_tokens_openai_v1_responses_input_tokens_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/openai/v1/responses/{response_id}": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -12619,6 +12650,37 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/responses/input_tokens": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/**
|
||||
* Responses Input Tokens
|
||||
* @description Count the input tokens of a Responses API request without calling the model.
|
||||
*
|
||||
* Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens
|
||||
*
|
||||
* ```bash
|
||||
* curl -X POST http://localhost:4000/v1/responses/input_tokens -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{
|
||||
* "model": "gpt-4o",
|
||||
* "input": "Hello, how are you?"
|
||||
* }'
|
||||
* ```
|
||||
*
|
||||
* Returns: `{"object": "response.input_tokens", "input_tokens": <count>}`
|
||||
*/
|
||||
post: operations["responses_input_tokens_responses_input_tokens_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/responses/{response_id}": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -19194,6 +19256,37 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/v1/responses/input_tokens": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/**
|
||||
* Responses Input Tokens
|
||||
* @description Count the input tokens of a Responses API request without calling the model.
|
||||
*
|
||||
* Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens
|
||||
*
|
||||
* ```bash
|
||||
* curl -X POST http://localhost:4000/v1/responses/input_tokens -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{
|
||||
* "model": "gpt-4o",
|
||||
* "input": "Hello, how are you?"
|
||||
* }'
|
||||
* ```
|
||||
*
|
||||
* Returns: `{"object": "response.input_tokens", "input_tokens": <count>}`
|
||||
*/
|
||||
post: operations["responses_input_tokens_v1_responses_input_tokens_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/v1/responses/{response_id}": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -51496,6 +51589,26 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
responses_input_tokens_openai_v1_responses_input_tokens_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": unknown;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
get_response_openai_v1_responses__response_id__get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -54460,6 +54573,26 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
responses_input_tokens_responses_input_tokens_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": unknown;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
get_response_responses__response_id__get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -62882,6 +63015,26 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
responses_input_tokens_v1_responses_input_tokens_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": unknown;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
get_response_v1_responses__response_id__get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue