diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 256bee7b348..c350bc5569e 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -693,6 +693,26 @@ def _count_document_tokens( ) +def _count_file_tokens( + file_value: object, + count_function: TokenCounterFunction, + use_default_image_token_count: bool, +) -> int: + """An OpenAI `file` block is the chat-completions spelling of a document, so it prices like one.""" + if not isinstance(file_value, Mapping): + return 0 + filename: Final = file_value.get("filename") + file_data: Final = file_value.get("file_data") + name_tokens: Final = count_function(filename) if isinstance(filename, str) and filename else 0 + if not isinstance(file_data, str) or not file_data: + return name_tokens + return name_tokens + calculate_img_tokens( + data=file_data, + mode="auto", + use_default_image_token_count=use_default_image_token_count, + ) + + def _count_anthropic_content( content: Mapping[str, Any], count_function: TokenCounterFunction, @@ -778,6 +798,12 @@ def _count_content_list( use_default_image_token_count, default_token_count, ) + elif c["type"] == "file": + num_tokens += _count_file_tokens( + c.get("file"), + count_function, + use_default_image_token_count, + ) elif c["type"] in ("tool_use", "tool_result"): num_tokens += _count_anthropic_content( c, @@ -807,7 +833,7 @@ def _count_content_list( raise ValueError( f"Invalid content item type: {content_type}. " f"Expected str or dict with 'type' field " - f"(text, image_url, image, document, tool_use, tool_result, thinking, tool_reference)." + f"(text, image_url, image, document, file, tool_use, tool_result, thinking, tool_reference)." ) return num_tokens except Exception as e: diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py index 6b2f4535df1..88f04c59e01 100644 --- a/litellm/llms/openai/responses/count_tokens/transformation.py +++ b/litellm/llms/openai/responses/count_tokens/transformation.py @@ -4,7 +4,100 @@ OpenAI Responses API token counting transformation logic. This module handles the transformation of requests to OpenAI's /v1/responses/input_tokens endpoint. """ -from typing import Any, Final +from collections.abc import Mapping, Sequence +from typing import Any, Final, Literal + +from typing_extensions import ReadOnly, TypedDict + + +class ResponsesInputTextPart(TypedDict): + type: ReadOnly[Literal["input_text"]] + text: ReadOnly[str] + + +class ResponsesInputImagePart(TypedDict): + type: ReadOnly[Literal["input_image"]] + image_url: ReadOnly[str] + detail: ReadOnly[str] + + +class ResponsesInputFilePart(TypedDict): + type: ReadOnly[Literal["input_file"]] + filename: ReadOnly[str] + file_data: ReadOnly[str] + + +ResponsesInputPart = ResponsesInputTextPart | ResponsesInputImagePart | ResponsesInputFilePart + +ResponsesContentRole = Literal["user", "assistant"] + + +def _chat_image_block_to_responses_part(image_url: object) -> ResponsesInputImagePart | None: + url: Final = image_url.get("url") if isinstance(image_url, Mapping) else image_url + if not isinstance(url, str) or not url: + return None + detail: Final = image_url.get("detail") if isinstance(image_url, Mapping) else None + part: Final[ResponsesInputImagePart] = { + "type": "input_image", + "image_url": url, + "detail": detail if isinstance(detail, str) and detail else "auto", + } + return part + + +def _chat_file_block_to_responses_part(file_value: object) -> ResponsesInputFilePart | None: + """Only an inline file round trips: OpenAI rejects `file_data` without the `filename` beside it.""" + if not isinstance(file_value, Mapping): + return None + filename: Final = file_value.get("filename") + file_data: Final = file_value.get("file_data") + if not isinstance(filename, str) or not filename or not isinstance(file_data, str) or not file_data: + return None + part: Final[ResponsesInputFilePart] = { + "type": "input_file", + "filename": filename, + "file_data": file_data, + } + return part + + +def _chat_block_to_responses_part(block: object, role: ResponsesContentRole) -> ResponsesInputPart | None: + if isinstance(block, str): + bare: Final[ResponsesInputTextPart] = {"type": "input_text", "text": block} + return bare + if not isinstance(block, Mapping): + return None + match block.get("type"): + case "text": + text_value: Final = block.get("text") + text: Final[ResponsesInputTextPart] = { + "type": "input_text", + "text": text_value if isinstance(text_value, str) else "", + } + return text + case "image_url" if role == "user": + return _chat_image_block_to_responses_part(block.get("image_url")) + case "file" if role == "user": + return _chat_file_block_to_responses_part(block.get("file")) + case _: + return None + + +def chat_content_blocks_to_responses_content( + content: Sequence[object], + role: ResponsesContentRole, +) -> str | tuple[ResponsesInputPart, ...]: + """Text-only content collapses to a joined string, which every role accepts and counts identically. + + Only a user turn may carry an image or file part: the Responses API rejects any part but + output_text and refusal inside an assistant turn. + """ + parts: Final = tuple( + part for part in (_chat_block_to_responses_part(block, role) for block in content) if part is not None + ) + if any(part["type"] != "input_text" for part in parts): + return parts + return "\n".join(part["text"] for part in parts if part["type"] == "input_text") class OpenAICountTokensConfig: @@ -120,18 +213,13 @@ class OpenAICountTokensConfig: instructions_parts.append("\n".join(text_parts)) elif role == "user": if isinstance(content, list): - # Extract text from content blocks for Responses API - text_parts = [] - for block in content: - if isinstance(block, dict) and block.get("type") == "text": - text_parts.append(block.get("text", "")) - elif isinstance(block, str): - text_parts.append(block) - content = "\n".join(text_parts) + content = chat_content_blocks_to_responses_content(content, "user") input_items.append({"role": "user", "content": content}) elif role == "assistant": # Map tool_calls to Responses API function_call items tool_calls = msg.get("tool_calls") + if isinstance(content, list): + content = chat_content_blocks_to_responses_content(content, "assistant") if content: input_items.append({"role": "assistant", "content": content}) if tool_calls: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index a84fae7fd23..7075fa5a31a 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -422,6 +422,9 @@ class LiteLLMRoutes(enum.Enum): "/responses/{response_id}/cancel", "/v1/responses/{response_id}/cancel", "/openai/v1/responses/{response_id}/cancel", + "/responses/input_tokens", + "/v1/responses/input_tokens", + "/openai/v1/responses/input_tokens", # vector stores "/vector_stores", "/v1/vector_stores", diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 5e56e822484..aa7595ed13d 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -1,14 +1,18 @@ import asyncio import json import time -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Awaitable, Mapping +from enum import Enum from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, NamedTuple, cast, get_args +from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, cast, get_args from uuid import uuid4 import fastapi from fastapi import APIRouter, Depends, HTTPException, Request, Response +from fastapi.responses import JSONResponse +from openai.types.responses.response_create_params import ResponseInputParam from starlette.websockets import WebSocket, WebSocketDisconnect +from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ModifyResponseException @@ -26,8 +30,13 @@ from litellm.proxy.common_utils.http_parsing_utils import ( _read_request_body, _safe_set_request_parsed_body, ) -from litellm.types.llms.openai import REASONING_EFFORT, ResponsesAPIResponse +from litellm.types.llms.openai import ( + REASONING_EFFORT, + ResponsesAPIOptionalRequestParams, + ResponsesAPIResponse, +) from litellm.types.responses.main import DeleteResponseResult +from litellm.types.utils import TokenCountResponse if TYPE_CHECKING: from litellm.router import Router @@ -35,7 +44,7 @@ if TYPE_CHECKING: router: Final = APIRouter() _user_api_key_auth_dep: Final = Depends(user_api_key_auth) -_RESPONSES_TAGS: Final = ["responses"] # mutable-ok: fastapi's route signature requires List[str] tags +_RESPONSES_TAGS: Final[list[str | Enum]] = ["responses"] # mutable-ok: fastapi's route signature requires list tags _TOOL_PAYLOAD_KEYS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType( { @@ -1017,6 +1026,152 @@ async def compact_response( ) +class _ResponsesApiErrorDetail(TypedDict): + message: ReadOnly[str] + type: ReadOnly[str] + param: ReadOnly[str | None] + code: ReadOnly[str | None] + + +class _ResponsesApiErrorBody(TypedDict): + error: ReadOnly[_ResponsesApiErrorDetail] + + +class _ResponsesInputTokensResult(TypedDict): + object: ReadOnly[str] + input_tokens: ReadOnly[int] + + +class _TokenCountPayload(TypedDict): + model: ReadOnly[str] + messages: ReadOnly[tuple[Mapping[str, object], ...]] + tools: ReadOnly[object] + + +class _TokenCounter(Protocol): + def __call__(self, request: TokenCountRequest, call_endpoint: bool) -> Awaitable[TokenCountResponse]: ... + + +def _proxy_token_counter() -> _TokenCounter: + from litellm.proxy.proxy_server import token_counter + + return token_counter + + +_token_counter_dep: Final = Depends(_proxy_token_counter) + + +def _responses_invalid_request_response(message: str, param: str | None, code: str | None) -> JSONResponse: + body: Final[_ResponsesApiErrorBody] = { + "error": { + "message": message, + "type": "invalid_request_error", + "param": param, + "code": code, + } + } + return JSONResponse(status_code=400, content=body) + + +def _missing_responses_param_response(param: str) -> JSONResponse: + return _responses_invalid_request_response( + message=f"Missing required parameter: '{param}'.", + param=param, + code="missing_required_parameter", + ) + + +def _responses_input_as_token_count_messages( + input_value: str | ResponseInputParam, + instructions: str | None, +) -> tuple[Mapping[str, object], ...]: + from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, + ) + + request_params: Final[ResponsesAPIOptionalRequestParams] = {"instructions": instructions} + transformed: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=input_value, + responses_api_request=request_params, + ) + return tuple( + message if isinstance(message, dict) else message.model_dump(exclude_none=True) for message in transformed + ) + + +@router.post( + "/v1/responses/input_tokens", + dependencies=(_user_api_key_auth_dep,), + tags=_RESPONSES_TAGS, +) +@router.post( + "/responses/input_tokens", + dependencies=(_user_api_key_auth_dep,), + tags=_RESPONSES_TAGS, +) +@router.post( + "/openai/v1/responses/input_tokens", + dependencies=(_user_api_key_auth_dep,), + tags=_RESPONSES_TAGS, +) +async def responses_input_tokens( + request: Request, + token_counter: _TokenCounter = _token_counter_dep, +): + """ + Count the input tokens of a Responses API request without calling the model. + + Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens + + ```bash + curl -X POST http://localhost:4000/v1/responses/input_tokens \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-4o", + "input": "Hello, how are you?" + }' + ``` + + Returns: `{"object": "response.input_tokens", "input_tokens": }` + """ + data: Final = await _read_request_body(request=request) + model_name: Final = data.get("model") + input_value: Final = data.get("input") + if not isinstance(model_name, str) or not model_name: + return _missing_responses_param_response("model") + if input_value is None: + return _missing_responses_param_response("input") + if isinstance(input_value, (str, list)) and not input_value: + return _responses_invalid_request_response( + message="""One of "input" or "previous_response_id" or 'prompt' or 'conversation' must be provided.""", + param=None, + code="missing_required_parameter", + ) + + try: + payload: Final[_TokenCountPayload] = { + "model": model_name, + "messages": _responses_input_as_token_count_messages( + input_value=input_value, + instructions=data.get("instructions"), + ), + "tools": data.get("tools"), + } + token_request: Final = TokenCountRequest.model_validate(payload) + except Exception as e: + return _responses_invalid_request_response( + message=f"Invalid request for token counting: {e}", param=None, code=None + ) + + token_response: Final = await token_counter(request=token_request, call_endpoint=True) + result: Final[_ResponsesInputTokensResult] = { + "object": "response.input_tokens", + "input_tokens": token_response.total_tokens, + } + return result + + @router.post( "/v1/responses/{response_id}/cancel", dependencies=[Depends(user_api_key_auth)], diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 8b2a5dd9312..2d113cfe355 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -172,7 +172,14 @@ async def reserve_budget_for_request( ) -> dict | None: if valid_token is None or not RouteChecks.is_llm_api_route(route=route): return None - if route in {"/models", "/v1/models", "/utils/token_counter"}: + if route in { + "/models", + "/v1/models", + "/utils/token_counter", + "/responses/input_tokens", + "/v1/responses/input_tokens", + "/openai/v1/responses/input_tokens", + }: return None if get_model_from_request(request_body, route, llm_router=llm_router) is None: return None diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index f39df38d069..4a416f61e1b 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1629,6 +1629,8 @@ class LiteLLMCompletionResponsesConfig: file_dict["file_id"] = file_id if item.get("file_data"): file_dict["file_data"] = item["file_data"] + if item.get("filename"): + file_dict["filename"] = item["filename"] new_item: Final[dict[str, object]] = {"type": "file", "file": file_dict} if "cache_control" in item: diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index 572b505e94c..4694fa8fbed 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -1377,3 +1377,38 @@ def test_anthropic_document_title_and_context_add_their_tokens(): {"type": "document", "source": source}, ] ) + + +def test_openai_file_block_prices_like_the_equivalent_anthropic_document(): + """An inline `file` is a `document` in the chat-completions dialect, so it must price identically, not raise. + + Before the fix `file` was missing from the content-block match even though `ChatCompletionFileObject` + is in the union this counter accepts, so every local count of a Responses `input_file` raised + `Invalid content item type: file` and surfaced as a 500 on /v1/responses/input_tokens. + """ + prompt = {"type": "text", "text": "Summarize this file."} + inline_file = { + "type": "file", + "file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0xLjQK"}, + } + document = { + "type": "document", + "title": "report.pdf", + "source": {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0xLjQK"}, + } + + assert _count_user_content([prompt, inline_file]) == _count_user_content([prompt, document]) + assert _count_user_content([prompt, inline_file]) > _count_user_content([prompt]) + + +def test_openai_file_block_without_inline_bytes_counts_what_it_carries(): + """A `file` block naming an uploaded file has no bytes to price, so it adds only the filename's tokens.""" + prompt = {"type": "text", "text": "Summarize this file."} + + by_id = {"type": "file", "file": {"file_id": "file-abc123"}} + assert _count_user_content([prompt, by_id]) == _count_user_content([prompt]) + + named = {"type": "file", "file": {"file_id": "file-abc123", "filename": "report.pdf"}} + assert _count_user_content([prompt, named]) == _count_user_content( + [prompt, {"type": "text", "text": "report.pdf"}] + ) diff --git a/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py index e1cc6a92927..c2efc1acdb9 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py @@ -163,6 +163,240 @@ def test_messages_to_responses_input_with_tool(): } +def test_messages_to_responses_input_preserves_images(): + """An image block must survive the round trip, or OpenAI counts only the text. + + A 256x256 image is worth 255 tokens to OpenAI's counting API; dropping it + turned a 268-token request into a 13-token one. + """ + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0KGgo=", "detail": "high"}, + }, + ], + } + ] + + input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert instructions is None + assert input_items == [ + { + "role": "user", + "content": ( + {"type": "input_text", "text": "What is in this image?"}, + { + "type": "input_image", + "image_url": "data:image/png;base64,iVBORw0KGgo=", + "detail": "high", + }, + ), + } + ] + + +def test_messages_to_responses_input_image_without_detail_defaults_to_auto(): + messages = [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items[0]["content"] == ( + {"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"}, + ) + + +def test_messages_to_responses_input_bare_string_image_url_is_preserved(): + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": "https://example.com/cat.png"}]}] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items[0]["content"] == ( + {"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"}, + ) + + +def test_messages_to_responses_input_text_only_blocks_stay_a_joined_string(): + """Text-only content must keep collapsing to a string so existing counts do not shift.""" + messages = [ + { + "role": "user", + "content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [{"role": "user", "content": "first\nsecond"}] + + +def test_messages_to_responses_input_drops_unmappable_blocks(): + """A block with no Responses API equivalent is skipped, never forwarded verbatim.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "hi"}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}, + {"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}, + ], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items[0]["content"] == ( + {"type": "input_text", "text": "hi"}, + {"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"}, + ) + + +def test_messages_to_responses_input_assistant_blocks_collapse_to_a_string(): + """An assistant turn must never forward chat `text` blocks. + + The Responses API only accepts output_text and refusal inside an assistant turn, so + forwarding them 400s the whole request and silently drops the count back to the local + tokenizer, which is exactly what defeats the image fix above. + """ + messages = [ + {"role": "user", "content": [{"type": "text", "text": "What is the capital of France?"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "Paris."}]}, + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [ + {"role": "user", "content": "What is the capital of France?"}, + {"role": "assistant", "content": "Paris."}, + ] + + +def test_messages_to_responses_input_assistant_image_block_is_dropped(): + """An image part is illegal inside an assistant turn, so it must not reach the provider.""" + messages = [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Here it is"}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}, + ], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [{"role": "assistant", "content": "Here it is"}] + + +def test_messages_to_responses_input_keeps_user_image_alongside_an_assistant_turn(): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}, + ], + }, + {"role": "assistant", "content": [{"type": "text", "text": "A cat."}]}, + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [ + { + "role": "user", + "content": ( + {"type": "input_text", "text": "What is in this image?"}, + {"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"}, + ), + }, + {"role": "assistant", "content": "A cat."}, + ] + + +def test_messages_to_responses_input_preserves_inline_files(): + """An inline file must survive the round trip, or the count silently drops the file. + + A small PDF is worth 36 tokens to OpenAI's counting API; dropping it left the same + request counting 13, the text-only total. + """ + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize this file."}, + { + "type": "file", + "file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0="}, + }, + ], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [ + { + "role": "user", + "content": ( + {"type": "input_text", "text": "Summarize this file."}, + { + "type": "input_file", + "filename": "report.pdf", + "file_data": "data:application/pdf;base64,JVBERi0=", + }, + ), + } + ] + + +def test_messages_to_responses_input_drops_a_file_with_no_inline_data(): + """OpenAI rejects `file_data` without a `filename`, and a rejected request loses the whole count.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize this file."}, + {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0="}}, + {"type": "file", "file": {"file_id": "file-abc123"}}, + ], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [{"role": "user", "content": "Summarize this file."}] + + +def test_messages_to_responses_input_assistant_file_block_is_dropped(): + """A file part is illegal inside an assistant turn, so it must not reach the provider.""" + messages = [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Here it is"}, + { + "type": "file", + "file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0="}, + }, + ], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [{"role": "assistant", "content": "Here it is"}] + + def test_validate_request_valid(): """Test that valid requests pass validation.""" config = OpenAICountTokensConfig() diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index 791d64c6428..d7010de6405 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -3,10 +3,12 @@ Test for response_api_endpoints/endpoints.py """ import unittest +from typing import Any from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient +from httpx import Response import litellm from litellm.proxy.proxy_server import app @@ -82,11 +84,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase): ResponseOutputMessage( type="message", role="assistant", - content=[ - ResponseOutputText( - type="output_text", text="Hello from Cursor!" - ) - ], + content=[ResponseOutputText(type="output_text", text="Hello from Cursor!")], ) ], ) @@ -121,9 +119,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase): @pytest.mark.asyncio @patch("litellm.proxy.proxy_server.llm_router") @patch("litellm.proxy.proxy_server.user_api_key_auth") - async def test_responses_api_key_spend_header_includes_response_cost( - self, mock_auth, mock_router - ): + async def test_responses_api_key_spend_header_includes_response_cost(self, mock_auth, mock_router): """ Test that x-litellm-key-spend header includes the current request's response_cost for /v1/responses endpoint. @@ -159,9 +155,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase): ResponseOutputMessage( type="message", role="assistant", - content=[ - ResponseOutputText(type="output_text", text="Test response") - ], + content=[ResponseOutputText(type="output_text", text="Test response")], ) ], ) @@ -356,6 +350,7 @@ class TestWSModelExtraction: from litellm.proxy.response_api_endpoints.endpoints import ( _extract_model_from_first_ws_event, ) + event = {"type": "response.create", "model": "gpt-4o", "input": "hello"} assert _extract_model_from_first_ws_event(event) == "gpt-4o" @@ -363,6 +358,7 @@ class TestWSModelExtraction: from litellm.proxy.response_api_endpoints.endpoints import ( _extract_model_from_first_ws_event, ) + event = {"type": "response.create", "response": {"model": "gpt-4o", "input": "hello"}} assert _extract_model_from_first_ws_event(event) == "gpt-4o" @@ -370,6 +366,7 @@ class TestWSModelExtraction: from litellm.proxy.response_api_endpoints.endpoints import ( _extract_model_from_first_ws_event, ) + event = { "type": "response.create", "model": "flat-model", @@ -381,6 +378,7 @@ class TestWSModelExtraction: from litellm.proxy.response_api_endpoints.endpoints import ( _extract_model_from_first_ws_event, ) + event = {"type": "response.create", "input": "hello"} assert _extract_model_from_first_ws_event(event) is None @@ -400,9 +398,7 @@ class TestResponsesWSFirstFrameValidation: ) ws = MagicMock() - ws.receive_text = AsyncMock( - return_value=json.dumps({"type": "session.update", "model": "gpt-4o"}) - ) + ws.receive_text = AsyncMock(return_value=json.dumps({"type": "session.update", "model": "gpt-4o"})) ws.send_text = AsyncMock() ws.close = AsyncMock() @@ -412,10 +408,7 @@ class TestResponsesWSFirstFrameValidation: ws.send_text.assert_awaited_once() ws.close.assert_awaited_once_with(code=1008, reason="Invalid first message") error_payload = json.loads(ws.send_text.await_args.args[0]) - assert ( - error_payload["error"]["message"] - == "First message must be a response.create JSON object." - ) + assert error_payload["error"]["message"] == "First message must be a response.create JSON object." @pytest.mark.asyncio async def test_rejects_non_object_json_first_frame(self): @@ -484,16 +477,12 @@ class TestResponsesWSFirstFrameModelAuth: ws.url = "ws://testserver/v1/responses" ws.accept = AsyncMock() ws.receive_text = AsyncMock( - return_value=json.dumps( - {"type": "response.create", "model": "gpt-4o-mini", "input": []} - ) + return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []}) ) ws.close = AsyncMock() processor = MagicMock() - processor.common_processing_pre_call_logic = AsyncMock( - return_value=({"model": "gpt-4o-mini"}, MagicMock()) - ) + processor.common_processing_pre_call_logic = AsyncMock(return_value=({"model": "gpt-4o-mini"}, MagicMock())) async def fake_llm_call(): return None @@ -529,9 +518,7 @@ class TestResponsesWSFirstFrameModelAuth: _enforce_responses_ws_first_frame_model_auth, ) - request = Request( - {"type": "http", "method": "POST", "path": "/v1/responses", "headers": []} - ) + request = Request({"type": "http", "method": "POST", "path": "/v1/responses", "headers": []}) user_api_key_dict = MagicMock() llm_router = MagicMock() @@ -593,9 +580,7 @@ class TestReadWSModelFromFirstFrameErrors: assert result is None ws.send_text.assert_not_awaited() - ws.close.assert_awaited_once_with( - code=1008, reason="Timed out waiting for first message" - ) + ws.close.assert_awaited_once_with(code=1008, reason="Timed out waiting for first message") @pytest.mark.asyncio async def test_invalid_json_sends_error_and_closes(self): @@ -613,9 +598,7 @@ class TestReadWSModelFromFirstFrameErrors: assert result is None payload = json.loads(ws.send_text.await_args.args[0]) assert payload["error"]["message"] == "First message is not valid JSON." - ws.close.assert_awaited_once_with( - code=1008, reason="Invalid JSON in first message" - ) + ws.close.assert_awaited_once_with(code=1008, reason="Invalid JSON in first message") @pytest.mark.asyncio async def test_missing_model_sends_error_and_closes(self): @@ -624,9 +607,7 @@ class TestReadWSModelFromFirstFrameErrors: ) ws = MagicMock() - ws.receive_text = AsyncMock( - return_value=json.dumps({"type": "response.create", "input": []}) - ) + ws.receive_text = AsyncMock(return_value=json.dumps({"type": "response.create", "input": []})) ws.send_text = AsyncMock() ws.close = AsyncMock() @@ -679,10 +660,7 @@ class TestManagedResponsesSameProvider: assert self._handler("gpt-4o")._same_provider("gpt-4o-mini") is True def test_different_provider_is_not_same(self): - assert ( - self._handler("gpt-4o")._same_provider("vertex_ai/gemini-2.0-flash") - is False - ) + assert self._handler("gpt-4o")._same_provider("vertex_ai/gemini-2.0-flash") is False def test_inject_credentials_keeps_provider_for_same_provider_model(self): handler = self._handler("gpt-4o", custom_llm_provider="openai") @@ -697,18 +675,14 @@ class TestManagedResponsesSameProvider: assert "custom_llm_provider" not in call_kwargs def test_unresolvable_connection_model_falls_back_to_custom_provider(self): - handler = self._handler( - "my-custom-deployment", custom_llm_provider="openai" - ) + handler = self._handler("my-custom-deployment", custom_llm_provider="openai") assert handler._same_provider("gpt-4o-mini") is True call_kwargs: dict = {} handler._inject_credentials(call_kwargs, model="gpt-4o-mini") assert call_kwargs["custom_llm_provider"] == "openai" def test_unresolvable_connection_model_still_drops_cross_provider(self): - handler = self._handler( - "my-custom-deployment", custom_llm_provider="openai" - ) + handler = self._handler("my-custom-deployment", custom_llm_provider="openai") call_kwargs: dict = {} handler._inject_credentials(call_kwargs, model="vertex_ai/gemini-2.0-flash") assert "custom_llm_provider" not in call_kwargs @@ -840,9 +814,7 @@ def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_s type="message", role="assistant", status="completed", - content=[ - ResponseOutputText(type="output_text", text="agent reply", annotations=[]) - ], + content=[ResponseOutputText(type="output_text", text="agent reply", annotations=[])], ) ], ) @@ -851,9 +823,12 @@ def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_s app.dependency_overrides[user_api_key_auth] = _auth_override try: - with patch.object(ps, "llm_router", mock_router), patch( - "litellm.proxy.response_api_endpoints.endpoints._read_request_body", - side_effect=capturing_read_request_body, + with ( + patch.object(ps, "llm_router", mock_router), + patch( + "litellm.proxy.response_api_endpoints.endpoints._read_request_body", + side_effect=capturing_read_request_body, + ), ): client = TestClient(app) response = client.post( @@ -1488,8 +1463,8 @@ def _router_serving_only(base_model: str) -> MagicMock: mock_router.router_general_settings.pass_through_all_models = False mock_router.default_deployment = None mock_router.pattern_router.patterns = {base_model: ["anthropic/*"]} - mock_router.pattern_router.get_pattern.side_effect = ( - lambda model: [{"model_name": "anthropic/*"}] if model == base_model else None + mock_router.pattern_router.get_pattern.side_effect = lambda model: ( + [{"model_name": "anthropic/*"}] if model == base_model else None ) return mock_router @@ -1739,9 +1714,7 @@ class TestCursorGateRecognizesRoutingGroups: from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant router = Router( - model_list=[ - {"model_name": "member-fast", "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}} - ], + model_list=[{"model_name": "member-fast", "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}}], routing_groups=[ {"group_name": "grouped-thinking-high", "models": ["member-fast"], "routing_strategy": "simple-shuffle"} ], @@ -1836,3 +1809,153 @@ class TestGuardrailBlockedResponsesUsage: assert usage["input_tokens"] == 0 assert usage["output_tokens"] == 0 assert usage["total_tokens"] == 0 + + +class TestResponsesInputTokens: + """Regression tests for POST /v1/responses/input_tokens. + + The docs promise OpenAI-format token counting on the proxy, but the route was + never registered, so the POST fell through to the GET/DELETE-only + /v1/responses/{response_id} route and returned 405.""" + + def _post_input_tokens( + self, + body: dict[str, Any], + path: str = "/v1/responses/input_tokens", + counter: AsyncMock | None = None, + ) -> tuple[Response, AsyncMock]: + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.response_api_endpoints.endpoints import _proxy_token_counter + from litellm.types.utils import TokenCountResponse + + token_counter_mock = ( + counter + if counter is not None + else AsyncMock( + return_value=TokenCountResponse( + total_tokens=13, + request_model=body.get("model", ""), + model_used=body.get("model", ""), + tokenizer_type="openai_api", + ) + ) + ) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-test", request_route=path) + app.dependency_overrides[_proxy_token_counter] = lambda: token_counter_mock + try: + client = TestClient(app) + response = client.post(path, json=body, headers={"Authorization": "Bearer sk-1234"}) + return response, token_counter_mock + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + app.dependency_overrides.pop(_proxy_token_counter, None) + + def test_string_input_returns_openai_input_tokens_shape(self): + response, counter = self._post_input_tokens({"model": "gpt-4o", "input": "Hello, how are you?"}) + + assert response.status_code == 200, response.text + assert response.json() == {"object": "response.input_tokens", "input_tokens": 13} + counter.assert_awaited_once() + assert counter.call_args.kwargs["call_endpoint"] is True + token_request = counter.call_args.kwargs["request"] + assert token_request.model == "gpt-4o" + assert token_request.messages == [{"role": "user", "content": "Hello, how are you?"}] + + def test_every_route_alias_is_registered(self): + for path in ("/v1/responses/input_tokens", "/responses/input_tokens", "/openai/v1/responses/input_tokens"): + response, _ = self._post_input_tokens({"model": "gpt-4o", "input": "hi"}, path=path) + assert response.status_code == 200, f"{path}: {response.status_code} {response.text}" + + def test_input_items_instructions_and_tools_are_forwarded(self): + tools = [ + { + "type": "function", + "name": "get_weather", + "description": "Get weather for a city", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + } + ] + response, counter = self._post_input_tokens( + { + "model": "gpt-4o", + "input": [{"role": "user", "content": "What is the weather in Paris?"}], + "instructions": "You are terse.", + "tools": tools, + } + ) + + assert response.status_code == 200, response.text + token_request = counter.call_args.kwargs["request"] + assert token_request.messages == [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "What is the weather in Paris?"}, + ] + assert token_request.tools == tools + + def test_missing_model_returns_openai_400(self): + response, counter = self._post_input_tokens({"input": "Hello"}) + + assert response.status_code == 400, response.text + assert response.json() == { + "error": { + "message": "Missing required parameter: 'model'.", + "type": "invalid_request_error", + "param": "model", + "code": "missing_required_parameter", + } + } + counter.assert_not_awaited() + + def test_missing_input_returns_openai_400(self): + response, counter = self._post_input_tokens({"model": "gpt-4o"}) + + assert response.status_code == 400, response.text + assert response.json() == { + "error": { + "message": "Missing required parameter: 'input'.", + "type": "invalid_request_error", + "param": "input", + "code": "missing_required_parameter", + } + } + counter.assert_not_awaited() + + @pytest.mark.parametrize("empty_input", ["", []]) + def test_empty_input_returns_openai_400(self, empty_input): + response, counter = self._post_input_tokens({"model": "gpt-4o", "input": empty_input}) + + assert response.status_code == 400, response.text + assert response.json() == { + "error": { + "message": """One of "input" or "previous_response_id" or 'prompt' or 'conversation' must be provided.""", + "type": "invalid_request_error", + "param": None, + "code": "missing_required_parameter", + } + } + counter.assert_not_awaited() + + def test_invalid_tools_returns_openai_400(self): + response, counter = self._post_input_tokens({"model": "gpt-4o", "input": "hi", "tools": "not-a-list"}) + + assert response.status_code == 400, response.text + error = response.json()["error"] + assert error["type"] == "invalid_request_error" + counter.assert_not_awaited() + + def test_provider_error_maps_status_code(self): + from litellm.proxy._types import ProxyException + + failing_counter = AsyncMock( + side_effect=ProxyException( + message="rate limited", + type="token_counting_error", + param="model", + code="429", + ) + ) + response, _ = self._post_input_tokens({"model": "gpt-4o", "input": "hi"}, counter=failing_counter) + + assert response.status_code == 429, response.text + assert response.json()["error"]["message"] == "rate limited" diff --git a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py new file mode 100644 index 00000000000..f65f68812a2 --- /dev/null +++ b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py @@ -0,0 +1,48 @@ +from typing import Final + +import pytest + +from litellm.caching import DualCache +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.spend_tracking.budget_reservation import reserve_budget_for_request +from litellm.proxy.utils import ProxyLogging + +TOKEN_COUNTING_ROUTES: Final = ( + "/responses/input_tokens", + "/v1/responses/input_tokens", + "/openai/v1/responses/input_tokens", + "/utils/token_counter", +) + + +def _budgeted_token() -> UserAPIKeyAuth: + return UserAPIKeyAuth(api_key="sk-test", token="hashed-token", max_budget=100.0, spend=0.0) + + +async def _reserve(route: str) -> dict | None: + return await reserve_budget_for_request( + request_body={"model": "gpt-4o", "input": "hello"}, + route=route, + llm_router=None, + valid_token=_budgeted_token(), + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=UserApiKeyCache(), + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", TOKEN_COUNTING_ROUTES) +async def test_token_counting_routes_are_exempt_from_budget_reservation(route): + assert await _reserve(route) is None + + +@pytest.mark.asyncio +async def test_non_exempt_llm_route_still_reserves_budget(): + reservation: Final = await _reserve("/v1/responses") + + assert reservation is not None + assert reservation["reserved_cost"] > 0 diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index b96d2eb5322..bb59e576568 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -124,6 +124,25 @@ class TestLiteLLMCompletionResponsesConfig: assert "extra_field" not in result["file"] assert "another_field" not in result["file"] + def test_transform_input_file_item_to_file_item_keeps_filename(self): + """OpenAI rejects file_data with no filename beside it, so dropping it 400s the request""" + result = ( + LiteLLMCompletionResponsesConfig._transform_input_file_item_to_file_item( + { + "type": "input_file", + "filename": "report.pdf", + "file_data": "data:application/pdf;base64,JVBERi0=", + } + ) + ) + assert result == { + "type": "file", + "file": { + "file_data": "data:application/pdf;base64,JVBERi0=", + "filename": "report.pdf", + }, + } + def test_transform_input_file_item_to_file_item_with_file_url(self): """file_url should be mapped to file_id for downstream URL handling""" result = ( diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 1ed5366dac1..6f6c3165e4b 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -9700,6 +9700,37 @@ export interface paths { patch?: never; trace?: never; }; + "/openai/v1/responses/input_tokens": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Responses Input Tokens + * @description Count the input tokens of a Responses API request without calling the model. + * + * Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens + * + * ```bash + * curl -X POST http://localhost:4000/v1/responses/input_tokens -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{ + * "model": "gpt-4o", + * "input": "Hello, how are you?" + * }' + * ``` + * + * Returns: `{"object": "response.input_tokens", "input_tokens": }` + */ + post: operations["responses_input_tokens_openai_v1_responses_input_tokens_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/openai/v1/responses/{response_id}": { parameters: { query?: never; @@ -12619,6 +12650,37 @@ export interface paths { patch?: never; trace?: never; }; + "/responses/input_tokens": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Responses Input Tokens + * @description Count the input tokens of a Responses API request without calling the model. + * + * Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens + * + * ```bash + * curl -X POST http://localhost:4000/v1/responses/input_tokens -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{ + * "model": "gpt-4o", + * "input": "Hello, how are you?" + * }' + * ``` + * + * Returns: `{"object": "response.input_tokens", "input_tokens": }` + */ + post: operations["responses_input_tokens_responses_input_tokens_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/responses/{response_id}": { parameters: { query?: never; @@ -19194,6 +19256,37 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/responses/input_tokens": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Responses Input Tokens + * @description Count the input tokens of a Responses API request without calling the model. + * + * Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens + * + * ```bash + * curl -X POST http://localhost:4000/v1/responses/input_tokens -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{ + * "model": "gpt-4o", + * "input": "Hello, how are you?" + * }' + * ``` + * + * Returns: `{"object": "response.input_tokens", "input_tokens": }` + */ + post: operations["responses_input_tokens_v1_responses_input_tokens_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/responses/{response_id}": { parameters: { query?: never; @@ -51496,6 +51589,26 @@ export interface operations { }; }; }; + responses_input_tokens_openai_v1_responses_input_tokens_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; get_response_openai_v1_responses__response_id__get: { parameters: { query?: never; @@ -54460,6 +54573,26 @@ export interface operations { }; }; }; + responses_input_tokens_responses_input_tokens_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; get_response_responses__response_id__get: { parameters: { query?: never; @@ -62882,6 +63015,26 @@ export interface operations { }; }; }; + responses_input_tokens_v1_responses_input_tokens_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; get_response_v1_responses__response_id__get: { parameters: { query?: never;