scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-14 09:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 214

Merge 121a0dffaf into aeede53112

Browse source
This commit is contained in:
Clément Fiere 2026-11-06 15:51:03 -05:00 • committed by GitHub
commit e6ded4131f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 193 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
litellm/proxy/_types.py
View file

@ -352,18 +352,23 @@ class LiteLLMRoutes(enum.Enum):
"/responses",
"/v1/responses",
"/openai/v1/responses",
"/openai/responses",
"/responses/{response_id}",
"/v1/responses/{response_id}",
"/openai/v1/responses/{response_id}",
"/openai/responses/{response_id}",
"/responses/{response_id}/input_items",
"/v1/responses/{response_id}/input_items",
"/openai/v1/responses/{response_id}/input_items",
"/openai/responses/{response_id}/input_items",
"/responses/{response_id}/cancel",
"/v1/responses/{response_id}/cancel",
"/openai/v1/responses/{response_id}/cancel",
"/openai/responses/{response_id}/cancel",
"/responses/input_tokens",
"/v1/responses/input_tokens",
"/openai/v1/responses/input_tokens",
"/openai/responses/input_tokens",
# vector stores
"/vector_stores",
"/v1/vector_stores",

1
litellm/proxy/agent_endpoints/auth/managed_authorization.py
View file

@ -47,6 +47,7 @@ _MANAGED_MODEL_ROUTES: Final = frozenset(
) | frozenset(
(
"/openai/v1/responses",
"/openai/responses",
"/v2/rerank",
"/claude_code_gateway/v1/messages",
"/claude_code_gateway/v1/messages/count_tokens",

9
litellm/proxy/auth/auth_utils.py
View file

@ -1030,12 +1030,9 @@ def normalize_request_route(route: str) -> str:
# Format: (regex_pattern, replacement_template)
patterns: Final = [
# Responses API - must come before generic patterns
(r"^(/(?:openai/)?v1/responses)/([^/]+)(/input_items)$", r"\1/{response_id}\3"),
(r"^(/(?:openai/)?v1/responses)/([^/]+)(/cancel)$", r"\1/{response_id}\3"),
(r"^(/(?:openai/)?v1/responses)/([^/]+)$", r"\1/{response_id}"),
(r"^(/responses)/([^/]+)(/input_items)$", r"\1/{response_id}\3"),
(r"^(/responses)/([^/]+)(/cancel)$", r"\1/{response_id}\3"),
(r"^(/responses)/([^/]+)$", r"\1/{response_id}"),
(r"^(/(?:openai/)?(?:v1/)?responses)/([^/]+)(/input_items)$", r"\1/{response_id}\3"),
(r"^(/(?:openai/)?(?:v1/)?responses)/([^/]+)(/cancel)$", r"\1/{response_id}\3"),
(r"^(/(?:openai/)?(?:v1/)?responses)/([^/]+)$", r"\1/{response_id}"),
# Threads API
(
r"^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)(/steps)/([^/]+)$",

1
litellm/proxy/auth/route_checks.py
View file

@ -103,6 +103,7 @@ _UNBILLED_ROUTES: Final[frozenset[str]] = frozenset(
"/responses/input_tokens",
"/v1/responses/input_tokens",
"/openai/v1/responses/input_tokens",
"/openai/responses/input_tokens",
}
)
_TOKEN_COUNTING_SEGMENTS: Final[frozenset[str]] = frozenset({"count_tokens", "count-tokens"})

42
litellm/proxy/response_api_endpoints/endpoints.py
View file

@ -312,6 +312,12 @@ async def _resolve_cursor_model_variant_before_auth(request: Request) -> None:
tags=["responses"],
responses=RESPONSES_API_RESPONSE_SCHEMAS,
)
@router.post(
"/openai/responses",
dependencies=[Depends(user_api_key_auth)],
tags=["responses"],
include_in_schema=False,
)
async def responses_api(
request: Request,
fastapi_response: Response,
@ -809,6 +815,12 @@ async def cursor_chat_completions(
responses=RESPONSES_API_RESPONSE_SCHEMAS,
openapi_extra=_RETRIEVE_QUERY_OPENAPI,
)
@router.get(
"/openai/responses/{response_id}",
dependencies=[Depends(user_api_key_auth)],
tags=["responses"],
include_in_schema=False,
)
async def get_response(
response_id: str,
request: Request,
@ -1240,6 +1252,12 @@ async def _bill_finished_stream(
tags=["responses"],
responses=DELETE_RESPONSE_SCHEMAS,
)
@router.delete(
"/openai/responses/{response_id}",
dependencies=[Depends(user_api_key_auth)],
tags=["responses"],
include_in_schema=False,
)
async def delete_response(
response_id: str,
request: Request,
@ -1356,6 +1374,12 @@ async def delete_response(
tags=["responses"],
responses=RESPONSE_ITEM_LIST_SCHEMAS,
)
@router.get(
"/openai/responses/{response_id}/input_items",
dependencies=[Depends(user_api_key_auth)],
tags=["responses"],
include_in_schema=False,
)
async def get_response_input_items(
response_id: str,
request: Request,
@ -1424,6 +1448,12 @@ async def get_response_input_items(
dependencies=[Depends(user_api_key_auth)],
tags=["responses"],
)
@router.post(
"/openai/responses/compact",
dependencies=[Depends(user_api_key_auth)],
tags=["responses"],
include_in_schema=False,
)
async def compact_response(
request: Request,
fastapi_response: Response,
@ -1736,6 +1766,12 @@ def _responses_input_as_token_count_messages(
dependencies=(_user_api_key_auth_dep,),
tags=_RESPONSES_TAGS,
)
@router.post(
"/openai/responses/input_tokens",
dependencies=(_user_api_key_auth_dep,),
tags=_RESPONSES_TAGS,
include_in_schema=False,
)
async def responses_input_tokens(
request: Request,
token_counter: _TokenCounter = _token_counter_dep,
@ -1868,6 +1904,12 @@ async def responses_input_tokens(
dependencies=[Depends(user_api_key_auth)],
tags=["responses"],
)
@router.post(
"/openai/responses/{response_id}/cancel",
dependencies=[Depends(user_api_key_auth)],
tags=["responses"],
include_in_schema=False,
)
async def cancel_response(
response_id: str,
request: Request,

3
litellm/types/utils.py
View file

@ -1141,12 +1141,15 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = {
"/responses": (CallTypes.aresponses, CallTypes.responses),
"/v1/responses": (CallTypes.aresponses, CallTypes.responses),
"/openai/v1/responses": (CallTypes.aresponses, CallTypes.responses),
"/openai/responses": (CallTypes.aresponses, CallTypes.responses),
"/responses/{response_id}": (CallTypes.aresponses, CallTypes.responses),
"/v1/responses/{response_id}": (CallTypes.aresponses, CallTypes.responses),
"/openai/v1/responses/{response_id}": (CallTypes.aresponses, CallTypes.responses),
"/openai/responses/{response_id}": (CallTypes.aresponses, CallTypes.responses),
"/responses/{response_id}/input_items": (CallTypes.alist_input_items,),
"/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,),
"/openai/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,),
"/openai/responses/{response_id}/input_items": (CallTypes.alist_input_items,),
# Realtime API
"/realtime": (CallTypes.arealtime,),
"/v1/realtime": (CallTypes.arealtime,),

3
tests/unit/integrations/test_prometheus_labels.py
View file

@ -463,6 +463,9 @@ def test_route_normalization_for_sub_routes():
"/openai/v1/responses/id4/input_items",
"/openai/v1/responses/{response_id}/input_items",
),
("/openai/responses/id5/cancel", "/openai/responses/{response_id}/cancel"),
("/openai/responses/id6/input_items", "/openai/responses/{response_id}/input_items"),
("/openai/responses/id7", "/openai/responses/{response_id}"),
]
for original, expected in sub_routes:

137
tests/unit/proxy/response_api_endpoints/test_endpoints.py
View file

@ -5273,7 +5273,12 @@ class TestResponsesInputTokens:
assert token_request.messages == [{"role": "user", "content": "Hello, how are you?"}]
def test_every_route_alias_is_registered(self):
for path in ("/v1/responses/input_tokens", "/responses/input_tokens", "/openai/v1/responses/input_tokens"):
for path in (
"/v1/responses/input_tokens",
"/responses/input_tokens",
"/openai/v1/responses/input_tokens",
"/openai/responses/input_tokens",
):
response, _ = self._post_input_tokens({"model": "gpt-4o", "input": "hi"}, path=path)
assert response.status_code == 200, f"{path}: {response.status_code} {response.text}"
@ -5498,7 +5503,13 @@ class TestResponsesInputTokens:
assert sent == [{"model": "gpt-4o-mini", "input": "hi"}]
@pytest.mark.parametrize(
"path", ["/v1/responses/input_tokens", "/responses/input_tokens", "/openai/v1/responses/input_tokens"]
"path",
[
"/v1/responses/input_tokens",
"/responses/input_tokens",
"/openai/v1/responses/input_tokens",
"/openai/responses/input_tokens",
],
)
def test_count_logs_one_line_with_the_callers_call_id_model_and_count(
self,
@ -5977,3 +5988,125 @@ def test_input_token_count_reads_a_compaction_item_as_its_summary():
assert _responses_input_as_token_count_messages(_COMPACTED_INPUT, None) == (
_responses_input_as_token_count_messages(_SUMMARY_INPUT, None)
)
_AZURE_SDK_RESOURCE: Final = "my-resource.openai.azure.com"
_AZURE_SDK_RESPONSE: Final = {
"id": "resp_azure_sdk",
"object": "response",
"created_at": 1,
"status": "completed",
"model": "my-deployment",
"output": [
{
"type": "message",
"id": "msg_1",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "Done.", "annotations": []}],
}
],
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
"usage": {"input_tokens": 10, "output_tokens": 2, "total_tokens": 12},
}
_AZURE_SDK_RESPONSE_PATHS: Final = (
("GET", "/openai/responses/{id}", ""),
("DELETE", "/openai/responses/{id}", ""),
("GET", "/openai/responses/{id}/input_items", "/input_items"),
("POST", "/openai/responses/{id}/cancel", "/cancel"),
)
def _azure_lifecycle_response(request: httpx.Request) -> httpx.Response:
if request.method == "DELETE":
return httpx.Response(200, json={"id": "resp_azure_sdk", "object": "response", "deleted": True})
if request.url.path.endswith("/input_items"):
return httpx.Response(200, json={"object": "list", "data": [], "has_more": False})
return httpx.Response(200, json=_AZURE_SDK_RESPONSE)
@pytest.fixture
def azure_sdk_proxy(monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter) -> TestClient:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm.proxy.proxy_server, "master_key", MASTER_KEY)
monkeypatch.setattr(litellm.proxy.proxy_server, "prisma_client", None)
monkeypatch.setattr(litellm.proxy.proxy_server, "general_settings", {})
monkeypatch.setattr(
litellm.proxy.proxy_server,
"llm_router",
litellm.Router(
model_list=[
{
"model_name": "my-model",
"litellm_params": {
"model": "azure/my-deployment",
"api_base": f"https://{_AZURE_SDK_RESOURCE}",
"api_key": "azure-deployment-key",
"api_version": "2025-04-01-preview",
},
}
]
),
)
litellm.in_memory_llm_clients_cache.flush_cache()
respx_mock.route(host="testserver").pass_through()
return TestClient(app, headers={"Authorization": f"Bearer {MASTER_KEY}"})
def test_azure_sdk_responses_path_reaches_the_azure_deployment(
azure_sdk_proxy: TestClient, respx_mock: respx.MockRouter
) -> None:
azure: Final = respx_mock.post(host=_AZURE_SDK_RESOURCE, path__regex=r"/responses$").respond(
json=_AZURE_SDK_RESPONSE
)
response: Final = azure_sdk_proxy.post(
"/openai/responses?api-version=2025-04-01-preview", json={"model": "my-model", "input": "hi"}
)
assert response.status_code == 200, response.text
assert response.json()["output"][0]["content"][0]["text"] == "Done."
assert azure.call_count == 1
assert azure.calls[0].request.headers["api-key"] == "azure-deployment-key"
assert json.loads(azure.calls[0].request.content)["model"] == "my-deployment"
@pytest.mark.parametrize(("method", "path", "azure_suffix"), _AZURE_SDK_RESPONSE_PATHS)
def test_azure_sdk_response_lifecycle_paths_reach_the_creating_deployment(
azure_sdk_proxy: TestClient, respx_mock: respx.MockRouter, method: str, path: str, azure_suffix: str
) -> None:
respx_mock.post(host=_AZURE_SDK_RESOURCE, path__regex=r"/responses$").respond(json=_AZURE_SDK_RESPONSE)
lifecycle: Final = respx_mock.route(
host=_AZURE_SDK_RESOURCE, path__regex=rf"/responses/resp_azure_sdk{azure_suffix}$"
).mock(side_effect=_azure_lifecycle_response)
created: Final = azure_sdk_proxy.post("/openai/responses", json={"model": "my-model", "input": "hi"})
assert created.status_code == 200, created.text
response: Final = azure_sdk_proxy.request(method, path.format(id=created.json()["id"]))
assert response.status_code == 200, response.text
assert [call.request.method for call in lifecycle.calls] == [method]
@pytest.mark.parametrize(
("method", "path"),
(
("POST", "/openai/responses"),
("POST", "/openai/responses/compact"),
("POST", "/openai/responses/input_tokens"),
*((method, path.format(id="resp_azure_sdk")) for method, path, _ in _AZURE_SDK_RESPONSE_PATHS),
),
)
def test_azure_sdk_responses_paths_require_a_virtual_key(
azure_sdk_proxy: TestClient, respx_mock: respx.MockRouter, method: str, path: str
) -> None:
azure: Final = respx_mock.route(host=_AZURE_SDK_RESOURCE).respond(json=_AZURE_SDK_RESPONSE)
response: Final = azure_sdk_proxy.request(
method, path, json={"model": "my-model", "input": "hi"}, headers={"Authorization": ""}
)
assert response.status_code == 401, response.text
assert not azure.called