scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-11 22:51:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 123

perf(proxy): lazy-load provider passthrough routes (#40691)

Browse source 
Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-09-11 09:48:53 -07:00 • committed by GitHub
parent 67cb34ceee
commit 729ea6b832
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 5317 additions and 394 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

98
litellm/proxy/_lazy_features.py
View file

@ -8,11 +8,13 @@ omits each feature's routes until the feature is warmed.
import asyncio
import importlib
from collections.abc import Callable
from collections.abc import Callable, Mapping, Sequence
from collections.abc import Set as AbstractSet
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
from starlette.routing import BaseRoute, Match
from starlette.types import Receive, Scope, Send
from litellm._logging import verbose_proxy_logger
@ -185,6 +187,31 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
module_path="litellm.proxy.management_endpoints.config_override_endpoints",
path_prefixes=("/config_overrides",),
),
LazyFeature(
name="llm_passthrough",
module_path="litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints",
path_prefixes=(
"/anthropic/",
"/assemblyai/",
"/azure/",
"/azure_ai/",
"/bedrock/",
"/cohere/",
"/comprehendmedical",
"/cursor/",
"/eu.assemblyai/",
"/gemini/",
"/gigachat/",
"/milvus/",
"/mistral/",
"/openai/",
"/openai_passthrough/",
"/vertex-ai/",
"/vertex_ai/",
"/vllm/",
"/watsonx/",
),
),
LazyFeature(
name="realtime",
module_path="litellm.proxy.realtime_endpoints.endpoints",
@ -308,14 +335,64 @@ class LazyFeatureMiddleware:
if root_path and path.startswith(root_path + "/"):
path = path[len(root_path) :] # rebind-ok: local strip after the boundary check above
for feat in self._features:
if feat.module_path in self._loaded:
if feat.module_path in self._loaded or not feat.matches(path):
continue
if feat.matches(path):
await _force_load(self._fastapi_app, feat)
if _eager_route_wins(self._fastapi_app, feat, scope):
continue
await _force_load(self._fastapi_app, feat, self._features)
await self.app(scope, receive, send)
async def _force_load(app: "FastAPI", feat: LazyFeature) -> bool:
def _lazy_slots(app: "FastAPI") -> Mapping[str, int]:
return app.state.lazy_slots if hasattr(app.state, "lazy_slots") else MappingProxyType({})
def reserve_lazy_slot(app: "FastAPI", name: str, features: tuple[LazyFeature, ...] = LAZY_FEATURES) -> None:
"""Record the table position the feature's router used to be included at, so its
routes are spliced back in there once it loads and keep the same precedence."""
feat: Final = next(f for f in features if f.name == name)
app.state.lazy_slots = MappingProxyType({**_lazy_slots(app), feat.module_path: len(app.router.routes)})
def _eager_route_wins(app: "FastAPI", feat: LazyFeature, scope: Scope) -> bool:
"""Routes ahead of a feature's reserved slot beat its routes in Starlette's scan,
so a request one of them fully matches never needs the feature loaded."""
slot: Final = _lazy_slots(app).get(feat.module_path)
if slot is None:
return False
return any(route.matches(scope)[0] is Match.FULL for route in app.router.routes[:slot])
def _in_registry_order(
routes: Sequence[BaseRoute],
lazy_routes: Mapping[str, tuple[BaseRoute, ...]],
features: tuple[LazyFeature, ...],
slots: Mapping[str, int],
) -> tuple[BaseRoute, ...]:
"""Lazy routers land in registry order, not first-request order, so overlapping
paths (/openai/{endpoint:path} vs /openai/v1/realtime/calls) resolve the same
way no matter which feature a deployment happens to hit first. Features with a
reserved slot go back where they were eagerly included; the rest follow every
eager route."""
rank: Final = MappingProxyType({f.module_path: i for i, f in enumerate(features)})
modules: Final = tuple(sorted(lazy_routes, key=lambda m: rank.get(m, len(rank))))
lazy_ids: Final = frozenset(id(route) for module_path in modules for route in lazy_routes[module_path])
eager: Final = tuple(route for route in routes if id(route) not in lazy_ids)
def slot_of(module_path: str) -> int:
return min(slots.get(module_path, len(eager)), len(eager))
return tuple(
route
for index in range(len(eager) + 1)
for route in (
*(r for module_path in modules if slot_of(module_path) == index for r in lazy_routes[module_path]),
*eager[index : index + 1],
)
)
async def _force_load(app: "FastAPI", feat: LazyFeature, features: tuple[LazyFeature, ...] = LAZY_FEATURES) -> bool:
"""Import + register a lazy feature exactly once per (app, module).
Shared by the middleware and the /lazy/warm endpoint."""
if not hasattr(app.state, "lazy_loaded"):
@ -330,7 +407,18 @@ async def _force_load(app: "FastAPI", feat: LazyFeature) -> bool:
# mutates app.router.routes, so it stays on the loop thread.
loop: Final = asyncio.get_running_loop()
module: Final = await loop.run_in_executor(None, importlib.import_module, feat.module_path)
before: Final = len(app.router.routes)
feat.register_fn(app, module)
previous: Final[Mapping[str, tuple[BaseRoute, ...]]] = (
app.state.lazy_routes if hasattr(app.state, "lazy_routes") else MappingProxyType({})
)
lazy_routes: Final[Mapping[str, tuple[BaseRoute, ...]]] = MappingProxyType(
{**previous, feat.module_path: tuple(app.router.routes[before:])}
)
app.state.lazy_routes = lazy_routes # rebind-ok: the app owns the record of which routes each feature added
app.router.routes[:] = _in_registry_order( # rebind-ok: the app owns its route table
app.router.routes, lazy_routes, features, _lazy_slots(app)
)
app.state.lazy_loaded.add(feat.module_path)
app.openapi_schema = None
verbose_proxy_logger.info(

4986
litellm/proxy/_lazy_openapi_snapshot.json
View file

File diff suppressed because it is too large Load diff

6
litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
View file

@ -97,7 +97,6 @@ else:
vertex_llm_base: Final = VertexBase()
router: Final = APIRouter()
openai_passthrough_router: Final = APIRouter()
default_vertex_config: Final = None
passthrough_endpoint_router: Final = PassthroughEndpointRouter()
@ -2297,11 +2296,6 @@ async def vertex_proxy_route(
)
@openai_passthrough_router.api_route(
"/openai_passthrough/{endpoint:path}",
methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
tags=["OpenAI Pass-through", "pass-through"],
)
@router.api_route(
"/openai/{endpoint:path}",
methods=["GET", "POST", "PUT", "DELETE", "PATCH"],

44
litellm/proxy/pass_through_endpoints/openai_passthrough_endpoints.py Normal file
View file

@ -0,0 +1,44 @@
"""/openai_passthrough must be matched ahead of the native /{provider}/v1/files and
/{provider}/v1/batches routes, so unlike the other provider passthrough routes it is
registered at startup and defers to the lazily loaded handler per call."""
from typing import Final
from fastapi import APIRouter, Depends, Request, Response
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
router: Final = APIRouter()
@router.api_route(
"/openai_passthrough/{endpoint:path}",
methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
tags=["OpenAI Pass-through", "pass-through"],
)
async def openai_passthrough_route(
endpoint: str,
request: Request,
fastapi_response: Response,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
) -> Response:
"""
Dedicated pass-through to the OpenAI API with no overlap with LiteLLM's native
implementations (e.g. the Responses API at /v1/responses).
Examples:
- /openai_passthrough/v1/responses
- /openai_passthrough/v1/responses/{response_id}
- /openai_passthrough/v1/responses/{response_id}/input_items
[Docs](https://docs.litellm.ai/docs/pass_through/openai_passthrough)
"""
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import openai_proxy_route
return await openai_proxy_route(
endpoint=endpoint,
request=request,
fastapi_response=fastapi_response,
user_api_key_dict=user_api_key_dict,
)

21
litellm/proxy/proxy_server.py
View file

@ -304,7 +304,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import (
)
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.proxy._lazy_features import attach_lazy_features
from litellm.proxy._lazy_features import attach_lazy_features, reserve_lazy_slot
from litellm.proxy._types import *
from litellm.proxy.analytics_endpoints.analytics_endpoints import (
router as analytics_router,
@ -639,13 +639,8 @@ from litellm.proxy.openai_files_endpoints.files_endpoints import (
from litellm.proxy.openai_files_endpoints.files_endpoints import (
set_files_config,
)
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
openai_passthrough_router,
passthrough_endpoint_router,
vertex_ai_live_websocket_passthrough,
)
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
router as llm_passthrough_router,
from litellm.proxy.pass_through_endpoints.openai_passthrough_endpoints import (
router as openai_passthrough_router,
)
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
initialize_pass_through_endpoints,
@ -6047,6 +6042,10 @@ class ProxyConfig:
set_files_config(config=files_config)
## default config for vertex ai routes
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
passthrough_endpoint_router,
)
default_vertex_config: Final = config.get("default_vertex_config", None)
passthrough_endpoint_router.set_default_vertex_config(config=default_vertex_config)
@ -11763,6 +11762,10 @@ async def vertex_ai_live_passthrough_endpoint(
This endpoint delegates to the WebSocket function defined in llm_passthrough_endpoints.py
"""
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
vertex_ai_live_websocket_passthrough,
)
return await vertex_ai_live_websocket_passthrough(
websocket=websocket,
model=model,
@ -18668,7 +18671,7 @@ app.include_router(credential_router)
app.include_router(openai_passthrough_router)
app.include_router(batches_router)
app.include_router(openai_files_router)
app.include_router(llm_passthrough_router)
reserve_lazy_slot(app, "llm_passthrough")
app.include_router(pass_through_router)
app.include_router(health_router)
app.include_router(key_management_router)

49
tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
View file

@ -4,7 +4,7 @@ import contextlib
import json
import os
import traceback
from collections.abc import Mapping
from collections.abc import Iterator, Mapping
from types import MappingProxyType, SimpleNamespace
from typing import Final
from unittest import mock
@ -12,7 +12,9 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch
import httpx
import pytest
import respx
from fastapi import HTTPException, Request, Response
from fastapi.routing import APIRoute
from fastapi.responses import StreamingResponse
from fastapi.testclient import TestClient
from starlette.datastructures import FormData
@ -45,6 +47,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
)
from litellm.proxy._types import LitellmUserRoles, SpecialHeaders, UserAPIKeyAuth
from litellm.proxy.auth.handle_jwt import JWTHandler
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials
@ -3339,8 +3342,11 @@ class TestOpenAIPassthroughRoute:
def _resolve_route_name(method: str, path: str) -> str | None:
from starlette.routing import Match
from litellm.proxy._lazy_features import LAZY_FEATURES, _force_load
from litellm.proxy.proxy_server import app
asyncio.run(_force_load(app, next(f for f in LAZY_FEATURES if f.name == "llm_passthrough")))
scope: Final = {
"type": "http",
"method": method,
@ -3350,8 +3356,8 @@ def _resolve_route_name(method: str, path: str) -> str | None:
"root_path": "",
}
for route in app.router.routes:
if route.matches(scope)[0] == Match.FULL:
return getattr(route, "name", None)
if isinstance(route, APIRoute) and route.matches(scope)[0] == Match.FULL:
return route.name
return None
@ -3376,7 +3382,7 @@ def test_openai_passthrough_prefix_wins_over_native_provider_routes(method, path
/{provider}/v1/files and /{provider}/v1/batches routes must never capture it
with provider="openai_passthrough" (which 500s on the LlmProviders lookup).
"""
assert _resolve_route_name(method, path) == "openai_proxy_route"
assert _resolve_route_name(method, path) == "openai_passthrough_route"
@pytest.mark.parametrize(
@ -3393,6 +3399,41 @@ def test_native_provider_routes_are_unchanged(method, path, expected_name):
assert _resolve_route_name(method, path) == expected_name
@pytest.fixture
def openai_passthrough_client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
from litellm.proxy.proxy_server import app
monkeypatch.setenv("OPENAI_API_KEY", "sk-upstream")
monkeypatch.delenv("SERVER_ROOT_PATH", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual"))
yield TestClient(app)
@pytest.mark.parametrize(
"method, path, body",
[
("POST", "/v1/responses", {"model": "gpt-5.1", "input": "hi"}),
("GET", "/v1/files", None),
("POST", "/v1/batches", {"input_file_id": "file-abc123", "endpoint": "/v1/responses"}),
],
)
def test_openai_passthrough_forwards_verbatim_to_openai(
openai_passthrough_client: TestClient, method: str, path: str, body: dict[str, str] | None
) -> None:
"""Every /openai_passthrough request, including the /v1/files and /v1/batches
paths that native provider routes also claim, must reach OpenAI unchanged."""
with respx.mock(assert_all_called=True) as upstream:
route = upstream.request(method, f"https://api.openai.com{path}").mock(
return_value=httpx.Response(200, json={"id": "upstream_123"})
)
response = openai_passthrough_client.request(method, f"/openai_passthrough{path}", json=body)
assert (response.status_code, response.json()) == (200, {"id": "upstream_123"})
assert route.calls.last.request.headers["authorization"] == "Bearer sk-upstream"
class TestCursorProxyRoute:
"""Tests for the Cursor Cloud Agents pass-through route."""

120
tests/test_litellm/proxy/test_proxy_server.py
View file

@ -9183,6 +9183,126 @@ class TestLazyFeaturesNotImportedAtStartup:
class TestLazyFeatureMiddleware:
"""Behavior of the middleware itself, exercised in isolation."""
@pytest.mark.asyncio
async def test_llm_passthrough_loads_on_first_provider_request(self, monkeypatch):
"""An app that never registered the provider passthrough routes 404s a
provider request; behind the middleware the same request registers the
routes and is forwarded to the provider with the configured key."""
import respx
from fastapi import FastAPI
from litellm.proxy._lazy_features import LAZY_FEATURES, LazyFeatureMiddleware
monkeypatch.setenv("MISTRAL_API_KEY", "sk-upstream")
monkeypatch.delenv("SERVER_ROOT_PATH", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
feat = next(f for f in LAZY_FEATURES if f.name == "llm_passthrough")
target_app = FastAPI()
target_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-virtual")
mw = LazyFeatureMiddleware(target_app, fastapi_app=target_app, features=(feat,))
with respx.mock() as upstream:
route = upstream.get("https://api.mistral.ai/v1/models").mock(
return_value=httpx.Response(200, json={"object": "list", "data": []})
)
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=target_app), base_url="http://t") as bare:
assert (await bare.get("/mistral/v1/models")).status_code == 404
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=mw), base_url="http://t") as lazy:
response = await lazy.get("/mistral/v1/models")
assert (response.status_code, response.json()) == (200, {"object": "list", "data": []})
assert route.calls.last.request.headers["authorization"] == "Bearer sk-upstream"
def test_llm_passthrough_prefixes_cover_every_route_the_module_registers(self):
"""A route the module registers under a prefix the feature does not claim
would 404 until an unrelated provider request happens to load the module."""
from litellm.proxy._lazy_features import LAZY_FEATURES
feat = next(f for f in LAZY_FEATURES if f.name == "llm_passthrough")
paths = [r.path for r in importlib.import_module(feat.module_path).router.routes]
assert {"/mistral/{endpoint:path}", "/openai/{endpoint:path}"} <= set(paths)
unreachable = [p for p in paths if not feat.matches(p.replace("{endpoint:path}", "x"))]
assert unreachable == [], f"routes the middleware would never load: {unreachable}"
@pytest.mark.asyncio
@pytest.mark.parametrize("first_hit", ["/v1/realtime/calls", "/openai/v1/models"])
async def test_lazy_routes_land_in_registry_order_not_first_hit_order(self, first_hit):
"""Two lazy features with overlapping paths must answer a request with the
same handler no matter which one a deployment happens to hit first."""
from fastapi import APIRouter, FastAPI
from litellm.proxy._lazy_features import LazyFeature, LazyFeatureMiddleware
def make_register(path, handler):
def register(app, module):
router = APIRouter()
router.add_api_route(path, lambda: {"handler": handler}, methods=["POST"])
app.include_router(router)
return register
catch_all = LazyFeature(
name="catch_all",
module_path="json",
path_prefixes=("/openai/",),
register_fn=make_register("/openai/{endpoint:path}", "catch_all"),
)
specific = LazyFeature(
name="specific",
module_path="base64",
path_prefixes=("/openai/v1/realtime", "/v1/realtime"),
register_fn=make_register("/openai/v1/realtime/calls", "specific"),
)
target_app = FastAPI()
mw = LazyFeatureMiddleware(target_app, fastapi_app=target_app, features=(catch_all, specific))
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=mw), base_url="http://t") as client:
await client.post(first_hit)
await client.post("/openai/v1/models")
response = await client.post("/openai/v1/realtime/calls")
assert response.json() == {"handler": "catch_all"}
@pytest.mark.asyncio
@pytest.mark.parametrize("root_path", ["", "/api"])
async def test_reserved_slot_keeps_lazy_catch_all_ahead_of_later_eager_routes(self, root_path):
"""/{mcp_server_name}/mcp is registered after the provider passthrough router
at startup, so /mistral/mcp must keep reaching the provider catch-all once
that router loads lazily instead of being swallowed by the MCP route. The
native /mistral/v1/files route sits ahead of it, so that path neither loads
the feature nor changes owner, with or without a SERVER_ROOT_PATH prefix."""
from fastapi import APIRouter, FastAPI
from litellm.proxy._lazy_features import LazyFeature, LazyFeatureMiddleware, reserve_lazy_slot
def register(app, module):
router = APIRouter()
router.add_api_route("/mistral/{endpoint:path}", lambda: {"handler": "passthrough"}, methods=["POST"])
app.include_router(router)
passthrough = LazyFeature(
name="llm_passthrough", module_path="json", path_prefixes=("/mistral/",), register_fn=register
)
target_app = FastAPI(root_path=root_path)
target_app.add_api_route("/mistral/v1/files", lambda: {"handler": "files"}, methods=["POST"])
reserve_lazy_slot(target_app, "llm_passthrough", features=(passthrough,))
target_app.add_api_route("/{mcp_server_name}/mcp", lambda: {"handler": "mcp"}, methods=["POST"])
target_app.add_middleware(LazyFeatureMiddleware, fastapi_app=target_app, features=(passthrough,))
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=target_app), base_url="http://t") as client:
files_first = (await client.post(f"{root_path}/mistral/v1/files")).json()["handler"]
loaded_after_files = frozenset(target_app.state.lazy_loaded)
handlers = [
(await client.post(f"{root_path}{path}")).json()["handler"]
for path in ("/mistral/mcp", "/mistral/v1/files")
]
assert (files_first, loaded_after_files) == ("files", frozenset())
assert handlers == ["passthrough", "files"]
@pytest.mark.asyncio
async def test_first_request_triggers_load_subsequent_does_not(self):
from fastapi import FastAPI

387
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -9547,26 +9547,6 @@ export interface paths {
patch?: never;
trace?: never;
};
"/openai/": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/**
* WebSocket: openai_websocket_proxy_route
* @description WebSocket connection endpoint
*/
get: operations["websocket_openai_websocket_proxy_route_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/openai/deployments/{model}/chat/completions": {
parameters: {
query?: never;
@ -10122,26 +10102,6 @@ export interface paths {
patch: operations["openai_proxy_route_openai__endpoint__patch"];
trace?: never;
};
"/openai_passthrough/": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/**
* WebSocket: openai_websocket_proxy_route
* @description WebSocket connection endpoint
*/
get: operations["websocket_openai_websocket_proxy_route_get_2"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/openai_passthrough/{endpoint}": {
parameters: {
query?: never;
@ -10150,132 +10110,72 @@ export interface paths {
cookie?: never;
};
/**
* Openai Proxy Route
* @description Pass-through endpoint for OpenAI API calls.
*
* Available on both routes:
* - /openai/{endpoint:path} - Standard OpenAI passthrough route
* - /openai_passthrough/{endpoint:path} - Dedicated passthrough route (recommended for Responses API)
*
* Use /openai_passthrough/* when you need guaranteed passthrough to OpenAI without conflicts
* with LiteLLM's native implementations (e.g., for the Responses API at /v1/responses).
* Openai Passthrough Route
* @description Dedicated pass-through to the OpenAI API with no overlap with LiteLLM's native
* implementations (e.g. the Responses API at /v1/responses).
*
* Examples:
* Standard route:
* - /openai/v1/chat/completions
* - /openai/v1/assistants
* - /openai/v1/threads
*
* Dedicated passthrough (for Responses API):
* - /openai_passthrough/v1/responses
* - /openai_passthrough/v1/responses/{response_id}
* - /openai_passthrough/v1/responses/{response_id}/input_items
*
* [Docs](https://docs.litellm.ai/docs/pass_through/openai_passthrough)
*/
get: operations["openai_proxy_route_openai_passthrough__endpoint__get"];
get: operations["openai_passthrough_route_openai_passthrough__endpoint__get"];
/**
* Openai Proxy Route
* @description Pass-through endpoint for OpenAI API calls.
*
* Available on both routes:
* - /openai/{endpoint:path} - Standard OpenAI passthrough route
* - /openai_passthrough/{endpoint:path} - Dedicated passthrough route (recommended for Responses API)
*
* Use /openai_passthrough/* when you need guaranteed passthrough to OpenAI without conflicts
* with LiteLLM's native implementations (e.g., for the Responses API at /v1/responses).
* Openai Passthrough Route
* @description Dedicated pass-through to the OpenAI API with no overlap with LiteLLM's native
* implementations (e.g. the Responses API at /v1/responses).
*
* Examples:
* Standard route:
* - /openai/v1/chat/completions
* - /openai/v1/assistants
* - /openai/v1/threads
*
* Dedicated passthrough (for Responses API):
* - /openai_passthrough/v1/responses
* - /openai_passthrough/v1/responses/{response_id}
* - /openai_passthrough/v1/responses/{response_id}/input_items
*
* [Docs](https://docs.litellm.ai/docs/pass_through/openai_passthrough)
*/
put: operations["openai_proxy_route_openai_passthrough__endpoint__put"];
put: operations["openai_passthrough_route_openai_passthrough__endpoint__put"];
/**
* Openai Proxy Route
* @description Pass-through endpoint for OpenAI API calls.
*
* Available on both routes:
* - /openai/{endpoint:path} - Standard OpenAI passthrough route
* - /openai_passthrough/{endpoint:path} - Dedicated passthrough route (recommended for Responses API)
*
* Use /openai_passthrough/* when you need guaranteed passthrough to OpenAI without conflicts
* with LiteLLM's native implementations (e.g., for the Responses API at /v1/responses).
* Openai Passthrough Route
* @description Dedicated pass-through to the OpenAI API with no overlap with LiteLLM's native
* implementations (e.g. the Responses API at /v1/responses).
*
* Examples:
* Standard route:
* - /openai/v1/chat/completions
* - /openai/v1/assistants
* - /openai/v1/threads
*
* Dedicated passthrough (for Responses API):
* - /openai_passthrough/v1/responses
* - /openai_passthrough/v1/responses/{response_id}
* - /openai_passthrough/v1/responses/{response_id}/input_items
*
* [Docs](https://docs.litellm.ai/docs/pass_through/openai_passthrough)
*/
post: operations["openai_proxy_route_openai_passthrough__endpoint__post"];
post: operations["openai_passthrough_route_openai_passthrough__endpoint__post"];
/**
* Openai Proxy Route
* @description Pass-through endpoint for OpenAI API calls.
*
* Available on both routes:
* - /openai/{endpoint:path} - Standard OpenAI passthrough route
* - /openai_passthrough/{endpoint:path} - Dedicated passthrough route (recommended for Responses API)
*
* Use /openai_passthrough/* when you need guaranteed passthrough to OpenAI without conflicts
* with LiteLLM's native implementations (e.g., for the Responses API at /v1/responses).
* Openai Passthrough Route
* @description Dedicated pass-through to the OpenAI API with no overlap with LiteLLM's native
* implementations (e.g. the Responses API at /v1/responses).
*
* Examples:
* Standard route:
* - /openai/v1/chat/completions
* - /openai/v1/assistants
* - /openai/v1/threads
*
* Dedicated passthrough (for Responses API):
* - /openai_passthrough/v1/responses
* - /openai_passthrough/v1/responses/{response_id}
* - /openai_passthrough/v1/responses/{response_id}/input_items
*
* [Docs](https://docs.litellm.ai/docs/pass_through/openai_passthrough)
*/
delete: operations["openai_proxy_route_openai_passthrough__endpoint__delete"];
delete: operations["openai_passthrough_route_openai_passthrough__endpoint__delete"];
options?: never;
head?: never;
/**
* Openai Proxy Route
* @description Pass-through endpoint for OpenAI API calls.
*
* Available on both routes:
* - /openai/{endpoint:path} - Standard OpenAI passthrough route
* - /openai_passthrough/{endpoint:path} - Dedicated passthrough route (recommended for Responses API)
*
* Use /openai_passthrough/* when you need guaranteed passthrough to OpenAI without conflicts
* with LiteLLM's native implementations (e.g., for the Responses API at /v1/responses).
* Openai Passthrough Route
* @description Dedicated pass-through to the OpenAI API with no overlap with LiteLLM's native
* implementations (e.g. the Responses API at /v1/responses).
*
* Examples:
* Standard route:
* - /openai/v1/chat/completions
* - /openai/v1/assistants
* - /openai/v1/threads
*
* Dedicated passthrough (for Responses API):
* - /openai_passthrough/v1/responses
* - /openai_passthrough/v1/responses/{response_id}
* - /openai_passthrough/v1/responses/{response_id}/input_items
*
* [Docs](https://docs.litellm.ai/docs/pass_through/openai_passthrough)
*/
patch: operations["openai_proxy_route_openai_passthrough__endpoint__patch"];
patch: operations["openai_passthrough_route_openai_passthrough__endpoint__patch"];
trace?: never;
};
"/organization/daily/activity": {
@ -21891,52 +21791,6 @@ export interface paths {
patch?: never;
trace?: never;
};
"/vertex-ai/{endpoint}": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/**
* Vertex Proxy Route
* @description Call LiteLLM proxy via Vertex AI SDK.
*
* [Docs](https://docs.litellm.ai/docs/pass_through/vertex_ai)
*/
get: operations["vertex_proxy_route_vertex_ai__endpoint__get_2"];
/**
* Vertex Proxy Route
* @description Call LiteLLM proxy via Vertex AI SDK.
*
* [Docs](https://docs.litellm.ai/docs/pass_through/vertex_ai)
*/
put: operations["vertex_proxy_route_vertex_ai__endpoint__put_2"];
/**
* Vertex Proxy Route
* @description Call LiteLLM proxy via Vertex AI SDK.
*
* [Docs](https://docs.litellm.ai/docs/pass_through/vertex_ai)
*/
post: operations["vertex_proxy_route_vertex_ai__endpoint__post_2"];
/**
* Vertex Proxy Route
* @description Call LiteLLM proxy via Vertex AI SDK.
*
* [Docs](https://docs.litellm.ai/docs/pass_through/vertex_ai)
*/
delete: operations["vertex_proxy_route_vertex_ai__endpoint__delete_2"];
options?: never;
head?: never;
/**
* Vertex Proxy Route
* @description Call LiteLLM proxy via Vertex AI SDK.
*
* [Docs](https://docs.litellm.ai/docs/pass_through/vertex_ai)
*/
patch: operations["vertex_proxy_route_vertex_ai__endpoint__patch_2"];
trace?: never;
};
"/vertex_ai/discovery/{endpoint}": {
parameters: {
query?: never;
@ -52513,24 +52367,6 @@ export interface operations {
};
};
};
websocket_openai_websocket_proxy_route_get: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description WebSocket Protocol Switched */
101: {
headers: {
[name: string]: unknown;
};
content?: never;
};
};
};
chat_completion_openai_deployments__model__chat_completions_post: {
parameters: {
query?: never;
@ -53430,25 +53266,7 @@ export interface operations {
};
};
};
websocket_openai_websocket_proxy_route_get_2: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description WebSocket Protocol Switched */
101: {
headers: {
[name: string]: unknown;
};
content?: never;
};
};
};
openai_proxy_route_openai_passthrough__endpoint__get: {
openai_passthrough_route_openai_passthrough__endpoint__get: {
parameters: {
query?: never;
header?: never;
@ -53479,7 +53297,7 @@ export interface operations {
};
};
};
openai_proxy_route_openai_passthrough__endpoint__put: {
openai_passthrough_route_openai_passthrough__endpoint__put: {
parameters: {
query?: never;
header?: never;
@ -53510,7 +53328,7 @@ export interface operations {
};
};
};
openai_proxy_route_openai_passthrough__endpoint__post: {
openai_passthrough_route_openai_passthrough__endpoint__post: {
parameters: {
query?: never;
header?: never;
@ -53541,7 +53359,7 @@ export interface operations {
};
};
};
openai_proxy_route_openai_passthrough__endpoint__delete: {
openai_passthrough_route_openai_passthrough__endpoint__delete: {
parameters: {
query?: never;
header?: never;
@ -53572,7 +53390,7 @@ export interface operations {
};
};
};
openai_proxy_route_openai_passthrough__endpoint__patch: {
openai_passthrough_route_openai_passthrough__endpoint__patch: {
parameters: {
query?: never;
header?: never;
@ -67979,161 +67797,6 @@ export interface operations {
};
};
};
vertex_proxy_route_vertex_ai__endpoint__get_2: {
parameters: {
query?: never;
header?: never;
path: {
endpoint: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
vertex_proxy_route_vertex_ai__endpoint__put_2: {
parameters: {
query?: never;
header?: never;
path: {
endpoint: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
vertex_proxy_route_vertex_ai__endpoint__post_2: {
parameters: {
query?: never;
header?: never;
path: {
endpoint: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
vertex_proxy_route_vertex_ai__endpoint__delete_2: {
parameters: {
query?: never;
header?: never;
path: {
endpoint: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
vertex_proxy_route_vertex_ai__endpoint__patch_2: {
parameters: {
query?: never;
header?: never;
path: {
endpoint: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
vertex_discovery_proxy_route_vertex_ai_discovery__endpoint__get: {
parameters: {
query?: never;