scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 133

Merge f0c435b036 into 22a349ee70

Browse source
This commit is contained in:
halfcrazy 2026-08-27 19:47:39 -05:00 • committed by GitHub
commit 974918a5c5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 174 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
litellm/proxy/_types.py
View file

@ -2206,6 +2206,10 @@ class PassThroughGenericEndpoint(LiteLLMPydanticObjectBase):
default=0.0,
description="The USD cost per request to the target endpoint. This is used to calculate the cost of the request to the target endpoint.",
)
model: str | None = Field(
default=None,
description="Optional model name written to spend logs for this endpoint. Use when the pass-through is not an LLM completion but still needs per-model metering (e.g. multi-instance document parsers). Request-body `model` still takes precedence when present.",
)
timeout: float | None = Field(
default=None,
description="Upstream request timeout in seconds for this pass-through endpoint. If unset, uses general_settings.pass_through_request_timeout (default 600).",

25
litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
View file

@ -817,6 +817,7 @@ async def pass_through_request(
custom_llm_provider: str | None = None,
guardrails_config: dict | None = None,
timeout: float | None = None,
model: str | None = None,
):
"""
Pass through endpoint handler, makes the httpx request for pass-through endpoints and ensures logging hooks are called
@ -837,6 +838,7 @@ async def pass_through_request(
guardrails_config: Optional field - guardrails configuration for passthrough endpoint
timeout: Optional per-endpoint timeout in seconds. Falls back to
general_settings.pass_through_request_timeout, then 600s.
model: Optional field - model name for spend logs when the request body has none
"""
from litellm.exceptions import ModifyResponseException
from litellm.litellm_core_utils.litellm_logging import Logging
@ -924,9 +926,8 @@ async def pass_through_request(
verbose_proxy_logger.debug("Added guardrails to passthrough request metadata: %s", guardrails_to_run)
## LOGGING OBJECT ## - initialize before pre_call_hook so guardrails can access it
# Surface the requested model (when the body carries one) so logging/spans
# read e.g. ``chat gpt-4o`` instead of ``chat unknown``.
passthrough_model: Final = (_parsed_body.get("model") if isinstance(_parsed_body, dict) else None) or "unknown"
body_model: Final = _parsed_body.get("model") if isinstance(_parsed_body, dict) else None
passthrough_model: Final = body_model or model or "unknown"
start_time: Final = datetime.now()
logging_obj = Logging(
model=passthrough_model,
@ -1720,6 +1721,7 @@ def create_pass_through_route(
guardrails: dict[str, object] | None = None,
config_file_path: str | None = None,
timeout: float | None = None,
model: str | None = None,
):
# check if target is an adapter.py or a url
from litellm._uuid import uuid
@ -1797,6 +1799,7 @@ def create_pass_through_route(
"cost_per_request": cost_per_request,
"guardrails": None,
"timeout": timeout,
"model": model,
}
if passthrough_params is not None:
@ -1811,6 +1814,8 @@ def create_pass_through_route(
param_guardrails: Final = target_params.get("guardrails", None)
param_default_query_params: Final = target_params.get("default_query_params", None)
param_timeout: Final = target_params.get("timeout", timeout)
_raw_model: Final = target_params.get("model", model)
param_model: Final[str | None] = _raw_model if isinstance(_raw_model, str) else None
# Construct the full target URL with subpath if needed
full_target: Final = HttpPassThroughEndpointHelpers.construct_target_url_with_subpath(
@ -1859,6 +1864,7 @@ def create_pass_through_route(
custom_llm_provider=custom_llm_provider,
guardrails_config=cast(dict | None, param_guardrails),
timeout=cast(float | None, param_timeout),
model=param_model,
)
finally:
if hasattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY):
@ -2670,6 +2676,7 @@ class InitPassThroughEndpointHelpers:
config_file_path: str | None = None,
auth: bool = False,
timeout: float | None = None,
model: str | None = None,
):
"""Add exact path route for pass-through endpoint"""
# Default to all methods if none specified (backward compatibility)
@ -2711,6 +2718,7 @@ class InitPassThroughEndpointHelpers:
guardrails=guardrails,
config_file_path=config_file_path,
timeout=timeout,
model=model,
),
methods=methods,
dependencies=dependencies,
@ -2733,6 +2741,7 @@ class InitPassThroughEndpointHelpers:
"cost_per_request": cost_per_request,
"guardrails": guardrails,
"timeout": timeout,
"model": model,
},
}
@ -2753,6 +2762,7 @@ class InitPassThroughEndpointHelpers:
config_file_path: str | None = None,
auth: bool = False,
timeout: float | None = None,
model: str | None = None,
):
"""Add wildcard route for sub-paths"""
# Default to all methods if none specified (backward compatibility)
@ -2795,6 +2805,7 @@ class InitPassThroughEndpointHelpers:
guardrails=guardrails,
config_file_path=config_file_path,
timeout=timeout,
model=model,
),
methods=methods,
dependencies=dependencies,
@ -2817,6 +2828,7 @@ class InitPassThroughEndpointHelpers:
"cost_per_request": cost_per_request,
"guardrails": guardrails,
"timeout": timeout,
"model": model,
},
}
@ -2991,6 +3003,7 @@ async def _register_pass_through_endpoint(
methods: Final = endpoint_data.get("methods")
cost_per_request: Final = endpoint_data.get("cost_per_request")
timeout: Final = endpoint_data.get("timeout")
model: Final = endpoint_data.get("model")
verbose_proxy_logger.debug("Initializing pass through endpoint: %s (ID: %s)", path, endpoint_id)
InitPassThroughEndpointHelpers.add_exact_path_route(
@ -3009,6 +3022,7 @@ async def _register_pass_through_endpoint(
config_file_path=config_file_path,
auth=auth_enforced,
timeout=timeout,
model=model,
)
methods_for_key: Final = methods if methods else ["GET", "POST", "PUT", "DELETE", "PATCH"]
@ -3036,6 +3050,7 @@ async def _register_pass_through_endpoint(
config_file_path=config_file_path,
auth=auth_enforced,
timeout=timeout,
model=model,
)
visited_endpoints.add(f"{endpoint_id}:subpath:{path}:{methods_str}")
@ -3413,6 +3428,7 @@ async def update_pass_through_endpoints(
default_query_params=updated_endpoint.default_query_params,
auth=updated_endpoint.auth,
timeout=updated_endpoint.timeout,
model=updated_endpoint.model,
)
else:
InitPassThroughEndpointHelpers.add_exact_path_route(
@ -3430,6 +3446,7 @@ async def update_pass_through_endpoints(
default_query_params=updated_endpoint.default_query_params,
auth=updated_endpoint.auth,
timeout=updated_endpoint.timeout,
model=updated_endpoint.model,
)
return PassThroughEndpointResponse(endpoints=[updated_endpoint] if updated_endpoint else [])
@ -3505,6 +3522,7 @@ async def create_pass_through_endpoints(
default_query_params=created_endpoint.default_query_params,
auth=created_endpoint.auth,
timeout=created_endpoint.timeout,
model=created_endpoint.model,
)
else:
InitPassThroughEndpointHelpers.add_exact_path_route(
@ -3522,6 +3540,7 @@ async def create_pass_through_endpoints(
default_query_params=created_endpoint.default_query_params,
auth=created_endpoint.auth,
timeout=created_endpoint.timeout,
model=created_endpoint.model,
)
return PassThroughEndpointResponse(endpoints=[created_endpoint])

143
tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
View file

@ -1195,6 +1195,149 @@ async def test_create_pass_through_route_forwards_timeout():
assert call_kwargs["timeout"] == 1800
def test_pass_through_generic_endpoint_accepts_model():
from litellm.proxy._types import PassThroughGenericEndpoint
endpoint = PassThroughGenericEndpoint(
path="/parsers/east",
target="http://parser.example.com",
model="doc-parser-east",
)
assert endpoint.model == "doc-parser-east"
@pytest.mark.asyncio
async def test_create_pass_through_route_forwards_model():
unique_path = "/test/path/unique/model"
endpoint_func = create_pass_through_route(
endpoint=unique_path,
target="http://example.com",
custom_headers={},
_forward_headers=True,
_merge_query_params=False,
dependencies=[],
model="doc-parser-east",
)
with (
patch(
"litellm.proxy.pass_through_endpoints.pass_through_endpoints.pass_through_request"
) as mock_pass_through,
patch(
"litellm.proxy.pass_through_endpoints.pass_through_endpoints.InitPassThroughEndpointHelpers.is_registered_pass_through_route"
) as mock_is_registered,
patch(
"litellm.proxy.pass_through_endpoints.pass_through_endpoints.InitPassThroughEndpointHelpers.get_registered_pass_through_route"
) as mock_get_registered,
):
sentinel_response = MagicMock()
mock_pass_through.return_value = sentinel_response
mock_is_registered.return_value = True
mock_get_registered.return_value = None
mock_request = MagicMock(spec=Request)
mock_request.url = MagicMock()
mock_request.url.path = unique_path
mock_request.path_params = {}
mock_request.query_params = QueryParams({})
mock_request.method = "POST"
mock_request.headers = Headers({})
mock_request.body = AsyncMock(return_value=b"{}")
mock_user_api_key_dict = MagicMock()
mock_user_api_key_dict.api_key = "test-key"
result = await endpoint_func(
request=mock_request,
user_api_key_dict=mock_user_api_key_dict,
fastapi_response=MagicMock(),
)
assert result is sentinel_response
assert mock_pass_through.call_args[1]["model"] == "doc-parser-east"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"custom_body,configured_model,expected",
[
({"query": "parse this"}, "doc-parser-east", "doc-parser-east"),
({"model": "gpt-4o", "messages": []}, "doc-parser-east", "gpt-4o"),
({"query": "parse this"}, None, "unknown"),
],
)
async def test_pass_through_request_resolves_model_for_spend_logs(
custom_body, configured_model, expected
):
with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging:
with patch(
"litellm.litellm_core_utils.litellm_logging.Logging"
) as mock_logging_cls:
with patch(
"litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client"
) as mock_get_client:
mock_proxy_logging.pre_call_hook = AsyncMock(
side_effect=lambda **kwargs: kwargs["data"]
)
mock_proxy_logging.post_call_failure_hook = AsyncMock()
mock_logging_cls.return_value = MagicMock()
mock_client = MagicMock()
mock_client.client = MagicMock()
mock_client.client.send = AsyncMock(
side_effect=httpx.HTTPError("Request failed")
)
mock_get_client.return_value = mock_client
mock_request = MagicMock(spec=Request)
mock_request.method = "POST"
mock_request.headers = Headers({})
mock_request.query_params = QueryParams({})
with pytest.raises(ProxyException, match="Request failed"):
await pass_through_request(
request=mock_request,
target="http://test.com",
custom_headers={},
user_api_key_dict=MagicMock(),
custom_body=custom_body,
model=configured_model,
)
mock_logging_cls.assert_called()
assert mock_logging_cls.call_args.kwargs["model"] == expected
def test_add_exact_path_route_stores_model():
mock_app = MagicMock()
endpoint_id = "test-model-hint-endpoint"
InitPassThroughEndpointHelpers.add_exact_path_route(
app=mock_app,
path="/parsers/east",
target="http://parser.example.com",
custom_headers={},
forward_headers=False,
merge_query_params=False,
dependencies=[],
cost_per_request=0.0,
endpoint_id=endpoint_id,
model="doc-parser-east",
)
stored = next(
value
for key, value in _registered_pass_through_routes.items()
if value["endpoint_id"] == endpoint_id
)
assert stored["passthrough_params"]["model"] == "doc-parser-east"
keys_to_remove = [
key
for key, value in list(_registered_pass_through_routes.items())
if value["endpoint_id"] == endpoint_id
]
for key in keys_to_remove:
del _registered_pass_through_routes[key]
def test_initialize_pass_through_endpoints_with_cost_per_request():
"""
Test that initialize_pass_through_endpoints correctly passes cost_per_request to route creation

5
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -31951,6 +31951,11 @@ export interface components {
* @description List of HTTP methods this endpoint handles (e.g., ['GET', 'POST']). If None or empty, all methods (GET, POST, PUT, DELETE, PATCH) are supported for backward compatibility. This allows the same path to have different targets for different HTTP methods.
*/
methods?: string[] | null;
/**
* Model
* @description Optional model name written to spend logs for this endpoint. Use when the pass-through is not an LLM completion but still needs per-model metering (e.g. multi-instance document parsers). Request-body `model` still takes precedence when present.
*/
model?: string | null;
/**
* Path
* @description The route to be added to the LiteLLM Proxy Server.