From f0c435b0369220b4e1fda56b3a0a1b46c0be8eac Mon Sep 17 00:00:00 2001 From: Yan Zhu Date: Thu, 13 Aug 2026 18:13:14 +0800 Subject: [PATCH] feat(passthrough): add optional model hint for spend logs Non-LLM pass-throughs (document parsers, etc.) have no request-body model, so spend logs always showed unknown. Honor an optional endpoint config model, while still preferring the body model when present. Co-authored-by: Cursor --- litellm/proxy/_types.py | 4 + .../pass_through_endpoints.py | 25 ++- .../test_pass_through_endpoints.py | 143 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 + 4 files changed, 174 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 0840d37ffa1..d77e9eccdf3 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2192,6 +2192,10 @@ class PassThroughGenericEndpoint(LiteLLMPydanticObjectBase): default=0.0, description="The USD cost per request to the target endpoint. This is used to calculate the cost of the request to the target endpoint.", ) + model: str | None = Field( + default=None, + description="Optional model name written to spend logs for this endpoint. Use when the pass-through is not an LLM completion but still needs per-model metering (e.g. multi-instance document parsers). Request-body `model` still takes precedence when present.", + ) timeout: float | None = Field( default=None, description="Upstream request timeout in seconds for this pass-through endpoint. If unset, uses general_settings.pass_through_request_timeout (default 600).", diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 1915a853983..53a66843dc4 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -809,6 +809,7 @@ async def pass_through_request( custom_llm_provider: str | None = None, guardrails_config: dict | None = None, timeout: float | None = None, + model: str | None = None, ): """ Pass through endpoint handler, makes the httpx request for pass-through endpoints and ensures logging hooks are called @@ -829,6 +830,7 @@ async def pass_through_request( guardrails_config: Optional field - guardrails configuration for passthrough endpoint timeout: Optional per-endpoint timeout in seconds. Falls back to general_settings.pass_through_request_timeout, then 600s. + model: Optional field - model name for spend logs when the request body has none """ from litellm.exceptions import ModifyResponseException from litellm.litellm_core_utils.litellm_logging import Logging @@ -916,9 +918,8 @@ async def pass_through_request( verbose_proxy_logger.debug("Added guardrails to passthrough request metadata: %s", guardrails_to_run) ## LOGGING OBJECT ## - initialize before pre_call_hook so guardrails can access it - # Surface the requested model (when the body carries one) so logging/spans - # read e.g. ``chat gpt-4o`` instead of ``chat unknown``. - passthrough_model: Final = (_parsed_body.get("model") if isinstance(_parsed_body, dict) else None) or "unknown" + body_model: Final = _parsed_body.get("model") if isinstance(_parsed_body, dict) else None + passthrough_model: Final = body_model or model or "unknown" start_time: Final = datetime.now() logging_obj = Logging( model=passthrough_model, @@ -1702,6 +1703,7 @@ def create_pass_through_route( guardrails: dict[str, object] | None = None, config_file_path: str | None = None, timeout: float | None = None, + model: str | None = None, ): # check if target is an adapter.py or a url from litellm._uuid import uuid @@ -1779,6 +1781,7 @@ def create_pass_through_route( "cost_per_request": cost_per_request, "guardrails": None, "timeout": timeout, + "model": model, } if passthrough_params is not None: @@ -1793,6 +1796,8 @@ def create_pass_through_route( param_guardrails: Final = target_params.get("guardrails", None) param_default_query_params: Final = target_params.get("default_query_params", None) param_timeout: Final = target_params.get("timeout", timeout) + _raw_model: Final = target_params.get("model", model) + param_model: Final[str | None] = _raw_model if isinstance(_raw_model, str) else None # Construct the full target URL with subpath if needed full_target: Final = HttpPassThroughEndpointHelpers.construct_target_url_with_subpath( @@ -1841,6 +1846,7 @@ def create_pass_through_route( custom_llm_provider=custom_llm_provider, guardrails_config=cast(dict | None, param_guardrails), timeout=cast(float | None, param_timeout), + model=param_model, ) finally: if hasattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY): @@ -2622,6 +2628,7 @@ class InitPassThroughEndpointHelpers: config_file_path: str | None = None, auth: bool = False, timeout: float | None = None, + model: str | None = None, ): """Add exact path route for pass-through endpoint""" # Default to all methods if none specified (backward compatibility) @@ -2663,6 +2670,7 @@ class InitPassThroughEndpointHelpers: guardrails=guardrails, config_file_path=config_file_path, timeout=timeout, + model=model, ), methods=methods, dependencies=dependencies, @@ -2685,6 +2693,7 @@ class InitPassThroughEndpointHelpers: "cost_per_request": cost_per_request, "guardrails": guardrails, "timeout": timeout, + "model": model, }, } @@ -2705,6 +2714,7 @@ class InitPassThroughEndpointHelpers: config_file_path: str | None = None, auth: bool = False, timeout: float | None = None, + model: str | None = None, ): """Add wildcard route for sub-paths""" # Default to all methods if none specified (backward compatibility) @@ -2747,6 +2757,7 @@ class InitPassThroughEndpointHelpers: guardrails=guardrails, config_file_path=config_file_path, timeout=timeout, + model=model, ), methods=methods, dependencies=dependencies, @@ -2769,6 +2780,7 @@ class InitPassThroughEndpointHelpers: "cost_per_request": cost_per_request, "guardrails": guardrails, "timeout": timeout, + "model": model, }, } @@ -2943,6 +2955,7 @@ async def _register_pass_through_endpoint( methods: Final = endpoint_data.get("methods") cost_per_request: Final = endpoint_data.get("cost_per_request") timeout: Final = endpoint_data.get("timeout") + model: Final = endpoint_data.get("model") verbose_proxy_logger.debug("Initializing pass through endpoint: %s (ID: %s)", path, endpoint_id) InitPassThroughEndpointHelpers.add_exact_path_route( @@ -2961,6 +2974,7 @@ async def _register_pass_through_endpoint( config_file_path=config_file_path, auth=auth_enforced, timeout=timeout, + model=model, ) methods_for_key: Final = methods if methods else ["GET", "POST", "PUT", "DELETE", "PATCH"] @@ -2988,6 +3002,7 @@ async def _register_pass_through_endpoint( config_file_path=config_file_path, auth=auth_enforced, timeout=timeout, + model=model, ) visited_endpoints.add(f"{endpoint_id}:subpath:{path}:{methods_str}") @@ -3360,6 +3375,7 @@ async def update_pass_through_endpoints( default_query_params=updated_endpoint.default_query_params, auth=updated_endpoint.auth, timeout=updated_endpoint.timeout, + model=updated_endpoint.model, ) else: InitPassThroughEndpointHelpers.add_exact_path_route( @@ -3377,6 +3393,7 @@ async def update_pass_through_endpoints( default_query_params=updated_endpoint.default_query_params, auth=updated_endpoint.auth, timeout=updated_endpoint.timeout, + model=updated_endpoint.model, ) return PassThroughEndpointResponse(endpoints=[updated_endpoint] if updated_endpoint else []) @@ -3452,6 +3469,7 @@ async def create_pass_through_endpoints( default_query_params=created_endpoint.default_query_params, auth=created_endpoint.auth, timeout=created_endpoint.timeout, + model=created_endpoint.model, ) else: InitPassThroughEndpointHelpers.add_exact_path_route( @@ -3469,6 +3487,7 @@ async def create_pass_through_endpoints( default_query_params=created_endpoint.default_query_params, auth=created_endpoint.auth, timeout=created_endpoint.timeout, + model=created_endpoint.model, ) return PassThroughEndpointResponse(endpoints=[created_endpoint]) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 4c6ba23c88c..998777c0c7a 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -1160,6 +1160,149 @@ async def test_create_pass_through_route_forwards_timeout(): assert call_kwargs["timeout"] == 1800 +def test_pass_through_generic_endpoint_accepts_model(): + from litellm.proxy._types import PassThroughGenericEndpoint + + endpoint = PassThroughGenericEndpoint( + path="/parsers/east", + target="http://parser.example.com", + model="doc-parser-east", + ) + assert endpoint.model == "doc-parser-east" + + +@pytest.mark.asyncio +async def test_create_pass_through_route_forwards_model(): + unique_path = "/test/path/unique/model" + endpoint_func = create_pass_through_route( + endpoint=unique_path, + target="http://example.com", + custom_headers={}, + _forward_headers=True, + _merge_query_params=False, + dependencies=[], + model="doc-parser-east", + ) + + with ( + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.pass_through_request" + ) as mock_pass_through, + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.InitPassThroughEndpointHelpers.is_registered_pass_through_route" + ) as mock_is_registered, + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.InitPassThroughEndpointHelpers.get_registered_pass_through_route" + ) as mock_get_registered, + ): + sentinel_response = MagicMock() + mock_pass_through.return_value = sentinel_response + mock_is_registered.return_value = True + mock_get_registered.return_value = None + + mock_request = MagicMock(spec=Request) + mock_request.url = MagicMock() + mock_request.url.path = unique_path + mock_request.path_params = {} + mock_request.query_params = QueryParams({}) + mock_request.method = "POST" + mock_request.headers = Headers({}) + mock_request.body = AsyncMock(return_value=b"{}") + + mock_user_api_key_dict = MagicMock() + mock_user_api_key_dict.api_key = "test-key" + + result = await endpoint_func( + request=mock_request, + user_api_key_dict=mock_user_api_key_dict, + fastapi_response=MagicMock(), + ) + + assert result is sentinel_response + assert mock_pass_through.call_args[1]["model"] == "doc-parser-east" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "custom_body,configured_model,expected", + [ + ({"query": "parse this"}, "doc-parser-east", "doc-parser-east"), + ({"model": "gpt-4o", "messages": []}, "doc-parser-east", "gpt-4o"), + ({"query": "parse this"}, None, "unknown"), + ], +) +async def test_pass_through_request_resolves_model_for_spend_logs( + custom_body, configured_model, expected +): + with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging: + with patch( + "litellm.litellm_core_utils.litellm_logging.Logging" + ) as mock_logging_cls: + with patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client" + ) as mock_get_client: + mock_proxy_logging.pre_call_hook = AsyncMock( + side_effect=lambda **kwargs: kwargs["data"] + ) + mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_logging_cls.return_value = MagicMock() + + mock_client = MagicMock() + mock_client.client = MagicMock() + mock_client.client.send = AsyncMock( + side_effect=httpx.HTTPError("Request failed") + ) + mock_get_client.return_value = mock_client + + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.headers = Headers({}) + mock_request.query_params = QueryParams({}) + + with pytest.raises(ProxyException, match="Request failed"): + await pass_through_request( + request=mock_request, + target="http://test.com", + custom_headers={}, + user_api_key_dict=MagicMock(), + custom_body=custom_body, + model=configured_model, + ) + + mock_logging_cls.assert_called() + assert mock_logging_cls.call_args.kwargs["model"] == expected + + +def test_add_exact_path_route_stores_model(): + mock_app = MagicMock() + endpoint_id = "test-model-hint-endpoint" + InitPassThroughEndpointHelpers.add_exact_path_route( + app=mock_app, + path="/parsers/east", + target="http://parser.example.com", + custom_headers={}, + forward_headers=False, + merge_query_params=False, + dependencies=[], + cost_per_request=0.0, + endpoint_id=endpoint_id, + model="doc-parser-east", + ) + stored = next( + value + for key, value in _registered_pass_through_routes.items() + if value["endpoint_id"] == endpoint_id + ) + assert stored["passthrough_params"]["model"] == "doc-parser-east" + keys_to_remove = [ + key + for key, value in list(_registered_pass_through_routes.items()) + if value["endpoint_id"] == endpoint_id + ] + for key in keys_to_remove: + del _registered_pass_through_routes[key] + + def test_initialize_pass_through_endpoints_with_cost_per_request(): """ Test that initialize_pass_through_endpoints correctly passes cost_per_request to route creation diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index cf55dc69e86..e6cc8fb4065 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -30723,6 +30723,11 @@ export interface components { * @description List of HTTP methods this endpoint handles (e.g., ['GET', 'POST']). If None or empty, all methods (GET, POST, PUT, DELETE, PATCH) are supported for backward compatibility. This allows the same path to have different targets for different HTTP methods. */ methods?: string[] | null; + /** + * Model + * @description Optional model name written to spend logs for this endpoint. Use when the pass-through is not an LLM completion but still needs per-model metering (e.g. multi-instance document parsers). Request-body `model` still takes precedence when present. + */ + model?: string | null; /** * Path * @description The route to be added to the LiteLLM Proxy Server.