diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index ed49ca2caa9..69cc64996d5 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2206,6 +2206,10 @@ class PassThroughGenericEndpoint(LiteLLMPydanticObjectBase): default=0.0, description="The USD cost per request to the target endpoint. This is used to calculate the cost of the request to the target endpoint.", ) + model: str | None = Field( + default=None, + description="Optional model name written to spend logs for this endpoint. Use when the pass-through is not an LLM completion but still needs per-model metering (e.g. multi-instance document parsers). Request-body `model` still takes precedence when present.", + ) timeout: float | None = Field( default=None, description="Upstream request timeout in seconds for this pass-through endpoint. If unset, uses general_settings.pass_through_request_timeout (default 600).", diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 3d60f4f5f3a..90d1f7813dd 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -817,6 +817,7 @@ async def pass_through_request( custom_llm_provider: str | None = None, guardrails_config: dict | None = None, timeout: float | None = None, + model: str | None = None, ): """ Pass through endpoint handler, makes the httpx request for pass-through endpoints and ensures logging hooks are called @@ -837,6 +838,7 @@ async def pass_through_request( guardrails_config: Optional field - guardrails configuration for passthrough endpoint timeout: Optional per-endpoint timeout in seconds. Falls back to general_settings.pass_through_request_timeout, then 600s. + model: Optional field - model name for spend logs when the request body has none """ from litellm.exceptions import ModifyResponseException from litellm.litellm_core_utils.litellm_logging import Logging @@ -924,9 +926,8 @@ async def pass_through_request( verbose_proxy_logger.debug("Added guardrails to passthrough request metadata: %s", guardrails_to_run) ## LOGGING OBJECT ## - initialize before pre_call_hook so guardrails can access it - # Surface the requested model (when the body carries one) so logging/spans - # read e.g. ``chat gpt-4o`` instead of ``chat unknown``. - passthrough_model: Final = (_parsed_body.get("model") if isinstance(_parsed_body, dict) else None) or "unknown" + body_model: Final = _parsed_body.get("model") if isinstance(_parsed_body, dict) else None + passthrough_model: Final = body_model or model or "unknown" start_time: Final = datetime.now() logging_obj = Logging( model=passthrough_model, @@ -1720,6 +1721,7 @@ def create_pass_through_route( guardrails: dict[str, object] | None = None, config_file_path: str | None = None, timeout: float | None = None, + model: str | None = None, ): # check if target is an adapter.py or a url from litellm._uuid import uuid @@ -1797,6 +1799,7 @@ def create_pass_through_route( "cost_per_request": cost_per_request, "guardrails": None, "timeout": timeout, + "model": model, } if passthrough_params is not None: @@ -1811,6 +1814,8 @@ def create_pass_through_route( param_guardrails: Final = target_params.get("guardrails", None) param_default_query_params: Final = target_params.get("default_query_params", None) param_timeout: Final = target_params.get("timeout", timeout) + _raw_model: Final = target_params.get("model", model) + param_model: Final[str | None] = _raw_model if isinstance(_raw_model, str) else None # Construct the full target URL with subpath if needed full_target: Final = HttpPassThroughEndpointHelpers.construct_target_url_with_subpath( @@ -1859,6 +1864,7 @@ def create_pass_through_route( custom_llm_provider=custom_llm_provider, guardrails_config=cast(dict | None, param_guardrails), timeout=cast(float | None, param_timeout), + model=param_model, ) finally: if hasattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY): @@ -2670,6 +2676,7 @@ class InitPassThroughEndpointHelpers: config_file_path: str | None = None, auth: bool = False, timeout: float | None = None, + model: str | None = None, ): """Add exact path route for pass-through endpoint""" # Default to all methods if none specified (backward compatibility) @@ -2711,6 +2718,7 @@ class InitPassThroughEndpointHelpers: guardrails=guardrails, config_file_path=config_file_path, timeout=timeout, + model=model, ), methods=methods, dependencies=dependencies, @@ -2733,6 +2741,7 @@ class InitPassThroughEndpointHelpers: "cost_per_request": cost_per_request, "guardrails": guardrails, "timeout": timeout, + "model": model, }, } @@ -2753,6 +2762,7 @@ class InitPassThroughEndpointHelpers: config_file_path: str | None = None, auth: bool = False, timeout: float | None = None, + model: str | None = None, ): """Add wildcard route for sub-paths""" # Default to all methods if none specified (backward compatibility) @@ -2795,6 +2805,7 @@ class InitPassThroughEndpointHelpers: guardrails=guardrails, config_file_path=config_file_path, timeout=timeout, + model=model, ), methods=methods, dependencies=dependencies, @@ -2817,6 +2828,7 @@ class InitPassThroughEndpointHelpers: "cost_per_request": cost_per_request, "guardrails": guardrails, "timeout": timeout, + "model": model, }, } @@ -2991,6 +3003,7 @@ async def _register_pass_through_endpoint( methods: Final = endpoint_data.get("methods") cost_per_request: Final = endpoint_data.get("cost_per_request") timeout: Final = endpoint_data.get("timeout") + model: Final = endpoint_data.get("model") verbose_proxy_logger.debug("Initializing pass through endpoint: %s (ID: %s)", path, endpoint_id) InitPassThroughEndpointHelpers.add_exact_path_route( @@ -3009,6 +3022,7 @@ async def _register_pass_through_endpoint( config_file_path=config_file_path, auth=auth_enforced, timeout=timeout, + model=model, ) methods_for_key: Final = methods if methods else ["GET", "POST", "PUT", "DELETE", "PATCH"] @@ -3036,6 +3050,7 @@ async def _register_pass_through_endpoint( config_file_path=config_file_path, auth=auth_enforced, timeout=timeout, + model=model, ) visited_endpoints.add(f"{endpoint_id}:subpath:{path}:{methods_str}") @@ -3413,6 +3428,7 @@ async def update_pass_through_endpoints( default_query_params=updated_endpoint.default_query_params, auth=updated_endpoint.auth, timeout=updated_endpoint.timeout, + model=updated_endpoint.model, ) else: InitPassThroughEndpointHelpers.add_exact_path_route( @@ -3430,6 +3446,7 @@ async def update_pass_through_endpoints( default_query_params=updated_endpoint.default_query_params, auth=updated_endpoint.auth, timeout=updated_endpoint.timeout, + model=updated_endpoint.model, ) return PassThroughEndpointResponse(endpoints=[updated_endpoint] if updated_endpoint else []) @@ -3505,6 +3522,7 @@ async def create_pass_through_endpoints( default_query_params=created_endpoint.default_query_params, auth=created_endpoint.auth, timeout=created_endpoint.timeout, + model=created_endpoint.model, ) else: InitPassThroughEndpointHelpers.add_exact_path_route( @@ -3522,6 +3540,7 @@ async def create_pass_through_endpoints( default_query_params=created_endpoint.default_query_params, auth=created_endpoint.auth, timeout=created_endpoint.timeout, + model=created_endpoint.model, ) return PassThroughEndpointResponse(endpoints=[created_endpoint]) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index a3f56adb86f..5a6766a92df 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -1195,6 +1195,149 @@ async def test_create_pass_through_route_forwards_timeout(): assert call_kwargs["timeout"] == 1800 +def test_pass_through_generic_endpoint_accepts_model(): + from litellm.proxy._types import PassThroughGenericEndpoint + + endpoint = PassThroughGenericEndpoint( + path="/parsers/east", + target="http://parser.example.com", + model="doc-parser-east", + ) + assert endpoint.model == "doc-parser-east" + + +@pytest.mark.asyncio +async def test_create_pass_through_route_forwards_model(): + unique_path = "/test/path/unique/model" + endpoint_func = create_pass_through_route( + endpoint=unique_path, + target="http://example.com", + custom_headers={}, + _forward_headers=True, + _merge_query_params=False, + dependencies=[], + model="doc-parser-east", + ) + + with ( + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.pass_through_request" + ) as mock_pass_through, + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.InitPassThroughEndpointHelpers.is_registered_pass_through_route" + ) as mock_is_registered, + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.InitPassThroughEndpointHelpers.get_registered_pass_through_route" + ) as mock_get_registered, + ): + sentinel_response = MagicMock() + mock_pass_through.return_value = sentinel_response + mock_is_registered.return_value = True + mock_get_registered.return_value = None + + mock_request = MagicMock(spec=Request) + mock_request.url = MagicMock() + mock_request.url.path = unique_path + mock_request.path_params = {} + mock_request.query_params = QueryParams({}) + mock_request.method = "POST" + mock_request.headers = Headers({}) + mock_request.body = AsyncMock(return_value=b"{}") + + mock_user_api_key_dict = MagicMock() + mock_user_api_key_dict.api_key = "test-key" + + result = await endpoint_func( + request=mock_request, + user_api_key_dict=mock_user_api_key_dict, + fastapi_response=MagicMock(), + ) + + assert result is sentinel_response + assert mock_pass_through.call_args[1]["model"] == "doc-parser-east" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "custom_body,configured_model,expected", + [ + ({"query": "parse this"}, "doc-parser-east", "doc-parser-east"), + ({"model": "gpt-4o", "messages": []}, "doc-parser-east", "gpt-4o"), + ({"query": "parse this"}, None, "unknown"), + ], +) +async def test_pass_through_request_resolves_model_for_spend_logs( + custom_body, configured_model, expected +): + with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging: + with patch( + "litellm.litellm_core_utils.litellm_logging.Logging" + ) as mock_logging_cls: + with patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client" + ) as mock_get_client: + mock_proxy_logging.pre_call_hook = AsyncMock( + side_effect=lambda **kwargs: kwargs["data"] + ) + mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_logging_cls.return_value = MagicMock() + + mock_client = MagicMock() + mock_client.client = MagicMock() + mock_client.client.send = AsyncMock( + side_effect=httpx.HTTPError("Request failed") + ) + mock_get_client.return_value = mock_client + + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.headers = Headers({}) + mock_request.query_params = QueryParams({}) + + with pytest.raises(ProxyException, match="Request failed"): + await pass_through_request( + request=mock_request, + target="http://test.com", + custom_headers={}, + user_api_key_dict=MagicMock(), + custom_body=custom_body, + model=configured_model, + ) + + mock_logging_cls.assert_called() + assert mock_logging_cls.call_args.kwargs["model"] == expected + + +def test_add_exact_path_route_stores_model(): + mock_app = MagicMock() + endpoint_id = "test-model-hint-endpoint" + InitPassThroughEndpointHelpers.add_exact_path_route( + app=mock_app, + path="/parsers/east", + target="http://parser.example.com", + custom_headers={}, + forward_headers=False, + merge_query_params=False, + dependencies=[], + cost_per_request=0.0, + endpoint_id=endpoint_id, + model="doc-parser-east", + ) + stored = next( + value + for key, value in _registered_pass_through_routes.items() + if value["endpoint_id"] == endpoint_id + ) + assert stored["passthrough_params"]["model"] == "doc-parser-east" + keys_to_remove = [ + key + for key, value in list(_registered_pass_through_routes.items()) + if value["endpoint_id"] == endpoint_id + ] + for key in keys_to_remove: + del _registered_pass_through_routes[key] + + def test_initialize_pass_through_endpoints_with_cost_per_request(): """ Test that initialize_pass_through_endpoints correctly passes cost_per_request to route creation diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 405ec9a01bf..7e8ac4ddfc2 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -31951,6 +31951,11 @@ export interface components { * @description List of HTTP methods this endpoint handles (e.g., ['GET', 'POST']). If None or empty, all methods (GET, POST, PUT, DELETE, PATCH) are supported for backward compatibility. This allows the same path to have different targets for different HTTP methods. */ methods?: string[] | null; + /** + * Model + * @description Optional model name written to spend logs for this endpoint. Use when the pass-through is not an LLM completion but still needs per-model metering (e.g. multi-instance document parsers). Request-body `model` still takes precedence when present. + */ + model?: string | null; /** * Path * @description The route to be added to the LiteLLM Proxy Server.