From 69be041da7a8fc905862c58cf47cac714c402558 Mon Sep 17 00:00:00 2001 From: yassin Date: Tue, 15 Sep 2026 22:11:08 +0000 Subject: [PATCH] feat(proxy): add /nvidia_nim passthrough route for NIM object detection and OCR /v1/infer Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../azure_ai/passthrough/transformation.py | 25 +- .../base_llm/passthrough/transformation.py | 23 +- .../llms/nvidia_nim/passthrough/__init__.py | 0 .../nvidia_nim/passthrough/transformation.py | 116 ++++++++++ litellm/proxy/_types.py | 1 + litellm/proxy/auth/auth_utils.py | 12 + .../llm_passthrough_endpoints.py | 67 ++++++ litellm/utils.py | 6 + ...t_nvidia_nim_passthrough_transformation.py | 216 ++++++++++++++++++ .../proxy/auth/test_auth_utils.py | 58 +++++ .../proxy/auth/test_route_checks.py | 1 + .../test_llm_pass_through_endpoints.py | 147 ++++++++++++ 12 files changed, 649 insertions(+), 23 deletions(-) create mode 100644 litellm/llms/nvidia_nim/passthrough/__init__.py create mode 100644 litellm/llms/nvidia_nim/passthrough/transformation.py create mode 100644 tests/test_litellm/llms/nvidia_nim/passthrough/test_nvidia_nim_passthrough_transformation.py diff --git a/litellm/llms/azure_ai/passthrough/transformation.py b/litellm/llms/azure_ai/passthrough/transformation.py index f2be1d95593..4007ac37948 100644 --- a/litellm/llms/azure_ai/passthrough/transformation.py +++ b/litellm/llms/azure_ai/passthrough/transformation.py @@ -5,7 +5,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final import httpx -from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger from litellm.llms.azure_ai.common_utils import ( @@ -18,6 +18,8 @@ from litellm.llms.base_llm.passthrough.transformation import ( BasePassthroughConfig, RelayShape, logged_relay_shape, + model_group_from, + relayed_body, strip_leading_model_segment, ) from litellm.types.llms.openai import AllMessageValues @@ -35,19 +37,6 @@ if TYPE_CHECKING: EMPTY_QUERY: Final[Mapping[str, object]] = MappingProxyType({}) -class PassthroughMetadata(BaseModel): - model_config = ConfigDict(extra="ignore") - - model_group: str = "" - - -def model_group_from(litellm_params: Mapping[str, object]) -> str: - try: - return PassthroughMetadata.model_validate(litellm_params.get("litellm_metadata")).model_group - except ValidationError: - return "" - - def api_version_from(litellm_params: Mapping[str, object]) -> str | None: try: return TypeAdapter(str | None).validate_python(litellm_params.get("api_version")) @@ -96,14 +85,6 @@ def relay_query_params( return MappingProxyType({**(request_query_params or EMPTY_QUERY), "api-version": api_version}) -def relayed_body(httpx_response: Response) -> str | dict: - try: - body: Final[object] = httpx_response.json() - except ValueError: - return httpx_response.text - return body if isinstance(body, dict) else httpx_response.text - - FOUNDRY_RELAY_SHAPES: Final = ( RelayShape("/rerank", CallTypes.arerank, RerankResponse.model_validate), RelayShape("/providers/blackforestlabs/v1/flux-2-pro", CallTypes.aimage_generation, ImageResponse.model_validate), diff --git a/litellm/llms/base_llm/passthrough/transformation.py b/litellm/llms/base_llm/passthrough/transformation.py index ec938889b88..f2a12c3f22d 100644 --- a/litellm/llms/base_llm/passthrough/transformation.py +++ b/litellm/llms/base_llm/passthrough/transformation.py @@ -6,7 +6,7 @@ from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, Final, Protocol, TypeAlias -from pydantic import TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm.types.utils import CallTypes @@ -29,6 +29,19 @@ if TYPE_CHECKING: RELAYED_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) +class PassthroughMetadata(BaseModel): + model_config = ConfigDict(extra="ignore") + + model_group: str = "" + + +def model_group_from(litellm_params: Mapping[str, object]) -> str: + try: + return PassthroughMetadata.model_validate(litellm_params.get("litellm_metadata")).model_group + except ValidationError: + return "" + + def strip_leading_model_segment(endpoint: str, model_names: tuple[str, ...]) -> str: path: Final = endpoint.lstrip("/") for model_name in model_names: @@ -55,6 +68,14 @@ def relayed_json_object(httpx_response: Response) -> Mapping[str, object] | None return None +def relayed_body(httpx_response: Response) -> str | dict: + try: + body: Final[object] = httpx_response.json() + except ValueError: + return httpx_response.text + return body if isinstance(body, dict) else httpx_response.text + + @dataclass(frozen=True, slots=True) class RelayShape: path_suffix: str diff --git a/litellm/llms/nvidia_nim/passthrough/__init__.py b/litellm/llms/nvidia_nim/passthrough/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/nvidia_nim/passthrough/transformation.py b/litellm/llms/nvidia_nim/passthrough/transformation.py new file mode 100644 index 00000000000..f6273daaabb --- /dev/null +++ b/litellm/llms/nvidia_nim/passthrough/transformation.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import re +from collections.abc import Collection, Mapping, Sequence +from typing import TYPE_CHECKING, Final + +import httpx + +from litellm.llms.base_llm.passthrough.transformation import ( + BasePassthroughConfig, + model_group_from, + relayed_body, + strip_leading_model_segment, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import StandardPassThroughResponseObject + +if TYPE_CHECKING: + from httpx import URL, Response + + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.llms.base_llm.ocr.transformation import OCRResponse + from litellm.llms.base_llm.passthrough.transformation import LoggedRelayResponse + + +API_VERSION_SEGMENT: Final = re.compile(r"^v\d+$") + + +def nvidia_nim_router_model_in_endpoint(endpoint: str, router_models: Collection[str]) -> str | None: + segments: Final = tuple(segment for segment in endpoint.split("/") if segment) + return next( + ( + "/".join(segments[:length]) + for length in range(len(segments), 0, -1) + if "/".join(segments[:length]) in router_models + ), + None, + ) + + +def without_repeated_version_prefix(api_base: str, native_endpoint: str) -> str: + url: Final = httpx.URL(api_base) + base_segments: Final = tuple(segment for segment in url.path.split("/") if segment) + first_native_segment: Final = native_endpoint.lstrip("/").split("/", 1)[0] + repeated: Final = ( + bool(base_segments) + and API_VERSION_SEGMENT.match(first_native_segment) is not None + and base_segments[-1] == first_native_segment + ) + kept_segments: Final = base_segments[:-1] if repeated else base_segments + return str(url.copy_with(path="/" + "/".join(kept_segments), query=None)).rstrip("/") + + +class NvidiaNimPassthroughConfig(BasePassthroughConfig): + def is_streaming_request(self, endpoint: str, request_data: dict) -> bool: + return bool(request_data.get("stream", False)) + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + endpoint: str, + request_query_params: dict | None, + litellm_params: dict, + ) -> tuple[URL, str]: + base_target_url: Final = self.get_api_base(api_base) + if base_target_url is None: + raise ValueError("NVIDIA NIM api base not found: set `api_base` on the deployment or NVIDIA_NIM_API_BASE") + native_endpoint: Final = strip_leading_model_segment(endpoint, (model_group_from(litellm_params), model)) + root: Final = without_repeated_version_prefix(base_target_url, native_endpoint) + return (self.format_url(native_endpoint, root, request_query_params), root) + + def validate_environment( + self, + headers: Mapping[str, str], + model: str, + messages: Sequence[AllMessageValues], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + api_key: str | None = None, + api_base: str | None = None, + ) -> dict[str, str]: # mutable-ok: base class contract returns dict for httpx + if api_key is None: + return dict(headers) # mutable-ok: base class contract returns dict for httpx + return { + **headers, + "Authorization": f"Bearer {api_key}", + } # mutable-ok: base class contract returns dict for httpx + + @staticmethod + def get_api_base(api_base: str | None = None) -> str | None: + return api_base or get_secret_str("NVIDIA_NIM_API_BASE") + + @staticmethod + def get_api_key(api_key: str | None = None) -> str | None: + return api_key or get_secret_str("NVIDIA_NIM_API_KEY") + + @staticmethod + def get_base_model(model: str) -> str | None: + return model + + def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: + return [] + + def logging_non_streaming_response( + self, + model: str, + custom_llm_provider: str, + httpx_response: Response, + request_data: Mapping[str, object], + logging_obj: Logging, + endpoint: str, + ) -> LoggedRelayResponse | OCRResponse | StandardPassThroughResponseObject | None: + return StandardPassThroughResponseObject(response=relayed_body(httpx_response)) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d4eda1c9540..b780b8d08ca 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -485,6 +485,7 @@ class LiteLLMRoutes(enum.Enum): "/milvus", "/gigachat", "/watsonx", + "/nvidia_nim", ] ######################################################### diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index dc304a156cf..529521dcba8 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -28,6 +28,7 @@ from litellm.litellm_core_utils.url_utils import ( validate_url, ) from litellm.llms.azure.passthrough.transformation import azure_router_model_in_endpoint +from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_router_model_in_endpoint from litellm.proxy._types import * from litellm.proxy.common_utils.http_parsing_utils import extract_nested_form_metadata from litellm.types.passthrough_endpoints.pass_through_endpoints import ( @@ -2040,9 +2041,20 @@ def get_model_from_request( azure_model: Final = _router_model_from_azure_route(route, llm_router) return model if azure_model is None else azure_model + if route.lower().startswith("/nvidia_nim/"): + nvidia_nim_model: Final = _router_model_from_nvidia_nim_route(route, llm_router) + return model if nvidia_nim_model is None else nvidia_nim_model + return model +def _router_model_from_nvidia_nim_route(route: str, llm_router: Router | None) -> str | None: + if llm_router is None: + return None + endpoint: Final = re.sub(r"^/nvidia_nim/", "", route, flags=re.IGNORECASE) + return nvidia_nim_router_model_in_endpoint(endpoint, frozenset(llm_router.get_model_names())) + + def _router_model_from_azure_route(route: str, llm_router: Router | None) -> str | None: if llm_router is None: return None diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 28a8bab1f24..d4b1bdbdc20 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -36,6 +36,7 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.azure.passthrough.transformation import foreign_azure_deployment from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_router_model_in_endpoint from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.passthrough.main import AsyncPassthroughStreamingResponse from litellm.proxy._types import * @@ -1545,6 +1546,26 @@ async def _relay_azure_router_model( "put the model group name in the deployments segment" } raise HTTPException(status_code=400, detail=rejection) + return await _relay_router_model( + llm_router=llm_router, + model=model, + endpoint=endpoint, + request=request, + request_body=request_body, + is_streaming_request=is_streaming_request, + user_api_key_dict=user_api_key_dict, + ) + + +async def _relay_router_model( + llm_router: litellm.Router, + model: str, + endpoint: str, + request: Request, + request_body: Mapping[str, object], + is_streaming_request: bool, + user_api_key_dict: UserAPIKeyAuth, +) -> Response: try: result: Final = await llm_router.allm_passthrough_route( model=model, @@ -1594,6 +1615,52 @@ async def _relay_azure_router_model( ) +@router.api_route( + "/nvidia_nim/{endpoint:path}", + methods=["GET", "POST", "PUT", "DELETE", "PATCH"], + tags=["NVIDIA NIM Pass-through", "pass-through"], +) +async def nvidia_nim_proxy_route( + endpoint: str, + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +): + """ + Relay a native NVIDIA NIM request through a LiteLLM model group. + + `{PROXY_BASE_URL}/nvidia_nim/{model_group}/v1/infer` forwards the body unchanged to the deployment's + `api_base`, so object detection and OCR NIMs whose payload carries no `model` field still go through + virtual key auth, model access checks, and spend logging. + """ + from litellm.proxy.proxy_server import llm_router + + model_group: Final = ( + nvidia_nim_router_model_in_endpoint(endpoint, llm_router.get_model_names()) if llm_router else None + ) + if llm_router is None or model_group is None: + rejection: Final[RelayRejection] = { + "error": "no LiteLLM model group in the path; call /nvidia_nim/{model_group}/v1/infer with a model " + "from your `model_list` whose `model` starts with `nvidia_nim/`" + } + raise HTTPException(status_code=400, detail=rejection) + + request_body: Final = await get_request_body(request) + is_streaming_request: Final = is_passthrough_request_streaming(request_body) + return await open_sse_before_first_byte( + _relay_router_model( + llm_router=llm_router, + model=model_group, + endpoint=endpoint, + request=request, + request_body=request_body, + is_streaming_request=is_streaming_request, + user_api_key_dict=user_api_key_dict, + ), + ping_interval_seconds=(litellm.sse_keepalive_ping_interval_seconds if is_streaming_request else None), + ) + + @router.api_route( "/azure_ai/{endpoint:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], diff --git a/litellm/utils.py b/litellm/utils.py index af22b11224b..7f151e1d581 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8989,6 +8989,12 @@ class ProviderConfigManager: ) return WatsonxPassthroughConfig() + elif LlmProviders.NVIDIA_NIM == provider: + from litellm.llms.nvidia_nim.passthrough.transformation import ( + NvidiaNimPassthroughConfig, + ) + + return NvidiaNimPassthroughConfig() return None @staticmethod diff --git a/tests/test_litellm/llms/nvidia_nim/passthrough/test_nvidia_nim_passthrough_transformation.py b/tests/test_litellm/llms/nvidia_nim/passthrough/test_nvidia_nim_passthrough_transformation.py new file mode 100644 index 00000000000..87de4a795e0 --- /dev/null +++ b/tests/test_litellm/llms/nvidia_nim/passthrough/test_nvidia_nim_passthrough_transformation.py @@ -0,0 +1,216 @@ +import json +from types import MappingProxyType + +import httpx +import pytest + +import litellm +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.nvidia_nim.passthrough.transformation import ( + NvidiaNimPassthroughConfig, + nvidia_nim_router_model_in_endpoint, +) +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + +NIM_BASE = "http://nim.internal:8000" +INFER_BODY = { + "input": [ + {"type": "image_url", "url": "data:image/png;base64,AAAA"}, + {"type": "image_url", "url": "data:image/png;base64,BBBB"}, + ] +} + + +@pytest.fixture(autouse=True) +def clear_nvidia_nim_env(monkeypatch): + for env_var in ("NVIDIA_NIM_API_BASE", "NVIDIA_NIM_API_KEY"): + monkeypatch.delenv(env_var, raising=False) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setattr(litellm, "api_key", None) + + +def test_provider_config_manager_resolves_nvidia_nim_passthrough_config(): + config = ProviderConfigManager.get_provider_passthrough_config( + model="nvidia/nemoretriever-page-elements-v2", provider=LlmProviders.NVIDIA_NIM + ) + + assert isinstance(config, NvidiaNimPassthroughConfig) + + +@pytest.mark.parametrize( + "api_base, endpoint, litellm_params, expected", + [ + (NIM_BASE, "nim-page/v1/infer", {"litellm_metadata": {"model_group": "nim-page"}}, f"{NIM_BASE}/v1/infer"), + ( + f"{NIM_BASE}/v1", + "nim-page/v1/infer", + {"litellm_metadata": {"model_group": "nim-page"}}, + f"{NIM_BASE}/v1/infer", + ), + (f"{NIM_BASE}/v1/", "/v1/infer", {}, f"{NIM_BASE}/v1/infer"), + (NIM_BASE, "v1/infer", {}, f"{NIM_BASE}/v1/infer"), + (f"{NIM_BASE}/v2", "v1/infer", {}, f"{NIM_BASE}/v2/v1/infer"), + (f"{NIM_BASE}/infer", "infer", {}, f"{NIM_BASE}/infer/infer"), + (NIM_BASE, "nvidia/nemoretriever-page-elements-v2/v1/infer", {}, f"{NIM_BASE}/v1/infer"), + ], +) +def test_relay_url_strips_the_model_group_and_never_doubles_the_api_version( + api_base, endpoint, litellm_params, expected +): + url, base = NvidiaNimPassthroughConfig().get_complete_url( + api_base=api_base, + api_key=None, + model="nvidia/nemoretriever-page-elements-v2", + endpoint=endpoint, + request_query_params=None, + litellm_params=litellm_params, + ) + + assert str(url) == expected + assert base == expected.removesuffix("/v1/infer").removesuffix("/infer") + + +def test_query_params_are_forwarded_on_the_relay_url(): + url, _ = NvidiaNimPassthroughConfig().get_complete_url( + api_base=NIM_BASE, + api_key=None, + model="nvidia/nemoretriever-page-elements-v2", + endpoint="v1/infer", + request_query_params={"timeout": "30"}, + litellm_params={}, + ) + + assert str(url) == f"{NIM_BASE}/v1/infer?timeout=30" + + +def test_env_api_base_is_used_when_the_deployment_has_none(monkeypatch): + monkeypatch.setenv("NVIDIA_NIM_API_BASE", f"{NIM_BASE}/v1") + + url, _ = NvidiaNimPassthroughConfig().get_complete_url( + api_base=None, + api_key=None, + model="nvidia/nemoretriever-page-elements-v2", + endpoint="v1/infer", + request_query_params=None, + litellm_params={}, + ) + + assert str(url) == f"{NIM_BASE}/v1/infer" + + +def test_missing_api_base_raises_instead_of_building_a_relative_url(): + with pytest.raises(ValueError, match="NVIDIA_NIM_API_BASE"): + NvidiaNimPassthroughConfig().get_complete_url( + api_base=None, + api_key=None, + model="nvidia/nemoretriever-page-elements-v2", + endpoint="v1/infer", + request_query_params=None, + litellm_params={}, + ) + + +def test_deployment_key_becomes_a_bearer_token_and_caller_headers_are_kept(): + caller_headers = MappingProxyType({"x-request-id": "abc"}) + + headers = NvidiaNimPassthroughConfig().validate_environment( + headers=caller_headers, + model="nvidia/nemoretriever-page-elements-v2", + messages=[], + optional_params={}, + litellm_params={}, + api_key="nvapi-secret", + ) + + assert headers == {"x-request-id": "abc", "Authorization": "Bearer nvapi-secret"} + + +def test_self_hosted_nim_without_a_key_sends_no_authorization_header(): + headers = NvidiaNimPassthroughConfig().validate_environment( + headers={}, model="nvidia/x", messages=[], optional_params={}, litellm_params={}, api_key=None + ) + + assert "Authorization" not in headers + + +def test_env_api_key_fills_in_when_the_deployment_has_none(monkeypatch): + monkeypatch.setenv("NVIDIA_NIM_API_KEY", "nvapi-from-env") + + assert NvidiaNimPassthroughConfig.get_api_key(None) == "nvapi-from-env" + assert NvidiaNimPassthroughConfig.get_api_key("nvapi-deployment") == "nvapi-deployment" + + +@pytest.mark.parametrize( + "endpoint, router_models, expected", + [ + ("nim-page/v1/infer", ("nim-page", "nim-table"), "nim-page"), + ("/nim-page/v1/infer", ("nim-page",), "nim-page"), + ( + "nvidia/nemoretriever-page-elements-v2/v1/infer", + ("nvidia/nemoretriever-page-elements-v2",), + "nvidia/nemoretriever-page-elements-v2", + ), + ("nim/v1/infer", ("nim", "nim/v1"), "nim/v1"), + ("v1/infer", ("nim-page",), None), + ("nim-page-elements/v1/infer", ("nim-page",), None), + ("", ("nim-page",), None), + ], +) +def test_router_model_in_endpoint_takes_the_longest_leading_model_group(endpoint, router_models, expected): + assert nvidia_nim_router_model_in_endpoint(endpoint, frozenset(router_models)) == expected + + +@pytest.mark.parametrize("request_data, expected", [({"stream": True}, True), ({"stream": False}, False), ({}, False)]) +def test_is_streaming_request_reads_the_stream_flag(request_data, expected): + assert NvidiaNimPassthroughConfig().is_streaming_request("v1/infer", request_data) is expected + + +def test_non_streaming_relay_logs_the_upstream_json_body(): + response = httpx.Response( + 200, + json={"data": [{"index": 0, "bounding_boxes": {}}]}, + request=httpx.Request("POST", f"{NIM_BASE}/v1/infer"), + ) + + result = NvidiaNimPassthroughConfig().logging_non_streaming_response( + model="nvidia/nemoretriever-page-elements-v2", + custom_llm_provider="nvidia_nim", + httpx_response=response, + request_data=INFER_BODY, + logging_obj=None, # pyright: ignore[reportArgumentType] # not read for a plain passthrough body + endpoint="v1/infer", + ) + + assert result == {"response": {"data": [{"index": 0, "bounding_boxes": {}}]}} + + +@pytest.mark.asyncio +async def test_object_detection_relay_sends_the_native_body_unchanged_to_v1_infer(): + upstream_requests: list[httpx.Request] = [] + + def nim(request: httpx.Request) -> httpx.Response: + upstream_requests.append(request) + return httpx.Response(200, json={"data": [{"index": 0}, {"index": 1}]}, headers={"x-nim": "1"}) + + client = AsyncHTTPHandler() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(nim)) + + response = await litellm.allm_passthrough_route( + model="nvidia_nim/nvidia/nemoretriever-page-elements-v2", + endpoint="nim-page/v1/infer", + method="POST", + api_base=f"{NIM_BASE}/v1", + api_key="nvapi-secret", + json=dict(INFER_BODY), + litellm_metadata={"model_group": "nim-page"}, + client=client, + ) + + (sent,) = upstream_requests + assert str(sent.url) == f"{NIM_BASE}/v1/infer" + assert json.loads(sent.content) == INFER_BODY + assert sent.headers["authorization"] == "Bearer nvapi-secret" + assert response.status_code == 200 + assert response.headers["x-nim"] == "1" + assert response.json() == {"data": [{"index": 0}, {"index": 1}]} diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index bd6a14cad21..81e505ea0a9 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -823,6 +823,64 @@ def test_get_model_from_request_azure_relay_routes_use_the_model_group_in_the_pa assert get_model_from_request(request_data=request_data, route=route, llm_router=_azure_relay_router()) == expected +def _nvidia_nim_relay_router(): + from litellm.router import Router + + return Router( + model_list=[ + { + "model_name": "nim-page-elements", + "litellm_params": { + "model": "nvidia_nim/nvidia/nemoretriever-page-elements-v2", + "api_base": "http://nim-a.internal:8000", + "api_key": "k", + }, + }, + { + "model_name": "nvidia/nemoretriever-table-structure-v1", + "litellm_params": { + "model": "nvidia_nim/nvidia/nemoretriever-table-structure-v1", + "api_base": "http://nim-b.internal:8000", + "api_key": "k", + }, + }, + ] + ) + + +NIM_INFER_BODY = {"input": [{"type": "image_url", "url": "data:image/png;base64,AAAA"}]} + + +@pytest.mark.parametrize( + "route, request_data, expected", + [ + ("/nvidia_nim/nim-page-elements/v1/infer", NIM_INFER_BODY, "nim-page-elements"), + ( + "/nvidia_nim/nim-page-elements/v1/infer", + {"model": "nvidia/nemoretriever-table-structure-v1"}, + "nim-page-elements", + ), + ( + "/nvidia_nim/nvidia/nemoretriever-table-structure-v1/v1/infer", + NIM_INFER_BODY, + "nvidia/nemoretriever-table-structure-v1", + ), + ("/nvidia_nim/v1/infer", NIM_INFER_BODY, None), + ("/nvidia_nim/unknown-group/v1/infer", NIM_INFER_BODY, None), + ("/nvidia_nim/nim-page-elements-v2/v1/infer", NIM_INFER_BODY, None), + ], +) +def test_get_model_from_request_nvidia_nim_relay_routes_use_the_model_group_in_the_path(route, request_data, expected): + assert ( + get_model_from_request(request_data=request_data, route=route, llm_router=_nvidia_nim_relay_router()) + == expected + ) + + +def test_get_model_from_request_nvidia_nim_relay_without_a_router_has_no_model(): + assert get_model_from_request(request_data=NIM_INFER_BODY, route="/nvidia_nim/nim-page-elements/v1/infer") is None + + def test_get_model_from_request_includes_file_endpoint_header_model(): assert ( get_model_from_request( diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 0950b56bf03..806c55d51ce 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -693,6 +693,7 @@ def test_virtual_key_allowed_routes_with_litellm_routes_member_name_denied(): "/anthropic/v1/count_tokens", "/gemini/v1/models", "/gemini/countTokens", + "/nvidia_nim/nim-page-elements/v1/infer", ], ) def test_virtual_key_llm_api_route_includes_passthrough_prefix(route): diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 7b285674145..4ef04a8e633 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -40,6 +40,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( llm_passthrough_factory_proxy_route, milvus_proxy_route, mistral_proxy_route, + nvidia_nim_proxy_route, openai_proxy_route, vertex_discovery_proxy_route, vertex_proxy_route, @@ -5375,6 +5376,152 @@ class TestRouterModelRelayUpstreamContract: assert result.headers["x-ms-request-id"] == "req-1" +NIM_INFER_BODY = { + "input": [ + {"type": "image_url", "url": "data:image/png;base64,AAAA"}, + {"type": "image_url", "url": "data:image/png;base64,BBBB"}, + ] +} + + +class TestNvidiaNimProxyRoute: + def _request(self) -> MagicMock: + request = MagicMock(spec=Request) + request.method = "POST" + request.headers = {"content-type": "application/json"} + request.query_params = {} + return request + + def _install_router(self, monkeypatch, router, body: dict) -> None: + import litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints as ep + import litellm.proxy.proxy_server as proxy_server + + async def fake_get_request_body(_request): + return dict(body) + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(ep, "get_request_body", fake_get_request_body) + + def _recording_router(self, captured: list[dict], model_names: tuple[str, ...]): + class RecordingRouter: + def get_model_names(self): + return list(model_names) + + async def allm_passthrough_route(self, **kwargs): + captured.append(kwargs) + return httpx.Response( + 200, json={"data": [{"index": 0, "bounding_boxes": {}}]}, headers={"x-nim-request": "r1"} + ) + + return RecordingRouter() + + @pytest.mark.asyncio + async def test_model_group_in_the_path_selects_the_deployment_and_the_body_stays_model_free(self, monkeypatch): + captured: list[dict] = [] + self._install_router( + monkeypatch, self._recording_router(captured, ("nim-page-elements", "nim-table")), NIM_INFER_BODY + ) + + result = await nvidia_nim_proxy_route( + endpoint="nim-page-elements/v1/infer", + request=self._request(), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token", team_id="team-1"), + ) + + (relay,) = captured + assert relay["model"] == "nim-page-elements" + assert relay["endpoint"] == "nim-page-elements/v1/infer" + assert relay["method"] == "POST" + assert relay["json"] == NIM_INFER_BODY + assert "model" not in relay["json"] + assert relay["litellm_metadata"]["user_api_key_team_id"] == "team-1" + assert result.status_code == 200 + assert json.loads(result.body) == {"data": [{"index": 0, "bounding_boxes": {}}]} + assert result.headers["x-nim-request"] == "r1" + + @pytest.mark.asyncio + async def test_model_group_with_a_slash_is_matched_as_the_longest_leading_path(self, monkeypatch): + captured: list[dict] = [] + self._install_router( + monkeypatch, self._recording_router(captured, ("nvidia/nemoretriever-page-elements-v2",)), NIM_INFER_BODY + ) + + await nvidia_nim_proxy_route( + endpoint="nvidia/nemoretriever-page-elements-v2/v1/infer", + request=self._request(), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"), + ) + + assert captured[0]["model"] == "nvidia/nemoretriever-page-elements-v2" + + @pytest.mark.asyncio + @pytest.mark.parametrize("endpoint", ["v1/infer", "unknown-group/v1/infer", "nim-page-elements-v2/v1/infer"]) + async def test_path_without_a_configured_model_group_is_rejected_before_any_upstream_call( + self, monkeypatch, endpoint + ): + from fastapi import HTTPException + + captured: list[dict] = [] + self._install_router(monkeypatch, self._recording_router(captured, ("nim-page-elements",)), NIM_INFER_BODY) + + with pytest.raises(HTTPException) as exc_info: + await nvidia_nim_proxy_route( + endpoint=endpoint, + request=self._request(), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"), + ) + + assert exc_info.value.status_code == 400 + assert captured == [] + + @pytest.mark.asyncio + async def test_no_router_is_rejected_before_any_upstream_call(self, monkeypatch): + from fastapi import HTTPException + + self._install_router(monkeypatch, None, NIM_INFER_BODY) + + with pytest.raises(HTTPException) as exc_info: + await nvidia_nim_proxy_route( + endpoint="nim-page-elements/v1/infer", + request=self._request(), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"), + ) + + assert exc_info.value.status_code == 400 + + @pytest.mark.asyncio + async def test_upstream_rejection_is_relayed_with_its_status_body_and_headers(self, monkeypatch): + upstream_body = {"detail": "input[0].url must be a data URL"} + + class RejectingRouter: + def get_model_names(self): + return ["nim-page-elements"] + + async def allm_passthrough_route(self, **kwargs): + upstream_request = httpx.Request("POST", "http://nim.internal:8000/v1/infer") + upstream = httpx.Response( + 422, json=upstream_body, headers={"x-nim-request": "r2"}, request=upstream_request + ) + raise httpx.HTTPStatusError("422", request=upstream_request, response=upstream) + + self._install_router(monkeypatch, RejectingRouter(), {"input": [{"type": "image_url", "url": "x"}]}) + + result = await nvidia_nim_proxy_route( + endpoint="nim-page-elements/v1/infer", + request=self._request(), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"), + ) + + assert result.status_code == 422 + assert json.loads(result.body) == upstream_body + assert result.headers["x-nim-request"] == "r2" + + @pytest.mark.asyncio async def test_bedrock_count_tokens_error_forwards_provider_headers(): """The count tokens route converts BedrockError into an HTTPException, and dropping the