feat(nvidia_nim): discover upstream models in the dashboard

This commit is contained in:
mateo-berri 2026-11-06 11:35:41 -08:00
parent a94118c6cd
commit ad4905e01c
7 changed files with 622 additions and 2 deletions

View file

@ -67,6 +67,7 @@ from litellm.repositories.credentials_repository import CredentialsRepository
from litellm.types.router import server_owned_wif_fields_named, server_owned_wif_fields_written
from litellm.types.utils import CreateCredentialItem, CredentialItem
from .nim_model_discovery import router as nim_model_discovery_router
from .user_provider_credentials import (
GithubCopilotUserConnectionPayload,
decode_user_provider_credential,
@ -80,6 +81,7 @@ from .user_provider_credentials import (
)
router: Final = APIRouter()
router.include_router(nim_model_discovery_router)
_CREDENTIAL_DICT_ADAPTER: Final = TypeAdapter(dict[str, object])
_GITHUB_COPILOT_PROVIDER: Final = "github_copilot"
_DISPLAY_NAME_MAX_LENGTH: Final = 255

View file

@ -0,0 +1,167 @@
import hashlib
from collections.abc import AsyncIterator, Mapping
from typing import Annotated, Final, Literal, Protocol
from urllib.parse import urlsplit
import httpx
from fastapi import APIRouter, Depends, HTTPException
from openai import AsyncOpenAI, OpenAIError
from pydantic import BaseModel, ConfigDict, SecretStr, TypeAdapter, ValidationError
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.credential_hydration import (
hydrate_named_credential_authoritative,
stored_credential_provider,
)
from litellm.secret_managers.main import get_secret_str
class _ModelCache(Protocol):
def get_cache(self, key: str) -> object: ...
def set_cache(self, key: str, value: tuple[str, ...]) -> None: ...
def _new_model_cache() -> _ModelCache:
return InMemoryCache(default_ttl=60)
router: Final = APIRouter()
_CACHE: Final = _new_model_cache()
_MODELS: Final = TypeAdapter(tuple[str, ...])
_DEFAULT_API_BASE: Final = "https://integrate.api.nvidia.com/v1"
class _CredentialMetadata(BaseModel):
model_config = ConfigDict(frozen=True, from_attributes=True)
credential_info: Mapping[str, object]
class _ListedModel(BaseModel):
model_config = ConfigDict(frozen=True, strict=True)
id: str
class _ModelListing(BaseModel):
model_config = ConfigDict(frozen=True)
data: tuple[_ListedModel, ...]
class NimModelDiscoveryConnection(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
api_key: SecretStr | None = None
api_base: str | None = None
class NimModelDiscoveryRequest(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
credential_name: str | None = None
credential_values: NimModelDiscoveryConnection | None = None
class NimModelDiscoveryResponse(BaseModel):
model_config = ConfigDict(frozen=True)
status: Literal["success", "unavailable"]
models: tuple[str, ...] = ()
def _resolved_secret(value: str | None) -> str | None:
if value is not None and value.startswith("os.environ/"):
return get_secret_str(value.removeprefix("os.environ/"))
return value
def _text(value: object) -> str | None:
return value if isinstance(value, str) and value else None
async def nim_http_client() -> AsyncIterator[httpx.AsyncClient]:
async with httpx.AsyncClient(follow_redirects=False) as client:
yield client
@router.post(
"/credentials/nvidia_nim/models",
tags=["credential management"],
response_model=NimModelDiscoveryResponse,
)
async def discover_nim_models(
body: NimModelDiscoveryRequest,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
http_client: Annotated[httpx.AsyncClient, Depends(nim_http_client)],
) -> NimModelDiscoveryResponse:
"""List upstream NIM models using a saved credential or submitted connection settings.
Requires a proxy administrator. Returns model IDs only, without asserting inference access,
pricing or capabilities. Upstream failures return unavailable so the UI can use its catalog.
"""
from litellm.proxy.proxy_server import prisma_client
if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
raise HTTPException(status_code=403, detail="Only proxy admins can discover NIM models")
if body.credential_name and body.credential_values is not None:
raise HTTPException(status_code=400, detail="Saved credentials cannot be combined with connection overrides")
credential: Final = (
await hydrate_named_credential_authoritative(body.credential_name, prisma_client)
if body.credential_name
else None
)
if body.credential_name and credential is None:
raise HTTPException(status_code=404, detail="Credential not found")
provider: Final = (
stored_credential_provider(
_CredentialMetadata.model_validate(credential).credential_info.get("custom_llm_provider")
)
if credential is not None
else None
)
if provider is not None and provider != "nvidia_nim":
raise HTTPException(status_code=400, detail="Select an NVIDIA NIM credential")
submitted: Final = body.credential_values or NimModelDiscoveryConnection()
values: Final = credential.credential_values if credential is not None else {}
api_base: Final = _resolved_secret(_text(values.get("api_base")) or submitted.api_base) or (
get_secret_str("NVIDIA_NIM_API_BASE") or _DEFAULT_API_BASE
)
try:
parsed: Final = urlsplit(api_base)
except ValueError:
raise HTTPException(status_code=400, detail="Invalid NIM API base") from None
if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username or parsed.password:
raise HTTPException(
status_code=400, detail="API base must be an HTTP or HTTPS endpoint without user information"
)
if parsed.query or parsed.fragment:
raise HTTPException(status_code=400, detail="API base cannot contain a query or fragment")
api_key: Final = _resolved_secret(
_text(values.get("api_key"))
or (submitted.api_key.get_secret_value() if submitted.api_key is not None else None)
) or get_secret_str("NVIDIA_NIM_API_KEY")
cache_key: Final = hashlib.sha256(f"{api_base.rstrip('/')}\0{api_key or ''}".encode()).hexdigest()
cached: Final = _CACHE.get_cache(cache_key)
if cached is not None:
return NimModelDiscoveryResponse(status="success", models=_MODELS.validate_python(cached))
try:
async with AsyncOpenAI(
api_key=api_key or "not-required",
base_url=api_base,
timeout=10,
max_retries=0,
http_client=http_client,
) as client:
page: Final = await client.models.list()
listing: Final = _ModelListing.model_validate(page.model_dump())
models: Final = tuple(sorted(frozenset(f"nvidia_nim/{item.id}" for item in listing.data if item.id)))
if not models:
return NimModelDiscoveryResponse(status="unavailable")
_CACHE.set_cache(cache_key, models)
return NimModelDiscoveryResponse(status="success", models=models)
except (OpenAIError, httpx.HTTPError, ValidationError):
return NimModelDiscoveryResponse(status="unavailable")

View file

@ -0,0 +1,135 @@
from collections.abc import Iterator
from typing import Final
from unittest.mock import patch
import httpx
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
import litellm
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.credential_endpoints.nim_model_discovery import nim_http_client, router
from litellm.types.utils import CredentialItem
@pytest.fixture
def app() -> FastAPI:
application: Final = FastAPI()
application.include_router(router)
application.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)
return application
@pytest.fixture
def client(app: FastAPI) -> Iterator[TestClient]:
with (
patch("litellm.proxy.proxy_server.prisma_client", None),
patch.object(litellm, "credential_list", []),
patch("litellm.proxy.credential_endpoints.nim_model_discovery._CACHE", InMemoryCache()),
):
with TestClient(app) as test_client:
yield test_client
def test_saved_credential_controls_the_upstream_and_only_ids_return(client: TestClient, app: FastAPI) -> None:
credential: Final = CredentialItem(
credential_name="nim-saved",
credential_info={"custom_llm_provider": "nvidia_nim"},
credential_values={"api_key": "saved-secret", "api_base": "https://nim.example/v1"},
)
def upstream(request: httpx.Request) -> httpx.Response:
assert str(request.url) == "https://nim.example/v1/models"
assert request.headers["authorization"] == "Bearer saved-secret"
return httpx.Response(200, json={"object": "list", "data": [{"id": "new-model"}, {"id": "new-model"}]})
async def connection() -> httpx.AsyncClient:
return httpx.AsyncClient(transport=httpx.MockTransport(upstream))
app.dependency_overrides[nim_http_client] = connection
with patch.object(litellm, "credential_list", [credential]):
response: Final = client.post("/credentials/nvidia_nim/models", json={"credential_name": "nim-saved"})
assert response.status_code == 200, response.text
assert response.json() == {"status": "success", "models": ["nvidia_nim/new-model"]}
@pytest.mark.parametrize("status", (401, 403, 429, 500))
def test_upstream_errors_allow_catalog_fallback_without_leaking_secrets(
client: TestClient, app: FastAPI, status: int
) -> None:
async def connection() -> httpx.AsyncClient:
return httpx.AsyncClient(
transport=httpx.MockTransport(lambda request: httpx.Response(status, json={"error": "private-details"}))
)
app.dependency_overrides[nim_http_client] = connection
response: Final = client.post(
"/credentials/nvidia_nim/models", json={"credential_values": {"api_key": "private-key"}}
)
assert response.status_code == 200, response.text
assert response.json() == {"status": "unavailable", "models": []}
@pytest.mark.parametrize("role", (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY))
def test_only_admins_can_use_saved_credentials_for_discovery(
client: TestClient, app: FastAPI, role: LitellmUserRoles
) -> None:
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=role)
response: Final = client.post("/credentials/nvidia_nim/models", json={"credential_name": "nim-saved"})
assert response.status_code == 403, response.text
@pytest.mark.parametrize("override", ({"api_base": "https://elsewhere.example/v1"}, {"api_key": "replacement"}))
def test_cannot_redirect_a_saved_credential(client: TestClient, override: dict[str, str]) -> None:
response: Final = client.post(
"/credentials/nvidia_nim/models", json={"credential_name": "nim-saved", "credential_values": override}
)
assert response.status_code == 400, response.text
def test_missing_credential_is_not_replaced_with_environment_credentials(client: TestClient, app: FastAPI) -> None:
response: Final = client.post("/credentials/nvidia_nim/models", json={"credential_name": "missing"})
assert response.status_code == 404, response.text
@pytest.mark.parametrize("listing", ({"data": []}, {"data": [{"id": 42}]}, {"data": None}))
def test_empty_or_malformed_listing_allows_catalog_fallback(
client: TestClient, app: FastAPI, listing: dict[str, object]
) -> None:
async def connection() -> httpx.AsyncClient:
return httpx.AsyncClient(transport=httpx.MockTransport(lambda request: httpx.Response(200, json=listing)))
app.dependency_overrides[nim_http_client] = connection
response: Final = client.post("/credentials/nvidia_nim/models", json={"credential_values": {"api_key": "test-key"}})
assert response.status_code == 200, response.text
assert response.json() == {"status": "unavailable", "models": []}
def test_cached_models_are_scoped_to_endpoint_and_key(client: TestClient, app: FastAPI) -> None:
async def success() -> httpx.AsyncClient:
return httpx.AsyncClient(
transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": [{"id": "cached"}]}))
)
async def failure() -> httpx.AsyncClient:
return httpx.AsyncClient(transport=httpx.MockTransport(lambda request: httpx.Response(503)))
app.dependency_overrides[nim_http_client] = success
original: Final = {"api_base": "https://nim.example/v1", "api_key": "first-key"}
first: Final = client.post("/credentials/nvidia_nim/models", json={"credential_values": original})
assert first.json() == {"status": "success", "models": ["nvidia_nim/cached"]}
app.dependency_overrides[nim_http_client] = failure
cached: Final = client.post("/credentials/nvidia_nim/models", json={"credential_values": original})
assert cached.json() == first.json()
other_key: Final = client.post(
"/credentials/nvidia_nim/models", json={"credential_values": {**original, "api_key": "other-key"}}
)
assert other_key.json() == {"status": "unavailable", "models": []}
other_endpoint: Final = client.post(
"/credentials/nvidia_nim/models",
json={"credential_values": {**original, "api_base": "https://other.example/v1"}},
)
assert other_endpoint.json() == {"status": "unavailable", "models": []}

View file

@ -34,7 +34,7 @@ import { ProviderLogo } from "../molecules/models/ProviderLogo";
import AccessGroupTagsCombobox from "./AccessGroupTagsCombobox";
import AdvancedSettings from "./advanced_settings";
import ConditionalPublicModelName from "./conditional_public_model_name";
import LiteLLMModelNameField from "./litellm_model_name";
import NimModelNameField from "./NimModelNameField";
import ConnectionErrorDisplay from "./model_connection_test";
import ProviderSpecificFields from "./provider_specific_fields";
import { authTypesFor } from "./provider_auth_types";
@ -295,7 +295,9 @@ const AddModelForm: React.FC<AddModelFormProps> = ({
/>
)}
</MountedFormField>
<LiteLLMModelNameField
<NimModelNameField
accessToken={accessToken}
canDiscover={canCreateCredential}
selectedProvider={selectedProvider}
providerModels={providerModels}
getPlaceholder={getPlaceholder}

View file

@ -0,0 +1,124 @@
import { act, fireEvent, render, screen, waitFor } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { describe, expect, it, vi, beforeEach } from "vitest";
import { useFormContext } from "react-hook-form";
import { MountedFormHost } from "../../../tests/mounted-form-host";
import type { MountedFormValues } from "../common_components/MountedFormField";
import { apiClient } from "../networking";
import { getPlaceholder, Providers } from "../provider_info_helpers";
import NimModelNameField from "./NimModelNameField";
vi.mock("../networking", () => ({ apiClient: { post: vi.fn() } }));
const catalogModel = "nvidia_nim/catalog-model";
const discoveredModel = "nvidia_nim/discovered-model";
function ConnectionFields() {
const { register } = useFormContext<MountedFormValues>();
return <input aria-label="Credential" {...register("litellm_credential_name")} />;
}
function Form({
provider = "NVIDIA_NIM",
canDiscover = true,
credentialName = "nim-saved",
apiKey,
apiBase,
}: {
provider?: string;
canDiscover?: boolean;
credentialName?: string;
apiKey?: string;
apiBase?: string;
} = {}) {
return (
<MountedFormHost defaultValues={{ litellm_credential_name: credentialName, api_key: apiKey, api_base: apiBase }}>
<ConnectionFields />
<NimModelNameField
selectedProvider={provider}
providerModels={[catalogModel]}
getPlaceholder={getPlaceholder}
accessToken="proxy-token"
canDiscover={canDiscover}
/>
</MountedFormHost>
);
}
beforeEach(() => vi.resetAllMocks());
describe("NIM model discovery", () => {
it("uses the selected credential and makes discovered models selectable", async () => {
vi.mocked(apiClient.post).mockResolvedValue({ status: "success", models: [discoveredModel] });
render(<Form />);
expect(await screen.findByText(/Models listed by your NIM endpoint/)).toBeInTheDocument();
expect(apiClient.post).toHaveBeenCalledWith("/credentials/nvidia_nim/models", {
accessToken: "proxy-token",
body: { credential_name: "nim-saved", credential_values: undefined },
signal: expect.any(AbortSignal),
});
const user = userEvent.setup();
await user.click(screen.getByRole("combobox", { name: "Select models" }));
expect(screen.getByRole("option", { name: discoveredModel })).toBeInTheDocument();
expect(screen.queryByRole("option", { name: catalogModel })).not.toBeInTheDocument();
expect(screen.getByRole("option", { name: "Custom Model Name (Enter below)" })).toBeInTheDocument();
});
it("passes inline connection settings without a saved credential", async () => {
vi.mocked(apiClient.post).mockResolvedValue({ status: "success", models: [discoveredModel] });
render(<Form credentialName="" apiKey="inline-key" apiBase="https://nim.example/v1" />);
expect(await screen.findByText(/Models listed by your NIM endpoint/)).toBeInTheDocument();
expect(apiClient.post).toHaveBeenCalledWith("/credentials/nvidia_nim/models", {
accessToken: "proxy-token",
body: {
credential_name: undefined,
credential_values: { api_key: "inline-key", api_base: "https://nim.example/v1" },
},
signal: expect.any(AbortSignal),
});
});
it("keeps the catalog usable when upstream discovery is unavailable", async () => {
vi.mocked(apiClient.post).mockResolvedValue({ status: "unavailable", models: [] });
render(<Form />);
expect(await screen.findByText(/Could not refresh NIM models/)).toBeInTheDocument();
const user = userEvent.setup();
await user.click(screen.getByRole("combobox", { name: "Select models" }));
expect(screen.getByRole("option", { name: catalogModel })).toBeInTheDocument();
});
it("ignores a late response for a previous credential", async () => {
let completeOld: (value: unknown) => void = () => {};
vi.mocked(apiClient.post)
.mockImplementationOnce(
() =>
new Promise((resolve) => {
completeOld = resolve;
}),
)
.mockResolvedValueOnce({ status: "success", models: [discoveredModel] });
render(<Form />);
await waitFor(() => expect(apiClient.post).toHaveBeenCalledTimes(1));
fireEvent.change(screen.getByRole("textbox", { name: "Credential" }), { target: { value: "nim-new" } });
expect(await screen.findByText(/Models listed by your NIM endpoint/)).toBeInTheDocument();
completeOld({ status: "success", models: ["nvidia_nim/old-credential-model"] });
const user = userEvent.setup();
await user.click(screen.getByRole("combobox", { name: "Select models" }));
expect(screen.getByRole("option", { name: discoveredModel })).toBeInTheDocument();
expect(screen.queryByRole("option", { name: "nvidia_nim/old-credential-model" })).not.toBeInTheDocument();
});
it.each([{ provider: Providers.OpenAI }, { canDiscover: false }])(
"leaves other providers and roles on the catalog",
async (props) => {
vi.useFakeTimers();
try {
render(<Form {...props} />);
await act(() => vi.advanceTimersByTimeAsync(600));
expect(screen.queryByRole("button", { name: "Refresh NIM models" })).not.toBeInTheDocument();
expect(apiClient.post).not.toHaveBeenCalled();
} finally {
vi.useRealTimers();
}
},
);
});

View file

@ -0,0 +1,108 @@
import { useEffect, useMemo, useState, type ComponentProps } from "react";
import { useFormContext, useWatch } from "react-hook-form";
import { apiClient } from "../networking";
import { resolveLitellmProviderSlug } from "../provider_info_helpers";
import type { MountedFormValues } from "../common_components/MountedFormField";
import { Button } from "@/components/ui/button";
import LiteLLMModelNameField from "./litellm_model_name";
type DiscoveryResponse = {
status: "success" | "unavailable";
models: string[];
};
type Props = ComponentProps<typeof LiteLLMModelNameField> & {
accessToken: string | null;
canDiscover: boolean;
};
export default function NimModelNameField({ accessToken, canDiscover, ...props }: Props) {
const { control } = useFormContext<MountedFormValues>();
const credential = useWatch({ control, name: "litellm_credential_name" });
const apiKey = useWatch({ control, name: "api_key" });
const apiBase = useWatch({ control, name: "api_base" });
const enabled =
canDiscover &&
props.selectedProvider !== null &&
resolveLitellmProviderSlug(props.selectedProvider) === "nvidia_nim";
const connection = useMemo(
() => ({
credential_name: typeof credential === "string" && credential ? credential : undefined,
credential_values: credential
? undefined
: {
api_key: typeof apiKey === "string" && apiKey ? apiKey : undefined,
api_base: typeof apiBase === "string" && apiBase ? apiBase : undefined,
},
}),
[credential, apiKey, apiBase],
);
const [attempt, setAttempt] = useState(0);
const [result, setResult] = useState<{
connection: typeof connection;
accessToken: string | null;
attempt: number;
response: DiscoveryResponse;
}>();
const current =
result?.connection === connection && result.accessToken === accessToken && result.attempt === attempt
? result.response
: undefined;
useEffect(() => {
if (!enabled) return;
const controller = new AbortController();
const timer = setTimeout(async () => {
try {
const response = await apiClient.post<DiscoveryResponse>("/credentials/nvidia_nim/models", {
accessToken,
body: connection,
signal: controller.signal,
});
if (!controller.signal.aborted) {
const nextResult = { connection, accessToken, attempt, response };
setResult(nextResult);
}
} catch {
if (!controller.signal.aborted) {
const response: DiscoveryResponse = { status: "unavailable", models: [] };
const nextResult = { connection, accessToken, attempt, response };
setResult(nextResult);
}
}
}, 500);
return () => {
clearTimeout(timer);
controller.abort();
};
}, [enabled, connection, accessToken, attempt]);
return (
<>
<LiteLLMModelNameField
{...props}
providerModels={enabled && current?.status === "success" ? current.models : props.providerModels}
/>
{enabled && (
<div className="mb-4 text-sm text-muted-foreground">
<p role="status">
{!current && "Loading models from NVIDIA NIM"}
{current?.status === "success" &&
"Models listed by your NIM endpoint. Inference access and capabilities may vary"}
{current?.status === "unavailable" &&
"Could not refresh NIM models. Showing the built-in catalog. You can also enter a custom model"}
</p>
<Button
type="button"
variant="ghost"
size="sm"
disabled={!current}
onClick={() => setAttempt((value) => value + 1)}
>
Refresh NIM models
</Button>
</div>
)}
</>
);
}

View file

@ -3947,6 +3947,29 @@ export interface paths {
patch?: never;
trace?: never;
};
"/credentials/nvidia_nim/models": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
/**
* Discover Nim Models
* @description List upstream NIM models using a saved credential or submitted connection settings.
*
* Requires a proxy administrator. Returns model IDs only, without asserting inference access,
* pricing or capabilities. Upstream failures return unavailable so the UI can use its catalog.
*/
post: operations["discover_nim_models_credentials_nvidia_nim_models_post"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/credentials/user_connections": {
parameters: {
query?: never;
@ -40969,6 +40992,32 @@ export interface components {
/** User Role */
user_role?: ("proxy_admin" | "proxy_admin_viewer" | "internal_user" | "internal_user_viewer") | null;
};
/** NimModelDiscoveryConnection */
NimModelDiscoveryConnection: {
/** Api Base */
api_base?: string | null;
/** Api Key */
api_key?: string | null;
};
/** NimModelDiscoveryRequest */
NimModelDiscoveryRequest: {
/** Credential Name */
credential_name?: string | null;
credential_values?: components["schemas"]["NimModelDiscoveryConnection"] | null;
};
/** NimModelDiscoveryResponse */
NimModelDiscoveryResponse: {
/**
* Models
* @default []
*/
models: string[];
/**
* Status
* @enum {string}
*/
status: "success" | "unavailable";
};
/** NoulAnswer */
NoulAnswer: {
/** Noul */
@ -59170,6 +59219,39 @@ export interface operations {
};
};
};
discover_nim_models_credentials_nvidia_nim_models_post: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody: {
content: {
"application/json": components["schemas"]["NimModelDiscoveryRequest"];
};
};
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["NimModelDiscoveryResponse"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
list_user_connections_credentials_user_connections_get: {
parameters: {
query?: never;