mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
feat(nvidia_nim): discover upstream models in the dashboard
This commit is contained in:
parent
a94118c6cd
commit
ad4905e01c
7 changed files with 622 additions and 2 deletions
|
|
@ -67,6 +67,7 @@ from litellm.repositories.credentials_repository import CredentialsRepository
|
|||
from litellm.types.router import server_owned_wif_fields_named, server_owned_wif_fields_written
|
||||
from litellm.types.utils import CreateCredentialItem, CredentialItem
|
||||
|
||||
from .nim_model_discovery import router as nim_model_discovery_router
|
||||
from .user_provider_credentials import (
|
||||
GithubCopilotUserConnectionPayload,
|
||||
decode_user_provider_credential,
|
||||
|
|
@ -80,6 +81,7 @@ from .user_provider_credentials import (
|
|||
)
|
||||
|
||||
router: Final = APIRouter()
|
||||
router.include_router(nim_model_discovery_router)
|
||||
_CREDENTIAL_DICT_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
_GITHUB_COPILOT_PROVIDER: Final = "github_copilot"
|
||||
_DISPLAY_NAME_MAX_LENGTH: Final = 255
|
||||
|
|
|
|||
167
litellm/proxy/credential_endpoints/nim_model_discovery.py
Normal file
167
litellm/proxy/credential_endpoints/nim_model_discovery.py
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
import hashlib
|
||||
from collections.abc import AsyncIterator, Mapping
|
||||
from typing import Annotated, Final, Literal, Protocol
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from openai import AsyncOpenAI, OpenAIError
|
||||
from pydantic import BaseModel, ConfigDict, SecretStr, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils.credential_hydration import (
|
||||
hydrate_named_credential_authoritative,
|
||||
stored_credential_provider,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
|
||||
class _ModelCache(Protocol):
|
||||
def get_cache(self, key: str) -> object: ...
|
||||
|
||||
def set_cache(self, key: str, value: tuple[str, ...]) -> None: ...
|
||||
|
||||
|
||||
def _new_model_cache() -> _ModelCache:
|
||||
return InMemoryCache(default_ttl=60)
|
||||
|
||||
|
||||
router: Final = APIRouter()
|
||||
_CACHE: Final = _new_model_cache()
|
||||
_MODELS: Final = TypeAdapter(tuple[str, ...])
|
||||
_DEFAULT_API_BASE: Final = "https://integrate.api.nvidia.com/v1"
|
||||
|
||||
|
||||
class _CredentialMetadata(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, from_attributes=True)
|
||||
|
||||
credential_info: Mapping[str, object]
|
||||
|
||||
|
||||
class _ListedModel(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, strict=True)
|
||||
|
||||
id: str
|
||||
|
||||
|
||||
class _ModelListing(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
data: tuple[_ListedModel, ...]
|
||||
|
||||
|
||||
class NimModelDiscoveryConnection(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
api_key: SecretStr | None = None
|
||||
api_base: str | None = None
|
||||
|
||||
|
||||
class NimModelDiscoveryRequest(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
credential_name: str | None = None
|
||||
credential_values: NimModelDiscoveryConnection | None = None
|
||||
|
||||
|
||||
class NimModelDiscoveryResponse(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
status: Literal["success", "unavailable"]
|
||||
models: tuple[str, ...] = ()
|
||||
|
||||
|
||||
def _resolved_secret(value: str | None) -> str | None:
|
||||
if value is not None and value.startswith("os.environ/"):
|
||||
return get_secret_str(value.removeprefix("os.environ/"))
|
||||
return value
|
||||
|
||||
|
||||
def _text(value: object) -> str | None:
|
||||
return value if isinstance(value, str) and value else None
|
||||
|
||||
|
||||
async def nim_http_client() -> AsyncIterator[httpx.AsyncClient]:
|
||||
async with httpx.AsyncClient(follow_redirects=False) as client:
|
||||
yield client
|
||||
|
||||
|
||||
@router.post(
|
||||
"/credentials/nvidia_nim/models",
|
||||
tags=["credential management"],
|
||||
response_model=NimModelDiscoveryResponse,
|
||||
)
|
||||
async def discover_nim_models(
|
||||
body: NimModelDiscoveryRequest,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
http_client: Annotated[httpx.AsyncClient, Depends(nim_http_client)],
|
||||
) -> NimModelDiscoveryResponse:
|
||||
"""List upstream NIM models using a saved credential or submitted connection settings.
|
||||
|
||||
Requires a proxy administrator. Returns model IDs only, without asserting inference access,
|
||||
pricing or capabilities. Upstream failures return unavailable so the UI can use its catalog.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
|
||||
raise HTTPException(status_code=403, detail="Only proxy admins can discover NIM models")
|
||||
if body.credential_name and body.credential_values is not None:
|
||||
raise HTTPException(status_code=400, detail="Saved credentials cannot be combined with connection overrides")
|
||||
credential: Final = (
|
||||
await hydrate_named_credential_authoritative(body.credential_name, prisma_client)
|
||||
if body.credential_name
|
||||
else None
|
||||
)
|
||||
if body.credential_name and credential is None:
|
||||
raise HTTPException(status_code=404, detail="Credential not found")
|
||||
provider: Final = (
|
||||
stored_credential_provider(
|
||||
_CredentialMetadata.model_validate(credential).credential_info.get("custom_llm_provider")
|
||||
)
|
||||
if credential is not None
|
||||
else None
|
||||
)
|
||||
if provider is not None and provider != "nvidia_nim":
|
||||
raise HTTPException(status_code=400, detail="Select an NVIDIA NIM credential")
|
||||
submitted: Final = body.credential_values or NimModelDiscoveryConnection()
|
||||
values: Final = credential.credential_values if credential is not None else {}
|
||||
api_base: Final = _resolved_secret(_text(values.get("api_base")) or submitted.api_base) or (
|
||||
get_secret_str("NVIDIA_NIM_API_BASE") or _DEFAULT_API_BASE
|
||||
)
|
||||
try:
|
||||
parsed: Final = urlsplit(api_base)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail="Invalid NIM API base") from None
|
||||
if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username or parsed.password:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="API base must be an HTTP or HTTPS endpoint without user information"
|
||||
)
|
||||
if parsed.query or parsed.fragment:
|
||||
raise HTTPException(status_code=400, detail="API base cannot contain a query or fragment")
|
||||
api_key: Final = _resolved_secret(
|
||||
_text(values.get("api_key"))
|
||||
or (submitted.api_key.get_secret_value() if submitted.api_key is not None else None)
|
||||
) or get_secret_str("NVIDIA_NIM_API_KEY")
|
||||
cache_key: Final = hashlib.sha256(f"{api_base.rstrip('/')}\0{api_key or ''}".encode()).hexdigest()
|
||||
cached: Final = _CACHE.get_cache(cache_key)
|
||||
if cached is not None:
|
||||
return NimModelDiscoveryResponse(status="success", models=_MODELS.validate_python(cached))
|
||||
try:
|
||||
async with AsyncOpenAI(
|
||||
api_key=api_key or "not-required",
|
||||
base_url=api_base,
|
||||
timeout=10,
|
||||
max_retries=0,
|
||||
http_client=http_client,
|
||||
) as client:
|
||||
page: Final = await client.models.list()
|
||||
listing: Final = _ModelListing.model_validate(page.model_dump())
|
||||
models: Final = tuple(sorted(frozenset(f"nvidia_nim/{item.id}" for item in listing.data if item.id)))
|
||||
if not models:
|
||||
return NimModelDiscoveryResponse(status="unavailable")
|
||||
_CACHE.set_cache(cache_key, models)
|
||||
return NimModelDiscoveryResponse(status="success", models=models)
|
||||
except (OpenAIError, httpx.HTTPError, ValidationError):
|
||||
return NimModelDiscoveryResponse(status="unavailable")
|
||||
|
|
@ -0,0 +1,135 @@
|
|||
from collections.abc import Iterator
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import litellm
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.credential_endpoints.nim_model_discovery import nim_http_client, router
|
||||
from litellm.types.utils import CredentialItem
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app() -> FastAPI:
|
||||
application: Final = FastAPI()
|
||||
application.include_router(router)
|
||||
application.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
return application
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(app: FastAPI) -> Iterator[TestClient]:
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", None),
|
||||
patch.object(litellm, "credential_list", []),
|
||||
patch("litellm.proxy.credential_endpoints.nim_model_discovery._CACHE", InMemoryCache()),
|
||||
):
|
||||
with TestClient(app) as test_client:
|
||||
yield test_client
|
||||
|
||||
|
||||
def test_saved_credential_controls_the_upstream_and_only_ids_return(client: TestClient, app: FastAPI) -> None:
|
||||
credential: Final = CredentialItem(
|
||||
credential_name="nim-saved",
|
||||
credential_info={"custom_llm_provider": "nvidia_nim"},
|
||||
credential_values={"api_key": "saved-secret", "api_base": "https://nim.example/v1"},
|
||||
)
|
||||
|
||||
def upstream(request: httpx.Request) -> httpx.Response:
|
||||
assert str(request.url) == "https://nim.example/v1/models"
|
||||
assert request.headers["authorization"] == "Bearer saved-secret"
|
||||
return httpx.Response(200, json={"object": "list", "data": [{"id": "new-model"}, {"id": "new-model"}]})
|
||||
|
||||
async def connection() -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(transport=httpx.MockTransport(upstream))
|
||||
|
||||
app.dependency_overrides[nim_http_client] = connection
|
||||
with patch.object(litellm, "credential_list", [credential]):
|
||||
response: Final = client.post("/credentials/nvidia_nim/models", json={"credential_name": "nim-saved"})
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == {"status": "success", "models": ["nvidia_nim/new-model"]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", (401, 403, 429, 500))
|
||||
def test_upstream_errors_allow_catalog_fallback_without_leaking_secrets(
|
||||
client: TestClient, app: FastAPI, status: int
|
||||
) -> None:
|
||||
async def connection() -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(
|
||||
transport=httpx.MockTransport(lambda request: httpx.Response(status, json={"error": "private-details"}))
|
||||
)
|
||||
|
||||
app.dependency_overrides[nim_http_client] = connection
|
||||
response: Final = client.post(
|
||||
"/credentials/nvidia_nim/models", json={"credential_values": {"api_key": "private-key"}}
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == {"status": "unavailable", "models": []}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("role", (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY))
|
||||
def test_only_admins_can_use_saved_credentials_for_discovery(
|
||||
client: TestClient, app: FastAPI, role: LitellmUserRoles
|
||||
) -> None:
|
||||
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=role)
|
||||
response: Final = client.post("/credentials/nvidia_nim/models", json={"credential_name": "nim-saved"})
|
||||
assert response.status_code == 403, response.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("override", ({"api_base": "https://elsewhere.example/v1"}, {"api_key": "replacement"}))
|
||||
def test_cannot_redirect_a_saved_credential(client: TestClient, override: dict[str, str]) -> None:
|
||||
response: Final = client.post(
|
||||
"/credentials/nvidia_nim/models", json={"credential_name": "nim-saved", "credential_values": override}
|
||||
)
|
||||
assert response.status_code == 400, response.text
|
||||
|
||||
|
||||
def test_missing_credential_is_not_replaced_with_environment_credentials(client: TestClient, app: FastAPI) -> None:
|
||||
response: Final = client.post("/credentials/nvidia_nim/models", json={"credential_name": "missing"})
|
||||
assert response.status_code == 404, response.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("listing", ({"data": []}, {"data": [{"id": 42}]}, {"data": None}))
|
||||
def test_empty_or_malformed_listing_allows_catalog_fallback(
|
||||
client: TestClient, app: FastAPI, listing: dict[str, object]
|
||||
) -> None:
|
||||
async def connection() -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(transport=httpx.MockTransport(lambda request: httpx.Response(200, json=listing)))
|
||||
|
||||
app.dependency_overrides[nim_http_client] = connection
|
||||
response: Final = client.post("/credentials/nvidia_nim/models", json={"credential_values": {"api_key": "test-key"}})
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == {"status": "unavailable", "models": []}
|
||||
|
||||
|
||||
def test_cached_models_are_scoped_to_endpoint_and_key(client: TestClient, app: FastAPI) -> None:
|
||||
async def success() -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(
|
||||
transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": [{"id": "cached"}]}))
|
||||
)
|
||||
|
||||
async def failure() -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(transport=httpx.MockTransport(lambda request: httpx.Response(503)))
|
||||
|
||||
app.dependency_overrides[nim_http_client] = success
|
||||
original: Final = {"api_base": "https://nim.example/v1", "api_key": "first-key"}
|
||||
first: Final = client.post("/credentials/nvidia_nim/models", json={"credential_values": original})
|
||||
assert first.json() == {"status": "success", "models": ["nvidia_nim/cached"]}
|
||||
app.dependency_overrides[nim_http_client] = failure
|
||||
cached: Final = client.post("/credentials/nvidia_nim/models", json={"credential_values": original})
|
||||
assert cached.json() == first.json()
|
||||
other_key: Final = client.post(
|
||||
"/credentials/nvidia_nim/models", json={"credential_values": {**original, "api_key": "other-key"}}
|
||||
)
|
||||
assert other_key.json() == {"status": "unavailable", "models": []}
|
||||
other_endpoint: Final = client.post(
|
||||
"/credentials/nvidia_nim/models",
|
||||
json={"credential_values": {**original, "api_base": "https://other.example/v1"}},
|
||||
)
|
||||
assert other_endpoint.json() == {"status": "unavailable", "models": []}
|
||||
|
|
@ -34,7 +34,7 @@ import { ProviderLogo } from "../molecules/models/ProviderLogo";
|
|||
import AccessGroupTagsCombobox from "./AccessGroupTagsCombobox";
|
||||
import AdvancedSettings from "./advanced_settings";
|
||||
import ConditionalPublicModelName from "./conditional_public_model_name";
|
||||
import LiteLLMModelNameField from "./litellm_model_name";
|
||||
import NimModelNameField from "./NimModelNameField";
|
||||
import ConnectionErrorDisplay from "./model_connection_test";
|
||||
import ProviderSpecificFields from "./provider_specific_fields";
|
||||
import { authTypesFor } from "./provider_auth_types";
|
||||
|
|
@ -295,7 +295,9 @@ const AddModelForm: React.FC<AddModelFormProps> = ({
|
|||
/>
|
||||
)}
|
||||
</MountedFormField>
|
||||
<LiteLLMModelNameField
|
||||
<NimModelNameField
|
||||
accessToken={accessToken}
|
||||
canDiscover={canCreateCredential}
|
||||
selectedProvider={selectedProvider}
|
||||
providerModels={providerModels}
|
||||
getPlaceholder={getPlaceholder}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,124 @@
|
|||
import { act, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { describe, expect, it, vi, beforeEach } from "vitest";
|
||||
import { useFormContext } from "react-hook-form";
|
||||
import { MountedFormHost } from "../../../tests/mounted-form-host";
|
||||
import type { MountedFormValues } from "../common_components/MountedFormField";
|
||||
import { apiClient } from "../networking";
|
||||
import { getPlaceholder, Providers } from "../provider_info_helpers";
|
||||
import NimModelNameField from "./NimModelNameField";
|
||||
|
||||
vi.mock("../networking", () => ({ apiClient: { post: vi.fn() } }));
|
||||
const catalogModel = "nvidia_nim/catalog-model";
|
||||
const discoveredModel = "nvidia_nim/discovered-model";
|
||||
|
||||
function ConnectionFields() {
|
||||
const { register } = useFormContext<MountedFormValues>();
|
||||
return <input aria-label="Credential" {...register("litellm_credential_name")} />;
|
||||
}
|
||||
|
||||
function Form({
|
||||
provider = "NVIDIA_NIM",
|
||||
canDiscover = true,
|
||||
credentialName = "nim-saved",
|
||||
apiKey,
|
||||
apiBase,
|
||||
}: {
|
||||
provider?: string;
|
||||
canDiscover?: boolean;
|
||||
credentialName?: string;
|
||||
apiKey?: string;
|
||||
apiBase?: string;
|
||||
} = {}) {
|
||||
return (
|
||||
<MountedFormHost defaultValues={{ litellm_credential_name: credentialName, api_key: apiKey, api_base: apiBase }}>
|
||||
<ConnectionFields />
|
||||
<NimModelNameField
|
||||
selectedProvider={provider}
|
||||
providerModels={[catalogModel]}
|
||||
getPlaceholder={getPlaceholder}
|
||||
accessToken="proxy-token"
|
||||
canDiscover={canDiscover}
|
||||
/>
|
||||
</MountedFormHost>
|
||||
);
|
||||
}
|
||||
|
||||
beforeEach(() => vi.resetAllMocks());
|
||||
|
||||
describe("NIM model discovery", () => {
|
||||
it("uses the selected credential and makes discovered models selectable", async () => {
|
||||
vi.mocked(apiClient.post).mockResolvedValue({ status: "success", models: [discoveredModel] });
|
||||
render(<Form />);
|
||||
expect(await screen.findByText(/Models listed by your NIM endpoint/)).toBeInTheDocument();
|
||||
expect(apiClient.post).toHaveBeenCalledWith("/credentials/nvidia_nim/models", {
|
||||
accessToken: "proxy-token",
|
||||
body: { credential_name: "nim-saved", credential_values: undefined },
|
||||
signal: expect.any(AbortSignal),
|
||||
});
|
||||
const user = userEvent.setup();
|
||||
await user.click(screen.getByRole("combobox", { name: "Select models" }));
|
||||
expect(screen.getByRole("option", { name: discoveredModel })).toBeInTheDocument();
|
||||
expect(screen.queryByRole("option", { name: catalogModel })).not.toBeInTheDocument();
|
||||
expect(screen.getByRole("option", { name: "Custom Model Name (Enter below)" })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("passes inline connection settings without a saved credential", async () => {
|
||||
vi.mocked(apiClient.post).mockResolvedValue({ status: "success", models: [discoveredModel] });
|
||||
render(<Form credentialName="" apiKey="inline-key" apiBase="https://nim.example/v1" />);
|
||||
expect(await screen.findByText(/Models listed by your NIM endpoint/)).toBeInTheDocument();
|
||||
expect(apiClient.post).toHaveBeenCalledWith("/credentials/nvidia_nim/models", {
|
||||
accessToken: "proxy-token",
|
||||
body: {
|
||||
credential_name: undefined,
|
||||
credential_values: { api_key: "inline-key", api_base: "https://nim.example/v1" },
|
||||
},
|
||||
signal: expect.any(AbortSignal),
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps the catalog usable when upstream discovery is unavailable", async () => {
|
||||
vi.mocked(apiClient.post).mockResolvedValue({ status: "unavailable", models: [] });
|
||||
render(<Form />);
|
||||
expect(await screen.findByText(/Could not refresh NIM models/)).toBeInTheDocument();
|
||||
const user = userEvent.setup();
|
||||
await user.click(screen.getByRole("combobox", { name: "Select models" }));
|
||||
expect(screen.getByRole("option", { name: catalogModel })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("ignores a late response for a previous credential", async () => {
|
||||
let completeOld: (value: unknown) => void = () => {};
|
||||
vi.mocked(apiClient.post)
|
||||
.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
completeOld = resolve;
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce({ status: "success", models: [discoveredModel] });
|
||||
render(<Form />);
|
||||
await waitFor(() => expect(apiClient.post).toHaveBeenCalledTimes(1));
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Credential" }), { target: { value: "nim-new" } });
|
||||
expect(await screen.findByText(/Models listed by your NIM endpoint/)).toBeInTheDocument();
|
||||
completeOld({ status: "success", models: ["nvidia_nim/old-credential-model"] });
|
||||
const user = userEvent.setup();
|
||||
await user.click(screen.getByRole("combobox", { name: "Select models" }));
|
||||
expect(screen.getByRole("option", { name: discoveredModel })).toBeInTheDocument();
|
||||
expect(screen.queryByRole("option", { name: "nvidia_nim/old-credential-model" })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it.each([{ provider: Providers.OpenAI }, { canDiscover: false }])(
|
||||
"leaves other providers and roles on the catalog",
|
||||
async (props) => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
render(<Form {...props} />);
|
||||
await act(() => vi.advanceTimersByTimeAsync(600));
|
||||
expect(screen.queryByRole("button", { name: "Refresh NIM models" })).not.toBeInTheDocument();
|
||||
expect(apiClient.post).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
import { useEffect, useMemo, useState, type ComponentProps } from "react";
|
||||
import { useFormContext, useWatch } from "react-hook-form";
|
||||
import { apiClient } from "../networking";
|
||||
import { resolveLitellmProviderSlug } from "../provider_info_helpers";
|
||||
import type { MountedFormValues } from "../common_components/MountedFormField";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import LiteLLMModelNameField from "./litellm_model_name";
|
||||
|
||||
type DiscoveryResponse = {
|
||||
status: "success" | "unavailable";
|
||||
models: string[];
|
||||
};
|
||||
|
||||
type Props = ComponentProps<typeof LiteLLMModelNameField> & {
|
||||
accessToken: string | null;
|
||||
canDiscover: boolean;
|
||||
};
|
||||
|
||||
export default function NimModelNameField({ accessToken, canDiscover, ...props }: Props) {
|
||||
const { control } = useFormContext<MountedFormValues>();
|
||||
const credential = useWatch({ control, name: "litellm_credential_name" });
|
||||
const apiKey = useWatch({ control, name: "api_key" });
|
||||
const apiBase = useWatch({ control, name: "api_base" });
|
||||
const enabled =
|
||||
canDiscover &&
|
||||
props.selectedProvider !== null &&
|
||||
resolveLitellmProviderSlug(props.selectedProvider) === "nvidia_nim";
|
||||
const connection = useMemo(
|
||||
() => ({
|
||||
credential_name: typeof credential === "string" && credential ? credential : undefined,
|
||||
credential_values: credential
|
||||
? undefined
|
||||
: {
|
||||
api_key: typeof apiKey === "string" && apiKey ? apiKey : undefined,
|
||||
api_base: typeof apiBase === "string" && apiBase ? apiBase : undefined,
|
||||
},
|
||||
}),
|
||||
[credential, apiKey, apiBase],
|
||||
);
|
||||
const [attempt, setAttempt] = useState(0);
|
||||
const [result, setResult] = useState<{
|
||||
connection: typeof connection;
|
||||
accessToken: string | null;
|
||||
attempt: number;
|
||||
response: DiscoveryResponse;
|
||||
}>();
|
||||
const current =
|
||||
result?.connection === connection && result.accessToken === accessToken && result.attempt === attempt
|
||||
? result.response
|
||||
: undefined;
|
||||
|
||||
useEffect(() => {
|
||||
if (!enabled) return;
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(async () => {
|
||||
try {
|
||||
const response = await apiClient.post<DiscoveryResponse>("/credentials/nvidia_nim/models", {
|
||||
accessToken,
|
||||
body: connection,
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!controller.signal.aborted) {
|
||||
const nextResult = { connection, accessToken, attempt, response };
|
||||
setResult(nextResult);
|
||||
}
|
||||
} catch {
|
||||
if (!controller.signal.aborted) {
|
||||
const response: DiscoveryResponse = { status: "unavailable", models: [] };
|
||||
const nextResult = { connection, accessToken, attempt, response };
|
||||
setResult(nextResult);
|
||||
}
|
||||
}
|
||||
}, 500);
|
||||
return () => {
|
||||
clearTimeout(timer);
|
||||
controller.abort();
|
||||
};
|
||||
}, [enabled, connection, accessToken, attempt]);
|
||||
|
||||
return (
|
||||
<>
|
||||
<LiteLLMModelNameField
|
||||
{...props}
|
||||
providerModels={enabled && current?.status === "success" ? current.models : props.providerModels}
|
||||
/>
|
||||
{enabled && (
|
||||
<div className="mb-4 text-sm text-muted-foreground">
|
||||
<p role="status">
|
||||
{!current && "Loading models from NVIDIA NIM"}
|
||||
{current?.status === "success" &&
|
||||
"Models listed by your NIM endpoint. Inference access and capabilities may vary"}
|
||||
{current?.status === "unavailable" &&
|
||||
"Could not refresh NIM models. Showing the built-in catalog. You can also enter a custom model"}
|
||||
</p>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
disabled={!current}
|
||||
onClick={() => setAttempt((value) => value + 1)}
|
||||
>
|
||||
Refresh NIM models
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
82
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
82
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -3947,6 +3947,29 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/credentials/nvidia_nim/models": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/**
|
||||
* Discover Nim Models
|
||||
* @description List upstream NIM models using a saved credential or submitted connection settings.
|
||||
*
|
||||
* Requires a proxy administrator. Returns model IDs only, without asserting inference access,
|
||||
* pricing or capabilities. Upstream failures return unavailable so the UI can use its catalog.
|
||||
*/
|
||||
post: operations["discover_nim_models_credentials_nvidia_nim_models_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/credentials/user_connections": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -40969,6 +40992,32 @@ export interface components {
|
|||
/** User Role */
|
||||
user_role?: ("proxy_admin" | "proxy_admin_viewer" | "internal_user" | "internal_user_viewer") | null;
|
||||
};
|
||||
/** NimModelDiscoveryConnection */
|
||||
NimModelDiscoveryConnection: {
|
||||
/** Api Base */
|
||||
api_base?: string | null;
|
||||
/** Api Key */
|
||||
api_key?: string | null;
|
||||
};
|
||||
/** NimModelDiscoveryRequest */
|
||||
NimModelDiscoveryRequest: {
|
||||
/** Credential Name */
|
||||
credential_name?: string | null;
|
||||
credential_values?: components["schemas"]["NimModelDiscoveryConnection"] | null;
|
||||
};
|
||||
/** NimModelDiscoveryResponse */
|
||||
NimModelDiscoveryResponse: {
|
||||
/**
|
||||
* Models
|
||||
* @default []
|
||||
*/
|
||||
models: string[];
|
||||
/**
|
||||
* Status
|
||||
* @enum {string}
|
||||
*/
|
||||
status: "success" | "unavailable";
|
||||
};
|
||||
/** NoulAnswer */
|
||||
NoulAnswer: {
|
||||
/** Noul */
|
||||
|
|
@ -59170,6 +59219,39 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
discover_nim_models_credentials_nvidia_nim_models_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["NimModelDiscoveryRequest"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["NimModelDiscoveryResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
list_user_connections_credentials_user_connections_get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue