mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
feat(model hub): show admin-defined custom tags and metadata from model_info (#46064)
* feat(model hub): show admin-defined custom tags and metadata from model_info Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(router): move model hub custom field merging into router_utils Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(model hub): share the custom tag badge list between the column and details Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(model hub): cover public hub, patch, null, hostile, forwarding, and killed-worker paths for custom fields Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model hub): drop custom_metadata entries with non-string keys or non-finite numbers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(model hub): use isinstance narrowing instead of TypeGuard for custom_metadata values Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(model hub): build custom tags and metadata in the proxy instead of the router Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model hub): serve custom tags and metadata on model_group_alias rows Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * remove docstring from model_group_custom_fields.py * style(model hub): fix lint and format in custom metadata tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(model hub): drop leading blank line in custom fields helper Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
jesus-berri
GitHub
parent
5e1e66d1ee
commit
c8f7e0de7d
14 changed files with 604 additions and 8 deletions
|
|
@ -0,0 +1,68 @@
|
|||
import math
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
CustomMetadataValue: TypeAlias = str | int | float | bool
|
||||
|
||||
_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[object, object])
|
||||
_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
|
||||
|
||||
|
||||
def _string_tags(tags: object) -> Iterator[str]:
|
||||
if isinstance(tags, list):
|
||||
yield from (tag for tag in _OBJECT_LIST_ADAPTER.validate_python(tags) if isinstance(tag, str))
|
||||
|
||||
|
||||
def _is_non_finite(value: object) -> bool:
|
||||
return isinstance(value, float) and not math.isfinite(value)
|
||||
|
||||
|
||||
def _scalar_metadata(metadata: object) -> Iterator[tuple[str, CustomMetadataValue]]:
|
||||
if isinstance(metadata, dict):
|
||||
yield from (
|
||||
(key, value)
|
||||
for key, value in _OBJECT_MAPPING_ADAPTER.validate_python(metadata).items()
|
||||
if isinstance(key, str) and isinstance(value, (str, int, float, bool)) and not _is_non_finite(value)
|
||||
)
|
||||
|
||||
|
||||
def _custom_tag_values(model_infos: Sequence[Mapping[object, object]]) -> Iterator[str]:
|
||||
for model_info in model_infos:
|
||||
yield from _string_tags(model_info.get("custom_tags"))
|
||||
|
||||
|
||||
def _custom_metadata_entries(
|
||||
model_infos: Sequence[Mapping[object, object]],
|
||||
) -> Iterator[tuple[str, CustomMetadataValue]]:
|
||||
for model_info in model_infos:
|
||||
yield from _scalar_metadata(model_info.get("custom_metadata"))
|
||||
|
||||
|
||||
def _merged_custom_tags(model_infos: Sequence[Mapping[object, object]]) -> tuple[str, ...] | None:
|
||||
return tuple(dict.fromkeys(_custom_tag_values(model_infos))) or None
|
||||
|
||||
|
||||
def _merged_custom_metadata(
|
||||
model_infos: Sequence[Mapping[object, object]],
|
||||
) -> Mapping[str, CustomMetadataValue] | None:
|
||||
entries: Final = tuple(_custom_metadata_entries(model_infos))
|
||||
first_seen: Final = MappingProxyType(dict(reversed(entries)))
|
||||
return MappingProxyType({key: first_seen[key] for key in dict.fromkeys(key for key, _ in entries)}) or None
|
||||
|
||||
|
||||
def _model_info(deployment: Mapping[str, object]) -> Mapping[object, object]:
|
||||
model_info: Final = deployment.get("model_info")
|
||||
return _OBJECT_MAPPING_ADAPTER.validate_python(model_info) if isinstance(model_info, dict) else MappingProxyType({})
|
||||
|
||||
|
||||
def model_group_custom_fields(deployments: Sequence[Mapping[str, object]]) -> Mapping[str, object]:
|
||||
model_infos: Final = tuple(_model_info(deployment) for deployment in deployments)
|
||||
return MappingProxyType(
|
||||
{
|
||||
"custom_tags": _merged_custom_tags(model_infos),
|
||||
"custom_metadata": _merged_custom_metadata(model_infos),
|
||||
}
|
||||
)
|
||||
|
|
@ -166,6 +166,7 @@ from litellm.proxy.common_utils.token_count_settings import (
|
|||
resolve_token_count_settings,
|
||||
)
|
||||
from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog
|
||||
from litellm.proxy.management_helpers.model_group_custom_fields import model_group_custom_fields
|
||||
from litellm.router_utils.access_windows import access_windows_config_error
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
get_fallback_errors_from_headers,
|
||||
|
|
@ -17747,10 +17748,7 @@ def get_model_group_info(
|
|||
) -> list[ModelGroupInfoProxy]:
|
||||
model_groups: Final[list[ModelGroupInfoProxy]] = []
|
||||
|
||||
unique_models: Final = []
|
||||
for model in all_models_str:
|
||||
if model not in unique_models:
|
||||
unique_models.append(model)
|
||||
unique_models: Final = tuple(dict.fromkeys(all_models_str))
|
||||
|
||||
for model in unique_models:
|
||||
if model_group is not None and model_group != model:
|
||||
|
|
@ -17759,7 +17757,18 @@ def get_model_group_info(
|
|||
_model_group_info = llm_router.get_model_group_info(model_group=model, team_id=team_id)
|
||||
|
||||
if _model_group_info is not None:
|
||||
model_groups.append(ModelGroupInfoProxy.model_validate(_model_group_info.model_dump()))
|
||||
model_groups.append(
|
||||
ModelGroupInfoProxy.model_validate(
|
||||
{
|
||||
**_model_group_info.model_dump(),
|
||||
**model_group_custom_fields(
|
||||
llm_router.get_model_list_of_routed_group(
|
||||
resolve_model_group_alias(llm_router.model_group_alias, model) or model
|
||||
)
|
||||
),
|
||||
}
|
||||
)
|
||||
)
|
||||
else:
|
||||
model_group_info = ModelGroupInfoProxy(
|
||||
model_group=model,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from collections.abc import Mapping
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
|
|
@ -13,6 +14,8 @@ class ModelGroupInfoProxy(ModelGroupInfo):
|
|||
health_status: str | None = Field(default=None)
|
||||
health_response_time: float | None = Field(default=None)
|
||||
health_checked_at: str | None = Field(default=None)
|
||||
custom_tags: tuple[str, ...] | None = None
|
||||
custom_metadata: Mapping[str, str | int | float | bool] | None = None
|
||||
|
||||
|
||||
class UpdateUsefulLinksRequest(LiteLLMBaseModel):
|
||||
|
|
|
|||
276
tests/integration/management/test_model_hub_custom_metadata.py
Normal file
276
tests/integration/management/test_model_hub_custom_metadata.py
Normal file
|
|
@ -0,0 +1,276 @@
|
|||
import math
|
||||
import uuid
|
||||
from collections.abc import Mapping
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
from pydantic import JsonValue
|
||||
|
||||
from litellm.constants import PROXY_CONFIG_RELOAD_INTERVAL_SECONDS
|
||||
from tests.integration._support.client import Gateway, Scenario, eventually, object_value, string_value
|
||||
from tests.integration._support.process import graceful_stop_seconds, owned_proxy_process
|
||||
|
||||
LIFECYCLE_METADATA: Final = {"retirement_date": "2027-03-01", "ptu_max_tpm": 300000, "provisioned_ptu": 100}
|
||||
|
||||
|
||||
def _model_group(gateway: Gateway, model_group: str) -> dict[str, JsonValue]:
|
||||
data: Final = gateway.get("/model_group/info", params={"model_group": model_group})["data"]
|
||||
assert isinstance(data, list) and len(data) == 1, data
|
||||
return object_value(data[0])
|
||||
|
||||
|
||||
def _deployment(gateway: Gateway, scenario: Scenario, model_name: str, model_info: dict[str, JsonValue]) -> str:
|
||||
created: Final = gateway.post(
|
||||
"/model/new",
|
||||
{
|
||||
"model_name": model_name,
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_key": "integration-provider-key",
|
||||
"api_base": f"{gateway.upstream_url}/v1",
|
||||
},
|
||||
"model_info": model_info,
|
||||
},
|
||||
)
|
||||
identity: Final = string_value(object_value(created["model_info"])["id"])
|
||||
scenario.cleanups.callback(scenario.delete_model, identity)
|
||||
return identity
|
||||
|
||||
|
||||
def _custom_fields(row: dict[str, JsonValue]) -> tuple[JsonValue, JsonValue]:
|
||||
assert "custom_tags" in row and "custom_metadata" in row, row
|
||||
return row["custom_tags"], row["custom_metadata"]
|
||||
|
||||
|
||||
def _public_row(gateway: Gateway, path: str, model_group: str) -> dict[str, JsonValue] | None:
|
||||
response: Final = gateway.client.get(path)
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = response.json()
|
||||
rows: Final = body if isinstance(body, list) else object_value(body)["data"]
|
||||
assert isinstance(rows, list), response.text
|
||||
return next((object_value(row) for row in rows if object_value(row).get("model_group") == model_group), None)
|
||||
|
||||
|
||||
def test_model_group_info_serves_the_custom_tags_and_metadata_set_on_the_model(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model_info={"custom_tags": ["deprecated", "ptu"], "custom_metadata": LIFECYCLE_METADATA}
|
||||
)
|
||||
group: Final = _model_group(gateway, model)
|
||||
assert group.get("custom_tags") == ["deprecated", "ptu"], group
|
||||
assert group.get("custom_metadata") == LIFECYCLE_METADATA, group
|
||||
|
||||
|
||||
def test_model_group_info_merges_custom_tags_and_metadata_across_the_groups_deployments(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model_name: Final = f"integration-{uuid.uuid4().hex}"
|
||||
_deployment(
|
||||
gateway,
|
||||
scenario,
|
||||
model_name,
|
||||
{"custom_tags": ["ptu", "eu"], "custom_metadata": {"provisioned_ptu": 100, "region": "westeurope"}},
|
||||
)
|
||||
_deployment(
|
||||
gateway,
|
||||
scenario,
|
||||
model_name,
|
||||
{
|
||||
"custom_tags": ["eu", "deprecated"],
|
||||
"custom_metadata": {"provisioned_ptu": 50, "retirement_date": "2027-03-01"},
|
||||
},
|
||||
)
|
||||
group: Final = _model_group(gateway, model_name)
|
||||
assert sorted(string_value(tag) for tag in group.get("custom_tags") or []) == ["deprecated", "eu", "ptu"], group
|
||||
metadata: Final = object_value(group.get("custom_metadata"))
|
||||
assert metadata.get("region") == "westeurope", metadata
|
||||
assert metadata.get("retirement_date") == "2027-03-01", metadata
|
||||
assert metadata.get("provisioned_ptu") in (100, 50), metadata
|
||||
|
||||
|
||||
def test_model_group_info_ignores_custom_metadata_values_that_are_not_scalars(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model_info={
|
||||
"custom_tags": ["ok", 7, {"nested": True}],
|
||||
"custom_metadata": {"owner": "platform", "nested": {"a": 1}, "list": [1, 2]},
|
||||
}
|
||||
)
|
||||
group: Final = _model_group(gateway, model)
|
||||
assert group.get("custom_tags") == ["ok"], group
|
||||
assert group.get("custom_metadata") == {"owner": "platform"}, group
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/public/model_hub", "/public/v1/model_hub"])
|
||||
def test_public_model_hub_serves_the_custom_tags_and_metadata_of_a_public_model_group(
|
||||
gateway: Gateway, path: str
|
||||
) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model_info={"custom_tags": ["deprecated", "ptu"], "custom_metadata": LIFECYCLE_METADATA}
|
||||
)
|
||||
gateway.post("/model_group/make_public", {"model_groups": [model]})
|
||||
row: Final = eventually(
|
||||
lambda: _public_row(gateway, path, model),
|
||||
lambda found: found is not None,
|
||||
seconds=2 * PROXY_CONFIG_RELOAD_INTERVAL_SECONDS,
|
||||
)
|
||||
assert row is not None
|
||||
assert _custom_fields(row) == (["deprecated", "ptu"], LIFECYCLE_METADATA), row
|
||||
|
||||
|
||||
def test_patching_a_model_replaces_the_custom_tags_and_metadata_served_for_its_group(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model_name: Final = f"integration-{uuid.uuid4().hex}"
|
||||
identity: Final = _deployment(
|
||||
gateway, scenario, model_name, {"custom_tags": ["ptu"], "custom_metadata": LIFECYCLE_METADATA}
|
||||
)
|
||||
patched: Final = gateway.request(
|
||||
"PATCH",
|
||||
f"/model/{identity}/update",
|
||||
{"model_info": {"custom_tags": ["retiring"], "custom_metadata": {"retirement_date": "2027-06-01"}}},
|
||||
)
|
||||
assert patched.status_code == 200, patched.text
|
||||
group: Final = eventually(
|
||||
lambda: _model_group(gateway, model_name),
|
||||
lambda found: found.get("custom_tags") == ["retiring"],
|
||||
seconds=2 * PROXY_CONFIG_RELOAD_INTERVAL_SECONDS,
|
||||
)
|
||||
assert _custom_fields(group) == (["retiring"], {"retirement_date": "2027-06-01"}), group
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_info",
|
||||
[
|
||||
pytest.param({"description": "plain model"}, id="missing"),
|
||||
pytest.param({"custom_tags": [], "custom_metadata": {}}, id="empty"),
|
||||
pytest.param({"custom_tags": None, "custom_metadata": None}, id="null"),
|
||||
pytest.param({"custom_tags": "ptu", "custom_metadata": "retiring"}, id="wrong-shape"),
|
||||
pytest.param(
|
||||
{"custom_tags": [1, None, {"a": 1}], "custom_metadata": {"nested": {"a": 1}}}, id="no-valid-entry"
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_model_group_info_serves_null_custom_fields_when_the_model_sets_none_usable(
|
||||
gateway: Gateway, model_info: dict[str, JsonValue]
|
||||
) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model_info=model_info)
|
||||
group: Final = _model_group(gateway, model)
|
||||
assert _custom_fields(group) == (None, None), group
|
||||
|
||||
|
||||
def test_model_group_info_keeps_long_values_whole_and_serves_a_repeated_tag_once(gateway: Gateway) -> None:
|
||||
long_value: Final = "x" * 5000
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model_info={"custom_tags": ["ptu", "ptu", "eu", "ptu"], "custom_metadata": {"notes": long_value}}
|
||||
)
|
||||
group: Final = _model_group(gateway, model)
|
||||
assert _custom_fields(group) == (["ptu", "eu"], {"notes": long_value}), group
|
||||
|
||||
|
||||
def test_custom_tags_and_metadata_are_never_forwarded_to_the_provider(gateway: Gateway) -> None:
|
||||
marker: Final = f"custom-fields-{uuid.uuid4().hex}"
|
||||
with (
|
||||
gateway.scenario() as scenario,
|
||||
httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream,
|
||||
):
|
||||
model: Final = scenario.model(
|
||||
model_info={"custom_tags": ["deprecated"], "custom_metadata": {"retirement_date": "2027-03-01"}}
|
||||
)
|
||||
reply: Final = gateway.request(
|
||||
"POST", "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": marker}]}
|
||||
)
|
||||
assert reply.status_code == 200, reply.text
|
||||
observed: Final = upstream.get("/__observations")
|
||||
assert observed.status_code == 200, observed.text
|
||||
bodies: Final = [
|
||||
str(object_value(value)["body"])
|
||||
for value in object_value(observed.json())["requests"] or []
|
||||
if marker in str(object_value(value)["body"])
|
||||
]
|
||||
assert len(bodies) == 1, bodies
|
||||
assert "custom_tags" not in bodies[0] and "2027-03-01" not in bodies[0], bodies[0]
|
||||
|
||||
|
||||
def _yaml_config(
|
||||
directory: Path,
|
||||
upstream_url: str,
|
||||
model_name: str,
|
||||
model_info: Mapping[str, object] = MappingProxyType(
|
||||
{"custom_tags": ["deprecated", "ptu"], "custom_metadata": LIFECYCLE_METADATA}
|
||||
),
|
||||
model_group_alias: Mapping[str, str] = MappingProxyType({}),
|
||||
) -> Path:
|
||||
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
deployment: Final = {
|
||||
"model_name": model_name,
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_base": f"{upstream_url}/v1", "api_key": "yaml-key"},
|
||||
"model_info": dict(model_info),
|
||||
}
|
||||
path: Final = directory / "custom_fields.yaml"
|
||||
router_settings: Final = {**config.get("router_settings", {}), "model_group_alias": dict(model_group_alias)}
|
||||
path.write_text(
|
||||
yaml.safe_dump(
|
||||
{**config, "model_list": [*config["model_list"], deployment], "router_settings": router_settings}
|
||||
)
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
@pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
|
||||
def test_yaml_custom_fields_survive_a_killed_worker_on_every_worker(gateway: Gateway, tmp_path: Path) -> None:
|
||||
model_name: Final = f"integration-yaml-{uuid.uuid4().hex}"
|
||||
config: Final = _yaml_config(tmp_path, gateway.upstream_url, model_name)
|
||||
with owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2) as owned:
|
||||
victim: Final = min(owned.worker_pids())
|
||||
owned.kill_worker(victim)
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
groups: Final = tuple(pool.map(lambda _: _model_group(owned.gateway, model_name), range(24)))
|
||||
assert all(_custom_fields(group) == (["deprecated", "ptu"], LIFECYCLE_METADATA) for group in groups), groups
|
||||
|
||||
|
||||
@pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
|
||||
@pytest.mark.parametrize(
|
||||
"unservable_metadata",
|
||||
[
|
||||
pytest.param({2027: "retirement"}, id="non-string-key"),
|
||||
pytest.param({"nan": math.nan, "inf": math.inf, "neg_inf": -math.inf}, id="non-finite-number"),
|
||||
],
|
||||
)
|
||||
def test_yaml_custom_metadata_that_json_cannot_carry_is_dropped_without_breaking_completions_or_the_hub(
|
||||
gateway: Gateway, tmp_path: Path, unservable_metadata: Mapping[object, object]
|
||||
) -> None:
|
||||
model_name: Final = f"integration-yaml-{uuid.uuid4().hex}"
|
||||
config: Final = _yaml_config(
|
||||
tmp_path,
|
||||
gateway.upstream_url,
|
||||
model_name,
|
||||
{"custom_tags": ["ptu"], "custom_metadata": {**unservable_metadata, "retirement_date": "2027-03-01"}},
|
||||
)
|
||||
with owned_proxy_process(gateway, tmp_path, {}, config=config) as owned:
|
||||
reply: Final = owned.gateway.request(
|
||||
"POST", "/v1/chat/completions", {"model": model_name, "messages": [{"role": "user", "content": "hi"}]}
|
||||
)
|
||||
assert reply.status_code == 200, reply.text
|
||||
every_group: Final = owned.gateway.request("GET", "/model_group/info")
|
||||
assert every_group.status_code == 200, every_group.text
|
||||
group: Final = _model_group(owned.gateway, model_name)
|
||||
assert _custom_fields(group) == (["ptu"], {"retirement_date": "2027-03-01"}), group
|
||||
|
||||
|
||||
@pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
|
||||
def test_an_alias_group_serves_the_custom_tags_and_metadata_of_the_group_it_points_to(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
model_name: Final = f"integration-yaml-{uuid.uuid4().hex}"
|
||||
alias: Final = f"{model_name}-alias"
|
||||
config: Final = _yaml_config(tmp_path, gateway.upstream_url, model_name, model_group_alias={alias: model_name})
|
||||
with owned_proxy_process(gateway, tmp_path, {}, config=config) as owned:
|
||||
group: Final = _model_group(owned.gateway, alias)
|
||||
assert _custom_fields(group) == (["deprecated", "ptu"], LIFECYCLE_METADATA), group
|
||||
|
|
@ -10,7 +10,7 @@ from fastapi.testclient import TestClient
|
|||
import litellm
|
||||
from litellm.proxy._types import LiteLLMRoutes
|
||||
from litellm.proxy.proxy_server import app
|
||||
from litellm.types.router import ModelGroupInfo
|
||||
from litellm.types.router import DeploymentTypedDict, ModelGroupInfo
|
||||
from tests._master_key import MASTER_KEY
|
||||
|
||||
client = TestClient(app)
|
||||
|
|
@ -21,13 +21,17 @@ LEGACY_MODEL_HUB_PATH = "/public/model_hub"
|
|||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _FakeRouter:
|
||||
"""Stands in for the running Router: `_get_model_group_info` only ever asks it this."""
|
||||
"""Stands in for the running Router: `_get_model_group_info` only ever asks it these."""
|
||||
|
||||
infos: Mapping[str, ModelGroupInfo]
|
||||
model_group_alias: Mapping[str, str] = MappingProxyType({})
|
||||
|
||||
def get_model_group_info(self, model_group: str, team_id: str | None = None) -> ModelGroupInfo | None:
|
||||
return self.infos.get(model_group)
|
||||
|
||||
def get_model_list_of_routed_group(self, model_group: str) -> list[DeploymentTypedDict]:
|
||||
return []
|
||||
|
||||
|
||||
def _info(
|
||||
name: str,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import asyncio
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Final, List, Optional, Union
|
||||
from typing import Any, Dict, Final, List, Optional, TypeAlias, Union
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
|
@ -24,6 +24,7 @@ from litellm.proxy.litellm_pre_call_utils import (
|
|||
get_dynamic_logging_metadata,
|
||||
add_litellm_data_to_request,
|
||||
)
|
||||
from litellm.router import Router
|
||||
from pydantic import ValidationError
|
||||
|
||||
pytestmark = pytest.mark.xdist_group("proxy_heavy")
|
||||
|
|
@ -1602,6 +1603,109 @@ def test_get_model_group_info():
|
|||
assert len(model_list) == 1
|
||||
|
||||
|
||||
def _custom_metadata_router(*model_infos: dict[object, object]) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4",
|
||||
"litellm_params": {"model": "gpt-4", "api_key": f"fake-{index}"},
|
||||
"model_info": model_info,
|
||||
}
|
||||
for index, model_info in enumerate(model_infos)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
_CustomFieldsCase: TypeAlias = tuple[
|
||||
tuple[dict[object, object], ...], tuple[str, ...] | None, dict[str, str | int | float | bool] | None
|
||||
]
|
||||
_CUSTOM_FIELDS_CASES: Final[tuple[_CustomFieldsCase, ...]] = (
|
||||
((), None, None),
|
||||
(({},), None, None),
|
||||
(
|
||||
({"custom_tags": ["deprecated", "ptu"], "custom_metadata": {"retirement_date": "2027-03-01"}},),
|
||||
("deprecated", "ptu"),
|
||||
{"retirement_date": "2027-03-01"},
|
||||
),
|
||||
(
|
||||
({}, {"custom_tags": ["ptu"], "custom_metadata": {"provisioned_ptu": 100, "gsu": False}}),
|
||||
("ptu",),
|
||||
{"provisioned_ptu": 100, "gsu": False},
|
||||
),
|
||||
(
|
||||
(
|
||||
{"custom_tags": ["ptu", "eu"], "custom_metadata": {"provisioned_ptu": 100, "region": "eu"}},
|
||||
{"custom_tags": ["eu", "deprecated"], "custom_metadata": {"provisioned_ptu": 50, "max_tpm": 1.5}},
|
||||
),
|
||||
("ptu", "eu", "deprecated"),
|
||||
{"provisioned_ptu": 100, "region": "eu", "max_tpm": 1.5},
|
||||
),
|
||||
(
|
||||
({"custom_tags": ["ok", 7, None, {"x": 1}], "custom_metadata": {"owner": "ml", "nested": {"a": 1}}},),
|
||||
("ok",),
|
||||
{"owner": "ml"},
|
||||
),
|
||||
(({"custom_tags": "ptu", "custom_metadata": ["not", "a", "map"]},), None, None),
|
||||
(
|
||||
(
|
||||
{
|
||||
"custom_metadata": {
|
||||
2027: "retirement",
|
||||
"nan": float("nan"),
|
||||
"inf": float("inf"),
|
||||
"neg_inf": float("-inf"),
|
||||
"max_tpm": 1.5,
|
||||
}
|
||||
},
|
||||
),
|
||||
None,
|
||||
{"max_tpm": 1.5},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("model_infos", "expected_tags", "expected_metadata"), _CUSTOM_FIELDS_CASES)
|
||||
def test_get_model_group_info_merges_custom_tags_and_metadata_across_deployments(
|
||||
model_infos: tuple[dict[object, object], ...],
|
||||
expected_tags: tuple[str, ...] | None,
|
||||
expected_metadata: dict[str, str | int | float | bool] | None,
|
||||
) -> None:
|
||||
from litellm.proxy.proxy_server import get_model_group_info
|
||||
|
||||
router: Final = _custom_metadata_router({"description": "first"}, *model_infos)
|
||||
|
||||
(group,) = get_model_group_info(llm_router=router, all_models_str=["gpt-4"], model_group="gpt-4")
|
||||
|
||||
assert (group.custom_tags, group.custom_metadata) == (expected_tags, expected_metadata)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"alias_target",
|
||||
[pytest.param("gpt-4", id="string"), pytest.param({"model": "gpt-4", "hidden": False}, id="item")],
|
||||
)
|
||||
def test_get_model_group_info_serves_the_target_groups_custom_fields_on_an_alias_row(
|
||||
alias_target: str | dict[str, object],
|
||||
) -> None:
|
||||
from litellm.proxy.proxy_server import get_model_group_info
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4",
|
||||
"litellm_params": {"model": "gpt-4", "api_key": "fake"},
|
||||
"model_info": {"custom_tags": ["deprecated"], "custom_metadata": {"retirement_date": "2027-03-01"}},
|
||||
}
|
||||
],
|
||||
model_group_alias={"gpt-alias": alias_target},
|
||||
)
|
||||
|
||||
groups: Final = get_model_group_info(llm_router=router, all_models_str=["gpt-4", "gpt-alias"], model_group=None)
|
||||
|
||||
assert {group.model_group: (group.custom_tags, group.custom_metadata) for group in groups} == {
|
||||
model_group: (("deprecated",), {"retirement_date": "2027-03-01"}) for model_group in ("gpt-4", "gpt-alias")
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_team_data():
|
||||
return [
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
import { render, screen } from "@testing-library/react";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { ModelCustomMetadata } from "./ModelCustomMetadata";
|
||||
|
||||
describe("ModelCustomMetadata", () => {
|
||||
it("renders each tag and every metadata entry with its value", () => {
|
||||
render(
|
||||
<ModelCustomMetadata
|
||||
customTags={["deprecated", "ptu"]}
|
||||
customMetadata={{ retirement_date: "2027-03-01", provisioned_ptu: 100, gsu: false }}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByText("Tags:")).toBeInTheDocument();
|
||||
expect(screen.getByText("deprecated")).toBeInTheDocument();
|
||||
expect(screen.getByText("ptu")).toBeInTheDocument();
|
||||
const terms = screen.getAllByRole("term").map((term) => term.textContent);
|
||||
const definitions = screen.getAllByRole("definition").map((definition) => definition.textContent);
|
||||
expect(terms.map((term, index) => [term, definitions[index]])).toEqual([
|
||||
["retirement_date", "2027-03-01"],
|
||||
["provisioned_ptu", "100"],
|
||||
["gsu", "false"],
|
||||
]);
|
||||
});
|
||||
|
||||
it("renders nothing when the model has no tags or metadata", () => {
|
||||
const { container } = render(<ModelCustomMetadata customTags={null} customMetadata={{}} />);
|
||||
expect(container).toBeEmptyDOMElement();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
import { Badge } from "@/components/ui/badge";
|
||||
|
||||
export type ModelCustomMetadataValue = string | number | boolean;
|
||||
|
||||
interface ModelCustomMetadataProps {
|
||||
customTags?: string[] | null;
|
||||
customMetadata?: Record<string, ModelCustomMetadataValue> | null;
|
||||
}
|
||||
|
||||
export function ModelCustomTags({ tags, className = "" }: { tags: string[]; className?: string }) {
|
||||
return (
|
||||
<div className={`flex flex-wrap gap-1 ${className}`}>
|
||||
{tags.map((tag) => (
|
||||
<Badge key={tag} variant="outline">
|
||||
{tag}
|
||||
</Badge>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function ModelCustomMetadata({ customTags, customMetadata }: ModelCustomMetadataProps) {
|
||||
const tags = customTags ?? [];
|
||||
const entries = Object.entries(customMetadata ?? {});
|
||||
return (
|
||||
<>
|
||||
{tags.length > 0 && (
|
||||
<div className="col-span-2">
|
||||
<p className="font-medium">Tags:</p>
|
||||
<ModelCustomTags tags={tags} className="mt-1" />
|
||||
</div>
|
||||
)}
|
||||
{entries.length > 0 && (
|
||||
<div className="col-span-2">
|
||||
<p className="font-medium">Custom Metadata:</p>
|
||||
<dl className="mt-1 grid grid-cols-[max-content_1fr] gap-x-4 gap-y-1">
|
||||
{entries.map(([key, value]) => (
|
||||
<div key={key} className="contents">
|
||||
<dt className="text-muted-foreground">{key}</dt>
|
||||
<dd>{String(value)}</dd>
|
||||
</div>
|
||||
))}
|
||||
</dl>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
|
@ -54,6 +54,7 @@ import {
|
|||
isDecisionMode,
|
||||
} from "@/lib/decisionModels";
|
||||
import DecisionModelGarden from "@/components/AIHub/DecisionModelGarden";
|
||||
import { ModelCustomMetadata } from "./ModelCustomMetadata";
|
||||
|
||||
interface ModelHubTableProps {
|
||||
accessToken: string | null;
|
||||
|
|
@ -719,6 +720,10 @@ const ModelHubTable: React.FC<ModelHubTableProps> = ({
|
|||
<p className="mt-1">{selectedModel.description}</p>
|
||||
</div>
|
||||
)}
|
||||
<ModelCustomMetadata
|
||||
customTags={selectedModel.custom_tags}
|
||||
customMetadata={selectedModel.custom_metadata}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
|
|||
|
|
@ -61,6 +61,17 @@ describe("getModelHubTableColumns", () => {
|
|||
expect(screen.getByRole("cell", { name: "gpt-4o" })).not.toHaveClass("text-right");
|
||||
});
|
||||
|
||||
it("shows each custom tag as a badge and a dash when the model has none", () => {
|
||||
renderTable([
|
||||
{ ...mockModel, custom_tags: ["deprecated", "ptu"] },
|
||||
{ ...mockModel, model_group: "untagged", custom_tags: null },
|
||||
]);
|
||||
expect(screen.getByRole("columnheader", { name: "Tags" })).toBeInTheDocument();
|
||||
const [taggedRow, untaggedRow] = screen.getAllByRole("row").slice(1);
|
||||
expect(taggedRow).toHaveTextContent("deprecatedptu");
|
||||
expect(untaggedRow).not.toHaveTextContent("deprecated");
|
||||
});
|
||||
|
||||
it("shows capability badges only for supported features", () => {
|
||||
renderTable([mockModel]);
|
||||
expect(screen.getByText("Vision")).toBeInTheDocument();
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@ import { cn } from "@/lib/cva.config";
|
|||
import { displayMode } from "@/lib/decisionModels";
|
||||
import { copyToClipboard } from "@/utils/dataUtils";
|
||||
|
||||
import { ModelCustomTags, type ModelCustomMetadataValue } from "./ModelCustomMetadata";
|
||||
|
||||
export interface ModelHubData {
|
||||
model_group: string;
|
||||
providers: string[];
|
||||
|
|
@ -33,6 +35,8 @@ export interface ModelHubData {
|
|||
supported_openai_params?: string[];
|
||||
is_public_model_group: boolean;
|
||||
description?: string;
|
||||
custom_tags?: string[] | null;
|
||||
custom_metadata?: Record<string, ModelCustomMetadataValue> | null;
|
||||
[key: string]: any;
|
||||
}
|
||||
|
||||
|
|
@ -210,6 +214,20 @@ export const getModelHubTableColumns = ({ onModelClick }: ModelHubTableColumnsDe
|
|||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "custom_tags",
|
||||
meta: { title: "Tags", skeleton: "chips" },
|
||||
header: "Tags",
|
||||
size: 160,
|
||||
enableSorting: false,
|
||||
cell: ({ row }) => {
|
||||
const tags = row.original.custom_tags ?? [];
|
||||
if (tags.length === 0) {
|
||||
return <span className="text-xs text-muted-foreground">-</span>;
|
||||
}
|
||||
return <ModelCustomTags tags={tags} />;
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "is_public_model_group",
|
||||
accessorKey: "is_public_model_group",
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import { Badge } from "@/components/ui/badge";
|
|||
import { getProviderLogoAndName } from "@/components/provider_info_helpers";
|
||||
import { PUBLIC_MODEL_HUB_SORTABLE_FIELDS } from "@/components/publicModelHub/publicModelHubFilters";
|
||||
import { displayMode } from "@/lib/decisionModels";
|
||||
import type { ModelCustomMetadataValue } from "@/components/AIHub/ModelCustomMetadata";
|
||||
|
||||
export interface ModelGroupInfo {
|
||||
model_group: string;
|
||||
|
|
@ -27,6 +28,8 @@ export interface ModelGroupInfo {
|
|||
health_response_time?: number;
|
||||
health_checked_at?: string;
|
||||
description?: string;
|
||||
custom_tags?: string[] | null;
|
||||
custom_metadata?: Record<string, ModelCustomMetadataValue> | null;
|
||||
[key: string]: any;
|
||||
}
|
||||
|
||||
|
|
@ -260,6 +263,13 @@ export const getPublicModelHubColumns = ({ onModelClick }: PublicModelHubColumns
|
|||
return <OverflowChips items={features} />;
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "custom_tags",
|
||||
meta: { title: "Tags", skeleton: "chips" },
|
||||
header: "Tags",
|
||||
size: 140,
|
||||
cell: ({ row }) => <OverflowChips items={row.original.custom_tags ?? []} />,
|
||||
},
|
||||
{
|
||||
id: "health_status",
|
||||
accessorKey: "health_status",
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ import { getProviderLogoAndName } from "./provider_info_helpers";
|
|||
import { displayMode, isDecisionMode } from "@/lib/decisionModels";
|
||||
import { modelUsageExample } from "@/components/AIHub/modelUsageExample";
|
||||
import { filterBySearchTerm, rankBySearchRelevance } from "@/utils/searchUtils";
|
||||
import { ModelCustomMetadata } from "@/components/AIHub/ModelCustomMetadata";
|
||||
|
||||
interface PublicModelHubProps {
|
||||
accessToken?: string | null;
|
||||
|
|
@ -749,6 +750,10 @@ const PublicModelHub: React.FC<PublicModelHubProps> = ({ accessToken, isEmbedded
|
|||
<p className="mt-1">{selectedModel.description}</p>
|
||||
</div>
|
||||
)}
|
||||
<ModelCustomMetadata
|
||||
customTags={selectedModel.custom_tags}
|
||||
customMetadata={selectedModel.custom_metadata}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Wildcard Routing Note */}
|
||||
|
|
|
|||
6
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
6
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -38374,6 +38374,12 @@ export interface components {
|
|||
ModelGroupInfoProxy: {
|
||||
/** Configurable Clientside Auth Params */
|
||||
configurable_clientside_auth_params?: (string | components["schemas"]["ConfigurableClientsideParamsCustomAuth-Output"])[] | null;
|
||||
/** Custom Metadata */
|
||||
custom_metadata?: {
|
||||
[key: string]: string | number | boolean;
|
||||
} | null;
|
||||
/** Custom Tags */
|
||||
custom_tags?: string[] | null;
|
||||
/** Description */
|
||||
description?: string | null;
|
||||
/** Health Checked At */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue