diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py index 14d4a11788b..50fdb6cb5a5 100644 --- a/litellm/proxy/client/cli/commands/agents.py +++ b/litellm/proxy/client/cli/commands/agents.py @@ -517,6 +517,8 @@ class _OpenCodeReasoningOption(LiteLLMBaseModel): class _OpenCodeModel(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + name: str limit: _OpenCodeLimit | None = None reasoning: Literal[True] | None = None diff --git a/litellm/proxy/client/cli/commands/pi.py b/litellm/proxy/client/cli/commands/pi.py index f02395b98ef..19c4a94d380 100644 --- a/litellm/proxy/client/cli/commands/pi.py +++ b/litellm/proxy/client/cli/commands/pi.py @@ -79,6 +79,8 @@ class _ModelList(LiteLLMBaseModel): class _ModelGroup(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + model_group: str max_input_tokens: float | None = None max_output_tokens: float | None = None diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index 0d005efdf7d..3da9ca19e42 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -49,6 +49,7 @@ _BODY_SUFFIX: Final = "]}" _FIRST_CODEX_WITH_CUSTOM_EFFORTS: Final = Version("0.138.0") _EFFORTS_EVERY_CODEX_PARSES: Final = frozenset({"none", "minimal", "low", "medium", "high", "xhigh"}) _PREFERRED_DEFAULT_EFFORT: Final = "medium" +_CODEX_UNSET_EFFORT: Final = "none" class CodexServiceTier(LiteLLMBaseModel): @@ -97,6 +98,8 @@ class CodexFallbackModel(LiteLLMBaseModel): same as `codex -m` did. """ + model_config = ConfigDict(frozen=True) + slug: str display_name: str description: NoneType = None @@ -277,7 +280,8 @@ def codex_reasoning_levels( Codex before 0.138.0 fails the whole catalog on an effort it has no name for, so for it, and for a version that does not parse, only the efforts every release knows are offered. The default is - set whenever a level is offered, because Codex reads a missing one as `none`: medium, else the + set whenever a level is offered, because Codex reads a missing one as `none`: `none` itself when the + group accepts it, so a user who picks no level sends what Codex sent before, else medium, else the nearest stronger declared level, else the first one.""" parses_any: Final = _parses_custom_efforts(client_version) offered: Final = tuple(effort for effort in efforts if parses_any or effort in _EFFORTS_EVERY_CODEX_PARSES) @@ -288,6 +292,8 @@ def codex_reasoning_levels( CodexReasoningLevel(effort=effort, description=f"Sends reasoning_effort={effort} upstream") for effort in offered ) + if _CODEX_UNSET_EFFORT in offered: + return levels, _CODEX_UNSET_EFFORT return levels, nearest if nearest in offered else offered[0] diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3d6a9bd3c40..e83403f55d7 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12445,7 +12445,8 @@ async def model_list( admin_listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above if client_version is not None: return Response( - content=codex_model_list_body( + content=await asyncio.to_thread( + codex_model_list_body, admin_listing, admin_entries, llm_router, @@ -12520,7 +12521,8 @@ async def model_list( listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above if client_version is not None: return Response( - content=codex_model_list_body( + content=await asyncio.to_thread( + codex_model_list_body, listing, entries, llm_router, diff --git a/litellm/router.py b/litellm/router.py index 3e6d82d9b07..d0390255b6e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -12589,7 +12589,14 @@ class Router: configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None reasoning_efforts_initialized = False reasoning_efforts_unknown = False - model_list: Final = self.get_model_list_of_routed_group(model_group) + serves_team_deployments: Final = team_id is not None and bool( + self._team_deployments_named(team_id, model_group) + ) + model_list: Final = ( + self.get_all_deployments(model_name=model_group, team_id=team_id) + if serves_team_deployments + else self.get_model_list_of_routed_group(model_group) + ) model_group_description: Final[str | None] = next( (d for d in (_configured_model_info(m).get("description") for m in model_list) if isinstance(d, str)), None, @@ -12597,7 +12604,8 @@ class Router: for model in model_list: is_match = False if ( - "model_name" in model + serves_team_deployments + or "model_name" in model and model["model_name"] == model_group or "model_name" in model and self.pattern_router.route(model_group) is not None diff --git a/litellm/types/router.py b/litellm/types/router.py index ba9b82e2a57..a793d23217a 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -4,8 +4,10 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc import datetime import enum +import functools from collections.abc import Container, Mapping, Sequence from dataclasses import dataclass, field +from types import MappingProxyType from typing import ( TYPE_CHECKING, Annotated, @@ -957,9 +959,10 @@ class AlertingConfig(LiteLLMBaseModel): alerting_threshold: float | None = 300 +@functools.cache def _resolved_annotations(model_class: type[object]) -> Mapping[str, object]: """Resolve a class's annotations, keeping each resolved annotation opaque.""" - return get_type_hints(model_class) + return MappingProxyType(get_type_hints(model_class)) class ModelGroupInfo(LiteLLMBaseModel): diff --git a/tests/unit/proxy/common_utils/test_codex_model_catalog.py b/tests/unit/proxy/common_utils/test_codex_model_catalog.py index b23b2df15bd..c6376990ab9 100644 --- a/tests/unit/proxy/common_utils/test_codex_model_catalog.py +++ b/tests/unit/proxy/common_utils/test_codex_model_catalog.py @@ -144,8 +144,10 @@ _ABSENT = object() ("none", "low", "medium", "high", "xhigh", "max"), "0.165.0", ["none", "low", "medium", "high", "xhigh", "max"], - "medium", + "none", ), + (("low", "medium", "high"), "0.165.0", ["low", "medium", "high"], "medium"), + (("none", "low", "max"), "0.137.0", ["none", "low"], "none"), (("none",), "0.165.0", ["none"], "none"), (("low", "turbo"), "0.165.0", ["low", "turbo"], "low"), (("low", "high", "max"), "0.138.0", ["low", "high", "max"], "high"), diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 9993361d930..71ea394a4fd 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -221,39 +221,57 @@ def test_deleted_model_params_stay_absent_after_reload_and_another_save( from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo, updateDeployment, updateLiteLLMParams configured: Final = { - "model": "openai/test-model", "api_key": "synthetic-runtime-secret", - "use_litellm_proxy": True, "use_in_pass_through": True, "use_chat_completions_api": False, - "litellm_image_enabled": True, "timeout": 30, "input_cost_per_token": 0.003, - "team_id": "param-only-team", "temperature": 0, "metadata": {}, + "model": "openai/test-model", + "api_key": "synthetic-runtime-secret", + "use_litellm_proxy": True, + "use_in_pass_through": True, + "use_chat_completions_api": False, + "litellm_image_enabled": True, + "timeout": 30, + "input_cost_per_token": 0.003, + "team_id": "param-only-team", + "temperature": 0, + "metadata": {}, } original: Final = Deployment( - model_name="editor-model", litellm_params=LiteLLM_Params.model_validate(configured), + model_name="editor-model", + litellm_params=LiteLLM_Params.model_validate(configured), model_info=ModelInfo(id="editor-model", team_id="owner-team", access_groups=["retained-group"]), ) removals: Final = ("litellm_image_enabled", "use_litellm_proxy", "input_cost_per_token", "team_id") deleted: Final = update_db_model( db_model=original, updated_patch=PatchModelRequest( - litellm_params=updateLiteLLMParams(max_retries=0), litellm_params_to_remove=removals, + litellm_params=updateLiteLLMParams(max_retries=0), + litellm_params_to_remove=removals, model_info=ModelInfo(id="editor-model", input_cost_per_token=0.003), ), ) saved_params: Final = json.loads(deleted["litellm_params"]) saved_info: Final = json.loads(deleted["model_info"]) - assert saved_params == {**{key: value for key, value in configured.items() if key not in removals}, "max_retries": 0} + assert saved_params == { + **{key: value for key, value in configured.items() if key not in removals}, + "max_retries": 0, + } assert "input_cost_per_token" not in saved_info assert saved_info["team_id"] == "owner-team" assert saved_info["access_groups"] == ["retained-group"] reloaded: Final = Deployment.model_validate( {"model_name": original.model_name, "litellm_params": saved_params, "model_info": saved_info} ) - router: Final = litellm.Router(model_list=[{ - "model_name": "editor-model", "litellm_params": saved_params, - "model_info": {"id": "editor-model", "db_model": True}, - }]) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "editor-model", + "litellm_params": saved_params, + "model_info": {"id": "editor-model", "db_model": True}, + } + ] + ) stored_params: Final = {**saved_params, "api_key": "synthetic-stored-secret"} mock_prisma.db.litellm_proxymodeltable.find_unique.return_value = LiteLLM_ProxyModelTable( - model_id="editor-model", model_name="editor-model", + model_id="editor-model", + model_name="editor-model", litellm_params=json.dumps(stored_params) if encoded else stored_params, model_info={"id": "editor-model"}, ) @@ -989,6 +1007,50 @@ def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deplo assert ("/v1/batch" in (groups["gpt-mixed"]["supported_endpoints"] or ())) is lists_batch +@pytest.mark.parametrize(("team_id", "efforts"), [(_BYOK_TEAM_ID, ["low", "high", "max"]), ("team-other", None)]) +def test_model_group_info_describes_a_team_model_under_the_public_name_its_team_calls( + client, auth_as, monkeypatch, team_id: str, efforts: list[str] | None +): + """A team's own model is listed under its public name, so its row must describe the team's deployment + behind that name instead of a placeholder, and only for the team that owns it.""" + from litellm.proxy._types import LitellmUserRoles + from litellm.proxy.agent_endpoints import model_list_helpers + + router: Final = litellm.Router( + model_list=[ + { + "model_name": _BYOK_INTERNAL_NAME, + "litellm_params": {"model": "hosted_vllm/qa-local-coder", "api_base": "https://vllm.example/v1"}, + "model_info": { + "team_id": _BYOK_TEAM_ID, + "team_public_model_name": _BYOK_PUBLIC_NAME, + "supports_reasoning": True, + "reasoning_effort_levels": ["low", "high", "max"], + }, + } + ] + ) + + async def append_agents_to_model_group(*, model_groups, user_api_key_dict): + return model_groups + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr( + model_list_helpers, "append_agents_to_model_group", AsyncMock(side_effect=append_agents_to_model_group) + ) + + with auth_as(LitellmUserRoles.INTERNAL_USER, user_id=None, team_id=team_id, team_models=[_BYOK_PUBLIC_NAME]): + response: Final = client.get("/model_group/info") + + assert response.status_code == 200 + groups: Final = {group["model_group"]: group for group in response.json()["data"]} + assert groups[_BYOK_PUBLIC_NAME]["supported_reasoning_efforts"] == efforts + + # --------------------------------------------------------------------------- # GET /v2/model/info?exclude_auto_routers # --------------------------------------------------------------------------- diff --git a/tests/unit/proxy/proxy_server/test_routes_models.py b/tests/unit/proxy/proxy_server/test_routes_models.py index 33c95faaf2d..249fcbf16bd 100644 --- a/tests/unit/proxy/proxy_server/test_routes_models.py +++ b/tests/unit/proxy/proxy_server/test_routes_models.py @@ -17,14 +17,12 @@ import litellm from litellm.proxy import proxy_server from litellm.proxy import utils as proxy_utils from litellm.proxy.utils import create_model_info_response -from litellm.types.router import DeploymentModelListingInfo, ModelGroupInfo +from litellm.types.router import DeploymentModelListingInfo from .conftest import normalize # type: ignore[import-not-found] -def _stub_model_info_response( - model_id: str = "gpt-4", provider: str = "openai" -) -> dict: +def _stub_model_info_response(model_id: str = "gpt-4", provider: str = "openai") -> dict: return { "id": model_id, "object": "model", @@ -66,9 +64,7 @@ def patched_models(monkeypatch): def _fake_create_model_info_response(model_id, provider="openai", **kwargs): return _stub_model_info_response(model_id=model_id, provider=provider) - monkeypatch.setattr( - proxy_utils, "create_model_info_response", _fake_create_model_info_response - ) + monkeypatch.setattr(proxy_utils, "create_model_info_response", _fake_create_model_info_response) monkeypatch.setattr(proxy_utils, "validate_model_access", lambda **kwargs: None) @@ -107,9 +103,7 @@ def test_get_models_happy_path(client, auth_as, patched_models, path): @pytest.mark.parametrize("path", ["/v1/models", "/models"]) -def test_get_models_anthropic_format_when_header_present( - client, auth_as, patched_models, path -): +def test_get_models_anthropic_format_when_header_present(client, auth_as, patched_models, path): """Pins: ``GET /v1/models`` returns the Anthropic-native models shape when the caller sends an ``anthropic-version`` header (Claude Code gateway discovery), while the default OpenAI shape is unchanged without it.""" @@ -129,9 +123,7 @@ def test_get_models_anthropic_format_when_header_present( @pytest.mark.parametrize("path", ["/v1/models", "/models"]) -def test_anthropic_format_exposes_token_limits( - client, auth_as, patched_models, monkeypatch, path -): +def test_anthropic_format_exposes_token_limits(client, auth_as, patched_models, monkeypatch, path): """Claude Code sizes requests off the listing, so the Anthropic-native entries carry the same token limits the OpenAI listing resolves, with the output budget named max_tokens as the Messages API names it.""" @@ -146,9 +138,7 @@ def test_anthropic_format_exposes_token_limits( "max_output_tokens": 64000, } - monkeypatch.setattr( - proxy_utils, "create_model_info_response", _create_model_info_response - ) + monkeypatch.setattr(proxy_utils, "create_model_info_response", _create_model_info_response) with auth_as(): response = client.get(path, headers={"anthropic-version": "2023-06-01"}) @@ -223,9 +213,7 @@ def test_anthropic_format_uses_configured_display_name(client, auth_as, patched_ @pytest.mark.parametrize("params", [{}, {"scope": "expand"}]) -def test_anthropic_display_name_resolved_via_internal_team_key( - client, auth_as, patched_models, monkeypatch, params -): +def test_anthropic_display_name_resolved_via_internal_team_key(client, auth_as, patched_models, monkeypatch, params): """For a team-scoped row the configured display name must be looked up by the internal routing key while the entry itself is keyed by the public name, so the clean name lands on the id the client actually sees.""" @@ -258,14 +246,10 @@ def test_anthropic_display_name_resolved_via_internal_team_key( "get_available_models_for_user", _fake_get_available_models_for_user, ) - monkeypatch.setattr( - model_checks, "get_complete_model_list", lambda **kwargs: [internal_name] - ) + monkeypatch.setattr(model_checks, "get_complete_model_list", lambda **kwargs: [internal_name]) with auth_as(): - response = client.get( - "/v1/models", params=params, headers={"anthropic-version": "2023-06-01"} - ) + response = client.get("/v1/models", params=params, headers={"anthropic-version": "2023-06-01"}) assert response.status_code == 200 (entry,) = response.json()["data"] @@ -306,9 +290,7 @@ def test_get_model_by_id_not_found(client, auth_as, patched_models, path): @pytest.mark.parametrize("params", [{}, {"scope": "expand"}]) -def test_anthropic_format_returns_public_team_model_name( - client, auth_as, patched_models, monkeypatch, params -): +def test_anthropic_format_returns_public_team_model_name(client, auth_as, patched_models, monkeypatch, params): """Regression: the Anthropic-native listing must go through the same team name translation as the OpenAI listing, so a caller never sees the internal ``model_name_{team_id}_{uuid}`` routing key.""" @@ -338,14 +320,10 @@ def test_anthropic_format_returns_public_team_model_name( "get_available_models_for_user", _fake_get_available_models_for_user, ) - monkeypatch.setattr( - model_checks, "get_complete_model_list", lambda **kwargs: [internal_name] - ) + monkeypatch.setattr(model_checks, "get_complete_model_list", lambda **kwargs: [internal_name]) with auth_as(): - response = client.get( - "/v1/models", params=params, headers={"anthropic-version": "2023-06-01"} - ) + response = client.get("/v1/models", params=params, headers={"anthropic-version": "2023-06-01"}) assert response.status_code == 200 assert [m["id"] for m in response.json()["data"]] == ["gpt-4-team"] @@ -376,7 +354,9 @@ def test_anthropic_format_lists_claude_code_view_ids_for_claude_code( patched_models.model_group_alias = {} patched_models.has_model_id.return_value = False - patched_models.get_candidate_model_ids_for_route.side_effect = lambda name, team_id=None: frozenset({name}) if name in ("gpt-4", "claude-sonnet") else frozenset() + patched_models.get_candidate_model_ids_for_route.side_effect = lambda name, team_id=None: ( + frozenset({name}) if name in ("gpt-4", "claude-sonnet") else frozenset() + ) monkeypatch.setattr(proxy_utils, "create_model_info_response", _create_model_info_response) with auth_as(): @@ -396,7 +376,9 @@ def test_anthropic_format_lists_claude_code_view_ids_for_claude_code( def test_anthropic_format_keeps_served_ids_for_other_anthropic_clients(client, auth_as, patched_models, path): """An Anthropic SDK asking for the vendor shape gets the served ids: the view is Claude Code's alone.""" with auth_as(): - response = client.get(path, headers={"anthropic-version": "2023-06-01", "user-agent": "anthropic-sdk-python/0.40"}) + response = client.get( + path, headers={"anthropic-version": "2023-06-01", "user-agent": "anthropic-sdk-python/0.40"} + ) assert response.status_code == 200 assert [m["id"] for m in response.json()["data"]] == ["gpt-4", "claude-sonnet"] @@ -462,14 +444,21 @@ def test_codex_format_carries_configured_service_tiers(client, auth_as, patched_ @pytest.mark.parametrize("params", [{}, {"scope": "expand"}]) -def test_codex_reasoning_levels_are_the_ones_the_requesting_codex_parses(client, auth_as, patched_models, params): +def test_codex_reasoning_levels_are_the_ones_the_requesting_codex_parses(client, auth_as, monkeypatch, params): """Both listing paths hand the route's ``client_version`` to the catalog: Codex before 0.138.0 cannot parse max, so it is offered there only to a newer release.""" - patched_models.get_model_group_info = MagicMock( - return_value=ModelGroupInfo( - model_group="gpt-4", providers=["openai"], supported_reasoning_efforts=("low", "max") - ) + declared = {"supports_reasoning": True, "reasoning_effort_levels": ["low", "max"]} + vllm = {"model": "hosted_vllm/qa-local-coder", "api_base": "https://vllm.example/v1"} + router = litellm.Router( + model_list=[ + {"model_name": name, "litellm_params": vllm, "model_info": declared} for name in ("gpt-4", "claude-sonnet") + ] ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "prisma_client", None) with auth_as(): current = client.get("/v1/models", params={**params, "client_version": "0.165.0"})