scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-12 08:42:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 207

fix(lite): describe team models by their public name and keep Codex's none default

Browse source 
/model_group/info now looks up a team's own deployments when the caller's
team serves the requested name, so a team model reports its reasoning levels.
Codex fallback entries default to none when the group lists it, and the proxy
builds the Codex catalog in a worker thread
This commit is contained in:
mateo-berri 2026-10-28 18:16:44 -07:00
parent 00b2f11196
commit 0d118386a9
9 changed files with 136 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/proxy/client/cli/commands/agents.py
View file

@ -517,6 +517,8 @@ class _OpenCodeReasoningOption(LiteLLMBaseModel):
class _OpenCodeModel(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
name: str
limit: _OpenCodeLimit | None = None
reasoning: Literal[True] | None = None

2
litellm/proxy/client/cli/commands/pi.py
View file

@ -79,6 +79,8 @@ class _ModelList(LiteLLMBaseModel):
class _ModelGroup(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
model_group: str
max_input_tokens: float | None = None
max_output_tokens: float | None = None

8
litellm/proxy/common_utils/codex_model_catalog.py
View file

@ -49,6 +49,7 @@ _BODY_SUFFIX: Final = "]}"
_FIRST_CODEX_WITH_CUSTOM_EFFORTS: Final = Version("0.138.0")
_EFFORTS_EVERY_CODEX_PARSES: Final = frozenset({"none", "minimal", "low", "medium", "high", "xhigh"})
_PREFERRED_DEFAULT_EFFORT: Final = "medium"
_CODEX_UNSET_EFFORT: Final = "none"
class CodexServiceTier(LiteLLMBaseModel):
@ -97,6 +98,8 @@ class CodexFallbackModel(LiteLLMBaseModel):
same as `codex -m` did.
"""
model_config = ConfigDict(frozen=True)
slug: str
display_name: str
description: NoneType = None
@ -277,7 +280,8 @@ def codex_reasoning_levels(
Codex before 0.138.0 fails the whole catalog on an effort it has no name for, so for it, and for
a version that does not parse, only the efforts every release knows are offered. The default is
set whenever a level is offered, because Codex reads a missing one as `none`: medium, else the
set whenever a level is offered, because Codex reads a missing one as `none`: `none` itself when the
group accepts it, so a user who picks no level sends what Codex sent before, else medium, else the
nearest stronger declared level, else the first one."""
parses_any: Final = _parses_custom_efforts(client_version)
offered: Final = tuple(effort for effort in efforts if parses_any or effort in _EFFORTS_EVERY_CODEX_PARSES)
@ -288,6 +292,8 @@ def codex_reasoning_levels(
CodexReasoningLevel(effort=effort, description=f"Sends reasoning_effort={effort} upstream")
for effort in offered
)
if _CODEX_UNSET_EFFORT in offered:
return levels, _CODEX_UNSET_EFFORT
return levels, nearest if nearest in offered else offered[0]

6
litellm/proxy/proxy_server.py
View file

@ -12445,7 +12445,8 @@ async def model_list(
admin_listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above
if client_version is not None:
return Response(
content=codex_model_list_body(
content=await asyncio.to_thread(
codex_model_list_body,
admin_listing,
admin_entries,
llm_router,
@ -12520,7 +12521,8 @@ async def model_list(
listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above
if client_version is not None:
return Response(
content=codex_model_list_body(
content=await asyncio.to_thread(
codex_model_list_body,
listing,
entries,
llm_router,

12
litellm/router.py
View file

@ -12589,7 +12589,14 @@ class Router:
configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None
reasoning_efforts_initialized = False
reasoning_efforts_unknown = False
model_list: Final = self.get_model_list_of_routed_group(model_group)
serves_team_deployments: Final = team_id is not None and bool(
self._team_deployments_named(team_id, model_group)
)
model_list: Final = (
self.get_all_deployments(model_name=model_group, team_id=team_id)
if serves_team_deployments
else self.get_model_list_of_routed_group(model_group)
)
model_group_description: Final[str | None] = next(
(d for d in (_configured_model_info(m).get("description") for m in model_list) if isinstance(d, str)),
None,
@ -12597,7 +12604,8 @@ class Router:
for model in model_list:
is_match = False
if (
"model_name" in model
serves_team_deployments
or "model_name" in model
and model["model_name"] == model_group
or "model_name" in model
and self.pattern_router.route(model_group) is not None

5
litellm/types/router.py
View file

@ -4,8 +4,10 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc
import datetime
import enum
import functools
from collections.abc import Container, Mapping, Sequence
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
Annotated,
@ -957,9 +959,10 @@ class AlertingConfig(LiteLLMBaseModel):
alerting_threshold: float | None = 300
@functools.cache
def _resolved_annotations(model_class: type[object]) -> Mapping[str, object]:
"""Resolve a class's annotations, keeping each resolved annotation opaque."""
return get_type_hints(model_class)
return MappingProxyType(get_type_hints(model_class))
class ModelGroupInfo(LiteLLMBaseModel):

4
tests/unit/proxy/common_utils/test_codex_model_catalog.py
View file

@ -144,8 +144,10 @@ _ABSENT = object()
("none", "low", "medium", "high", "xhigh", "max"),
"0.165.0",
["none", "low", "medium", "high", "xhigh", "max"],
"medium",
"none",
),
(("low", "medium", "high"), "0.165.0", ["low", "medium", "high"], "medium"),
(("none", "low", "max"), "0.137.0", ["none", "low"], "none"),
(("none",), "0.165.0", ["none"], "none"),
(("low", "turbo"), "0.165.0", ["low", "turbo"], "low"),
(("low", "high", "max"), "0.138.0", ["low", "high", "max"], "high"),

86
tests/unit/proxy/proxy_server/test_routes_model_info.py
View file

@ -221,39 +221,57 @@ def test_deleted_model_params_stay_absent_after_reload_and_another_save(
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo, updateDeployment, updateLiteLLMParams
configured: Final = {
"model": "openai/test-model", "api_key": "synthetic-runtime-secret",
"use_litellm_proxy": True, "use_in_pass_through": True, "use_chat_completions_api": False,
"litellm_image_enabled": True, "timeout": 30, "input_cost_per_token": 0.003,
"team_id": "param-only-team", "temperature": 0, "metadata": {},
"model": "openai/test-model",
"api_key": "synthetic-runtime-secret",
"use_litellm_proxy": True,
"use_in_pass_through": True,
"use_chat_completions_api": False,
"litellm_image_enabled": True,
"timeout": 30,
"input_cost_per_token": 0.003,
"team_id": "param-only-team",
"temperature": 0,
"metadata": {},
}
original: Final = Deployment(
model_name="editor-model", litellm_params=LiteLLM_Params.model_validate(configured),
model_name="editor-model",
litellm_params=LiteLLM_Params.model_validate(configured),
model_info=ModelInfo(id="editor-model", team_id="owner-team", access_groups=["retained-group"]),
)
removals: Final = ("litellm_image_enabled", "use_litellm_proxy", "input_cost_per_token", "team_id")
deleted: Final = update_db_model(
db_model=original,
updated_patch=PatchModelRequest(
litellm_params=updateLiteLLMParams(max_retries=0), litellm_params_to_remove=removals,
litellm_params=updateLiteLLMParams(max_retries=0),
litellm_params_to_remove=removals,
model_info=ModelInfo(id="editor-model", input_cost_per_token=0.003),
),
)
saved_params: Final = json.loads(deleted["litellm_params"])
saved_info: Final = json.loads(deleted["model_info"])
assert saved_params == {**{key: value for key, value in configured.items() if key not in removals}, "max_retries": 0}
assert saved_params == {
**{key: value for key, value in configured.items() if key not in removals},
"max_retries": 0,
}
assert "input_cost_per_token" not in saved_info
assert saved_info["team_id"] == "owner-team"
assert saved_info["access_groups"] == ["retained-group"]
reloaded: Final = Deployment.model_validate(
{"model_name": original.model_name, "litellm_params": saved_params, "model_info": saved_info}
)
router: Final = litellm.Router(model_list=[{
"model_name": "editor-model", "litellm_params": saved_params,
"model_info": {"id": "editor-model", "db_model": True},
}])
router: Final = litellm.Router(
model_list=[
{
"model_name": "editor-model",
"litellm_params": saved_params,
"model_info": {"id": "editor-model", "db_model": True},
}
]
)
stored_params: Final = {**saved_params, "api_key": "synthetic-stored-secret"}
mock_prisma.db.litellm_proxymodeltable.find_unique.return_value = LiteLLM_ProxyModelTable(
model_id="editor-model", model_name="editor-model",
model_id="editor-model",
model_name="editor-model",
litellm_params=json.dumps(stored_params) if encoded else stored_params,
model_info={"id": "editor-model"},
)
@ -989,6 +1007,50 @@ def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deplo
assert ("/v1/batch" in (groups["gpt-mixed"]["supported_endpoints"] or ())) is lists_batch
@pytest.mark.parametrize(("team_id", "efforts"), [(_BYOK_TEAM_ID, ["low", "high", "max"]), ("team-other", None)])
def test_model_group_info_describes_a_team_model_under_the_public_name_its_team_calls(
client, auth_as, monkeypatch, team_id: str, efforts: list[str] | None
):
"""A team's own model is listed under its public name, so its row must describe the team's deployment
behind that name instead of a placeholder, and only for the team that owns it."""
from litellm.proxy._types import LitellmUserRoles
from litellm.proxy.agent_endpoints import model_list_helpers
router: Final = litellm.Router(
model_list=[
{
"model_name": _BYOK_INTERNAL_NAME,
"litellm_params": {"model": "hosted_vllm/qa-local-coder", "api_base": "https://vllm.example/v1"},
"model_info": {
"team_id": _BYOK_TEAM_ID,
"team_public_model_name": _BYOK_PUBLIC_NAME,
"supports_reasoning": True,
"reasoning_effort_levels": ["low", "high", "max"],
},
}
]
)
async def append_agents_to_model_group(*, model_groups, user_api_key_dict):
return model_groups
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list)
monkeypatch.setattr(proxy_server, "user_model", None)
monkeypatch.setattr(proxy_server, "general_settings", {})
monkeypatch.setattr(proxy_server, "prisma_client", None)
monkeypatch.setattr(
model_list_helpers, "append_agents_to_model_group", AsyncMock(side_effect=append_agents_to_model_group)
)
with auth_as(LitellmUserRoles.INTERNAL_USER, user_id=None, team_id=team_id, team_models=[_BYOK_PUBLIC_NAME]):
response: Final = client.get("/model_group/info")
assert response.status_code == 200
groups: Final = {group["model_group"]: group for group in response.json()["data"]}
assert groups[_BYOK_PUBLIC_NAME]["supported_reasoning_efforts"] == efforts
# ---------------------------------------------------------------------------
# GET /v2/model/info?exclude_auto_routers
# ---------------------------------------------------------------------------

71
tests/unit/proxy/proxy_server/test_routes_models.py
View file

@ -17,14 +17,12 @@ import litellm
from litellm.proxy import proxy_server
from litellm.proxy import utils as proxy_utils
from litellm.proxy.utils import create_model_info_response
from litellm.types.router import DeploymentModelListingInfo, ModelGroupInfo
from litellm.types.router import DeploymentModelListingInfo
from .conftest import normalize # type: ignore[import-not-found]
def _stub_model_info_response(
model_id: str = "gpt-4", provider: str = "openai"
) -> dict:
def _stub_model_info_response(model_id: str = "gpt-4", provider: str = "openai") -> dict:
return {
"id": model_id,
"object": "model",
@ -66,9 +64,7 @@ def patched_models(monkeypatch):
def _fake_create_model_info_response(model_id, provider="openai", **kwargs):
return _stub_model_info_response(model_id=model_id, provider=provider)
monkeypatch.setattr(
proxy_utils, "create_model_info_response", _fake_create_model_info_response
)
monkeypatch.setattr(proxy_utils, "create_model_info_response", _fake_create_model_info_response)
monkeypatch.setattr(proxy_utils, "validate_model_access", lambda **kwargs: None)
@ -107,9 +103,7 @@ def test_get_models_happy_path(client, auth_as, patched_models, path):
@pytest.mark.parametrize("path", ["/v1/models", "/models"])
def test_get_models_anthropic_format_when_header_present(
client, auth_as, patched_models, path
):
def test_get_models_anthropic_format_when_header_present(client, auth_as, patched_models, path):
"""Pins: ``GET /v1/models`` returns the Anthropic-native models shape when
the caller sends an ``anthropic-version`` header (Claude Code gateway
discovery), while the default OpenAI shape is unchanged without it."""
@ -129,9 +123,7 @@ def test_get_models_anthropic_format_when_header_present(
@pytest.mark.parametrize("path", ["/v1/models", "/models"])
def test_anthropic_format_exposes_token_limits(
client, auth_as, patched_models, monkeypatch, path
):
def test_anthropic_format_exposes_token_limits(client, auth_as, patched_models, monkeypatch, path):
"""Claude Code sizes requests off the listing, so the Anthropic-native entries
carry the same token limits the OpenAI listing resolves, with the output budget
named max_tokens as the Messages API names it."""
@ -146,9 +138,7 @@ def test_anthropic_format_exposes_token_limits(
"max_output_tokens": 64000,
}
monkeypatch.setattr(
proxy_utils, "create_model_info_response", _create_model_info_response
)
monkeypatch.setattr(proxy_utils, "create_model_info_response", _create_model_info_response)
with auth_as():
response = client.get(path, headers={"anthropic-version": "2023-06-01"})
@ -223,9 +213,7 @@ def test_anthropic_format_uses_configured_display_name(client, auth_as, patched_
@pytest.mark.parametrize("params", [{}, {"scope": "expand"}])
def test_anthropic_display_name_resolved_via_internal_team_key(
client, auth_as, patched_models, monkeypatch, params
):
def test_anthropic_display_name_resolved_via_internal_team_key(client, auth_as, patched_models, monkeypatch, params):
"""For a team-scoped row the configured display name must be looked up by the
internal routing key while the entry itself is keyed by the public name, so
the clean name lands on the id the client actually sees."""
@ -258,14 +246,10 @@ def test_anthropic_display_name_resolved_via_internal_team_key(
"get_available_models_for_user",
_fake_get_available_models_for_user,
)
monkeypatch.setattr(
model_checks, "get_complete_model_list", lambda **kwargs: [internal_name]
)
monkeypatch.setattr(model_checks, "get_complete_model_list", lambda **kwargs: [internal_name])
with auth_as():
response = client.get(
"/v1/models", params=params, headers={"anthropic-version": "2023-06-01"}
)
response = client.get("/v1/models", params=params, headers={"anthropic-version": "2023-06-01"})
assert response.status_code == 200
(entry,) = response.json()["data"]
@ -306,9 +290,7 @@ def test_get_model_by_id_not_found(client, auth_as, patched_models, path):
@pytest.mark.parametrize("params", [{}, {"scope": "expand"}])
def test_anthropic_format_returns_public_team_model_name(
client, auth_as, patched_models, monkeypatch, params
):
def test_anthropic_format_returns_public_team_model_name(client, auth_as, patched_models, monkeypatch, params):
"""Regression: the Anthropic-native listing must go through the same team
name translation as the OpenAI listing, so a caller never sees the internal
``model_name_{team_id}_{uuid}`` routing key."""
@ -338,14 +320,10 @@ def test_anthropic_format_returns_public_team_model_name(
"get_available_models_for_user",
_fake_get_available_models_for_user,
)
monkeypatch.setattr(
model_checks, "get_complete_model_list", lambda **kwargs: [internal_name]
)
monkeypatch.setattr(model_checks, "get_complete_model_list", lambda **kwargs: [internal_name])
with auth_as():
response = client.get(
"/v1/models", params=params, headers={"anthropic-version": "2023-06-01"}
)
response = client.get("/v1/models", params=params, headers={"anthropic-version": "2023-06-01"})
assert response.status_code == 200
assert [m["id"] for m in response.json()["data"]] == ["gpt-4-team"]
@ -376,7 +354,9 @@ def test_anthropic_format_lists_claude_code_view_ids_for_claude_code(
patched_models.model_group_alias = {}
patched_models.has_model_id.return_value = False
patched_models.get_candidate_model_ids_for_route.side_effect = lambda name, team_id=None: frozenset({name}) if name in ("gpt-4", "claude-sonnet") else frozenset()
patched_models.get_candidate_model_ids_for_route.side_effect = lambda name, team_id=None: (
frozenset({name}) if name in ("gpt-4", "claude-sonnet") else frozenset()
)
monkeypatch.setattr(proxy_utils, "create_model_info_response", _create_model_info_response)
with auth_as():
@ -396,7 +376,9 @@ def test_anthropic_format_lists_claude_code_view_ids_for_claude_code(
def test_anthropic_format_keeps_served_ids_for_other_anthropic_clients(client, auth_as, patched_models, path):
"""An Anthropic SDK asking for the vendor shape gets the served ids: the view is Claude Code's alone."""
with auth_as():
response = client.get(path, headers={"anthropic-version": "2023-06-01", "user-agent": "anthropic-sdk-python/0.40"})
response = client.get(
path, headers={"anthropic-version": "2023-06-01", "user-agent": "anthropic-sdk-python/0.40"}
)
assert response.status_code == 200
assert [m["id"] for m in response.json()["data"]] == ["gpt-4", "claude-sonnet"]
@ -462,14 +444,21 @@ def test_codex_format_carries_configured_service_tiers(client, auth_as, patched_
@pytest.mark.parametrize("params", [{}, {"scope": "expand"}])
def test_codex_reasoning_levels_are_the_ones_the_requesting_codex_parses(client, auth_as, patched_models, params):
def test_codex_reasoning_levels_are_the_ones_the_requesting_codex_parses(client, auth_as, monkeypatch, params):
"""Both listing paths hand the route's ``client_version`` to the catalog: Codex before 0.138.0 cannot
parse max, so it is offered there only to a newer release."""
patched_models.get_model_group_info = MagicMock(
return_value=ModelGroupInfo(
model_group="gpt-4", providers=["openai"], supported_reasoning_efforts=("low", "max")
)
declared = {"supports_reasoning": True, "reasoning_effort_levels": ["low", "max"]}
vllm = {"model": "hosted_vllm/qa-local-coder", "api_base": "https://vllm.example/v1"}
router = litellm.Router(
model_list=[
{"model_name": name, "litellm_params": vllm, "model_info": declared} for name in ("gpt-4", "claude-sonnet")
]
)
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list)
monkeypatch.setattr(proxy_server, "user_model", None)
monkeypatch.setattr(proxy_server, "general_settings", {})
monkeypatch.setattr(proxy_server, "prisma_client", None)
with auth_as():
current = client.get("/v1/models", params={**params, "client_version": "0.165.0"})