fix(proxy): let typesafe/* allowlists cover /typesafe Jev calls and list Jev in the model picker (#47697)

* fix(proxy): list TypeSafe Jev models in the admin model picker

* fix(proxy): let typesafe/* allowlists cover bare /typesafe Jev ids

Adds a typesafe-jev-ids routing rule to the cost map so a bare jev-* id resolves to TypeSafe, which lets typesafe/* on a key, team or project allowlist match /typesafe calls. The picker gate now only checks that a TypeSafe key is set, so an unresolvable OIDC reference cannot fail the listing, and the /v1/models route tests mock deployments with litellm_params like real ones.

* fix(typesafe): match only published Jev ids and keep picker loads off the secret manager

Narrows the typesafe-jev-ids rule to jev-latest, jev-preview and versioned ids so a model group such as jev-internal-router never resolves to TypeSafe, checks TYPESAFE_API_KEY in the environment instead of through the secret manager on each picker load, and moves the picker tests into the mapped test file.

* test(typesafe): type the picker test fixture and tests

* fix(typesafe): match only published Jev versions and cover non-version suffixes in tests

* fix(typesafe): match only priced Jev ids and offer TypeSafe in the no-team key picker

* fix(proxy): offer TypeSafe allowlist names only to the Admin UI allowlist pickers

The dashboard sends include_model_access_groups on every /models call, so
gate the TypeSafe names on wildcard routes too and skip access-group-only
listings. Also say the typesafe-jev-ids rule matches case-insensitively.

* fix(ui): offer only the TypeSafe names the cost-map rule routes, and keep /v1/models listing as on main

A typesafe/* router deployment registers typesafe/* in the cost map, so the
allowlist pickers offered a bare *, which allows every model. The pickers now
list only priced names the rule sends to TypeSafe. The non-expand list also
stops de-duplicating every entry and only skips TypeSafe names already listed.
This commit is contained in:
Mateo Wang 2026-10-28 20:19:58 -07:00 • committed by GitHub
parent 3a350cacc9
commit f46adf80e4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 385 additions and 5 deletions

View file

@ -63057,6 +63057,14 @@
"litellm_provider": "anthropic"
}
},
{
"name": "typesafe-jev-ids",
"pattern": "^jev-(?:latest|preview|1\\.13\\.0|1\\.14\\.0)$",
"description": "A bare Jev id that has a typesafe/ price entry (jev-latest, jev-preview, jev-1.13.0, jev-1.14.0), the name the /typesafe pass-through forwards to TypeSafe. Listed by name, matched case-insensitively like every rule here, so foo/jev-latest or a model group named like a Jev id TypeSafe does not serve, such as jev-1.0 or jev-internal-router, never resolves to typesafe. A new Jev version goes here together with its price entry. Provider inference feeds the proxy's provider/* wildcard access checks, so this rule is what lets typesafe/* on a key, team or project allowlist cover a jev-latest request.",
"model_info": {
"litellm_provider": "typesafe"
}
},
{
"name": "claude-family-baseline",
"pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?",

View file

@ -50,6 +50,7 @@ from litellm.constants import (
from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers
from litellm.llms.azure.passthrough.transformation import (
foreign_azure_deployment,
@ -698,6 +699,20 @@ async def typesafe_proxy_route(
return await endpoint_func(request, fastapi_response, user_api_key_dict)
_COST_MAP_ENTRIES: Final = TypeAdapter(Mapping[str, object])
def typesafe_allowlist_model_names() -> tuple[str, ...]:
if not passthrough_endpoint_router.has_credentials(custom_llm_provider="typesafe"):
return ()
cost_map: Final = _COST_MAP_ENTRIES.validate_python(litellm.model_cost) # pyright: ignore[reportUnknownMemberType] # legacy registry is validated at this boundary
priced_names: Final = (name.removeprefix("typesafe/") for name in cost_map if name.startswith("typesafe/"))
jev_models_typesafe_serves: Final = sorted(
name for name in priced_names if match_routing_generalization(name) == "typesafe"
)
return ("typesafe/*", *jev_models_typesafe_serves)
@router.post(
"/laya/v1/systemone",
tags=["Laya Pass-through", "pass-through"],

View file

@ -1,4 +1,5 @@
import json
import os
from collections.abc import Callable
from typing import TYPE_CHECKING, Final
@ -88,6 +89,14 @@ class PassthroughEndpointRouter:
)
return _resolve_oidc_reference(get_secret_str(_env_variable_name))
def has_credentials(self, custom_llm_provider: str) -> bool:
if self._get_deployment_api_key(custom_llm_provider=custom_llm_provider, region_name=None) is not None:
return True
env_variable_name: Final = self._get_default_env_variable_name_passthrough_endpoint(
custom_llm_provider=custom_llm_provider,
)
return bool(os.environ.get(env_variable_name))
def _get_deployment_api_key(
self,
custom_llm_provider: str,

View file

@ -12390,6 +12390,12 @@ async def model_list(
hidden_names: Final = blocked_names | unhealthy_names
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import typesafe_allowlist_model_names
asks_for_allowlist_names: Final = (
include_model_access_groups is True and return_wildcard_routes is True and only_model_access_groups is not True
)
# If scope=expand and user has admin privileges, return all proxy models
if should_expand_scope:
# Get all proxy models as if user is a proxy admin
@ -12410,7 +12416,10 @@ async def model_list(
all_models = get_complete_model_list(
key_models=[],
team_models=[],
proxy_model_list=proxy_model_list,
proxy_model_list=[
*proxy_model_list,
*(typesafe_allowlist_model_names() if asks_for_allowlist_names else ()),
],
user_model=None,
infer_model_from_keys=False,
return_wildcard_routes=include_wildcard_routes,
@ -12471,7 +12480,7 @@ async def model_list(
# Otherwise, use the normal behavior (current implementation)
# Get available models for the user
all_models = await get_available_models_for_user(
available_models: Final = await get_available_models_for_user(
user_api_key_dict=user_api_key_dict,
llm_router=llm_router,
general_settings=general_settings,
@ -12484,6 +12493,11 @@ async def model_list(
return_wildcard_routes=include_wildcard_routes,
user_api_key_cache=user_api_key_cache,
)
offers_typesafe_allowlist_names: Final = (
asks_for_allowlist_names and team_id is None and user_api_key_has_admin_view(user_api_key_dict)
)
typesafe_names_to_offer: Final = typesafe_allowlist_model_names() if offers_typesafe_allowlist_names else ()
all_models = [*available_models, *(name for name in typesafe_names_to_offer if name not in available_models)]
undiscoverable_names: Final = undiscoverable_model_names(
all_models, llm_router, user_api_key_dict, team_id or user_api_key_dict.team_id

View file

@ -63057,6 +63057,14 @@
"litellm_provider": "anthropic"
}
},
{
"name": "typesafe-jev-ids",
"pattern": "^jev-(?:latest|preview|1\\.13\\.0|1\\.14\\.0)$",
"description": "A bare Jev id that has a typesafe/ price entry (jev-latest, jev-preview, jev-1.13.0, jev-1.14.0), the name the /typesafe pass-through forwards to TypeSafe. Listed by name, matched case-insensitively like every rule here, so foo/jev-latest or a model group named like a Jev id TypeSafe does not serve, such as jev-1.0 or jev-internal-router, never resolves to typesafe. A new Jev version goes here together with its price entry. Provider inference feeds the proxy's provider/* wildcard access checks, so this rule is what lets typesafe/* on a key, team or project allowlist cover a jev-latest request.",
"model_info": {
"litellm_provider": "typesafe"
}
},
{
"name": "claude-family-baseline",
"pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?",

View file

@ -714,6 +714,48 @@ class TestClaudeModelPatternMatching:
litellm.get_llm_provider(model="claude-opus-4-9")
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
class TestTypesafeJevPatternMatching:
def test_every_priced_jev_id_routes_to_typesafe(
self, shipped_generalizations: None
) -> None:
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
priced_ids: Final = tuple(
key.removeprefix("typesafe/")
for key in GetModelCostMap.load_local_model_cost_map()
if key.startswith("typesafe/")
)
assert "jev-latest" in priced_ids
assert {
model: litellm.get_llm_provider(model=model)[1] for model in priced_ids
} == dict.fromkeys(priced_ids, "typesafe")
@pytest.mark.parametrize(
"model",
[
"foo/jev-latest",
"openjev-latest",
"jev-",
"jev-latest model",
"jev-internal-router",
"jev-2.0-internal",
"jev-1.0-acme.router",
"jev-1.0",
"jev-2.0",
"jev-1.13",
],
)
def test_namespaced_or_partial_jev_name_does_not_match_rule(
self, model: str, shipped_generalizations: None
) -> None:
from litellm.litellm_core_utils.fallback_generalizations import (
match_routing_generalization,
)
assert match_routing_generalization(model) is None
@pytest.mark.parametrize("prefix", ["openai", "anthropic", "custom_openai", "openai_like"])
def test_opencode_affinity_resolution_preserves_existing_transport(prefix: str) -> None:
api_base: Final = "https://opencode.ai/zen/go/v1"

View file

@ -12,7 +12,7 @@ load_dotenv()
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from copy import deepcopy
from types import SimpleNamespace
from typing import Final
from typing import Final, Literal
from unittest.mock import AsyncMock, MagicMock, patch
import pytest, litellm
@ -721,6 +721,51 @@ async def test_can_key_call_model_wildcard_access(key_models, model, expect_to_w
)
@pytest.fixture
def shipped_routing_rules() -> Iterator[None]:
from litellm.litellm_core_utils.fallback_generalizations import (
get_fallback_generalization_rules,
set_fallback_generalizations,
)
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
previous_rules: Final = list(get_fallback_generalization_rules())
set_fallback_generalizations(GetModelCostMap.load_local_model_cost_map()["fallback_generalizations"]["rules"])
yield
set_fallback_generalizations(previous_rules)
@pytest.mark.parametrize("object_type", ["team", "project"])
@pytest.mark.parametrize(
"allowed_models, model, expect_to_work",
[
(["typesafe/*"], "jev-latest", True),
(["typesafe/*"], "jev-1.14.0", True),
(["jev-latest"], "jev-latest", True),
(["openai/*"], "jev-latest", False),
(["typesafe/*"], "foo/jev-latest", False),
(["typesafe/*"], "jev-internal-router", False),
(["typesafe/*"], "jev-2.0-internal", False),
(["typesafe/*"], "jev-1.0", False),
],
)
def test_typesafe_provider_wildcard_covers_bare_jev_model(
shipped_routing_rules: None,
object_type: Literal["team", "project"],
allowed_models: list[str],
model: str,
expect_to_work: bool,
) -> None:
from litellm.proxy._types import ProxyException
from litellm.proxy.auth.auth_checks import can_object_call_model
if expect_to_work:
assert can_object_call_model(model=model, llm_router=None, models=allowed_models, object_type=object_type)
return
with pytest.raises(ProxyException):
can_object_call_model(model=model, llm_router=None, models=allowed_models, object_type=object_type)
@pytest.mark.parametrize(
"key_models, model, expect_to_work",
[

View file

@ -11,11 +11,14 @@ import contextlib
import os
import sys
from pathlib import Path
from typing import Any, AsyncIterator, Callable, Dict, Iterator, List, Optional
from typing import TYPE_CHECKING, Any, AsyncIterator, Callable, Dict, Final, Iterator, List, Optional
from unittest.mock import AsyncMock, MagicMock
import pytest
if TYPE_CHECKING:
from litellm import Router
# Repo root, anchored to this file (not CWD) so the path is correct no
# matter where pytest is invoked from. With the project installed via
# uv this is defensive — `litellm` already resolves through site-packages
@ -536,3 +539,25 @@ def reset_login_throttle(monkeypatch):
_drop_throttle_keys()
yield _drop_throttle_keys
_drop_throttle_keys()
@pytest.fixture
def openai_only_router(monkeypatch: pytest.MonkeyPatch) -> Router:
from litellm import Router
from litellm.proxy import proxy_server
router: Final = Router(
model_list=[
{
"model_name": "gpt-4",
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"},
"model_info": {"id": "gpt-4-id"},
}
]
)
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list)
monkeypatch.setattr(proxy_server, "prisma_client", None)
monkeypatch.setattr(proxy_server, "general_settings", {})
monkeypatch.setattr(proxy_server, "user_model", None)
return router

View file

@ -9,15 +9,20 @@ Pins (PR2):
from __future__ import annotations
from typing import Final
from unittest.mock import MagicMock
import pytest
import litellm
from litellm import Router
from litellm.integrations.custom_secret_manager import CustomSecretManager
from litellm.proxy import proxy_server
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy import utils as proxy_utils
from litellm.proxy.utils import create_model_info_response
from litellm.types.router import DeploymentModelListingInfo
from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem
from .conftest import normalize # type: ignore[import-not-found]
@ -237,6 +242,7 @@ def test_anthropic_display_name_resolved_via_internal_team_key(
return_value=[
{
"model_name": internal_name,
"litellm_params": {"model": "openai/gpt-4"},
"model_info": {
"team_id": "team-1",
"team_public_model_name": "gpt-4-team",
@ -320,6 +326,7 @@ def test_anthropic_format_returns_public_team_model_name(
return_value=[
{
"model_name": internal_name,
"litellm_params": {"model": "openai/gpt-4"},
"model_info": {
"team_id": "team-1",
"team_public_model_name": "gpt-4-team",
@ -494,7 +501,11 @@ def test_codex_service_tiers_resolved_via_internal_team_key(client, auth_as, pat
patched_models.get_model_list = MagicMock(
return_value=[
{"model_name": internal_name, "model_info": {"team_id": "team-1", "team_public_model_name": "gpt-4-team"}}
{
"model_name": internal_name,
"litellm_params": {"model": "openai/gpt-4"},
"model_info": {"team_id": "team-1", "team_public_model_name": "gpt-4-team"},
}
]
)
patched_models.get_model_names = MagicMock(return_value=[internal_name])
@ -514,3 +525,206 @@ def test_codex_service_tiers_resolved_via_internal_team_key(client, auth_as, pat
assert response.status_code == 200
(entry,) = response.json()["models"]
assert (entry["slug"], [tier["id"] for tier in entry["service_tiers"]]) == ("gpt-4-team", ["ultrafast"])
def _priced_jev_models() -> frozenset[str]:
return frozenset(name.removeprefix("typesafe/") for name in litellm.model_cost if name.startswith("typesafe/"))
async def _admin_picker_models() -> set[str]:
admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN)
response: Final = await proxy_server.model_list(
user_api_key_dict=admin, return_wildcard_routes=True, include_model_access_groups=True, scope="expand"
)
return {model["id"] for model in response["data"]}
@pytest.mark.asyncio
async def test_scope_expand_offers_typesafe_models_when_typesafe_is_configured(
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
jev_models: Final = _priced_jev_models()
picker_models: Final = await _admin_picker_models()
assert jev_models
assert {"gpt-4", "typesafe/*", *jev_models} <= picker_models
@pytest.mark.asyncio
async def test_scope_expand_still_lists_models_when_the_typesafe_key_reference_cannot_resolve(
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("LITELLM_QA_UNSET_TYPESAFE_TOKEN", raising=False)
monkeypatch.setenv("TYPESAFE_API_KEY", "oidc/env/LITELLM_QA_UNSET_TYPESAFE_TOKEN")
picker_models: Final = await _admin_picker_models()
assert {"gpt-4", "typesafe/*"} <= picker_models
@pytest.mark.asyncio
async def test_scope_expand_offers_no_typesafe_models_without_typesafe_credentials(
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("TYPESAFE_API_KEY", raising=False)
picker_models: Final = await _admin_picker_models()
assert "gpt-4" in picker_models
assert picker_models.isdisjoint({"typesafe/*", *_priced_jev_models()})
@pytest.mark.asyncio
async def test_scope_expand_does_not_read_the_secret_manager_for_typesafe_credentials(
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
secret_manager: Final = MagicMock(spec=CustomSecretManager)
secret_manager.sync_read_secret.return_value = None
monkeypatch.delenv("TYPESAFE_API_KEY", raising=False)
monkeypatch.setattr(litellm, "secret_manager_client", secret_manager)
monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.CUSTOM)
monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings(access_mode="read_only"))
picker_models: Final = await _admin_picker_models()
secrets_read: Final = {
call.kwargs["secret_name"]
for call in (*secret_manager.sync_read_secret.call_args_list, *secret_manager.async_read_secret.call_args_list)
}
assert "gpt-4" in picker_models
assert "TYPESAFE_API_KEY" not in secrets_read
@pytest.mark.asyncio
async def test_default_model_list_does_not_offer_typesafe_models(
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN)
response: Final = await proxy_server.model_list(user_api_key_dict=admin)
assert {model["id"] for model in response["data"]}.isdisjoint({"typesafe/*", *_priced_jev_models()})
async def _key_dialog_models(user_role: LitellmUserRoles, team_id: str | None) -> set[str]:
caller: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=user_role)
response: Final = await proxy_server.model_list(
user_api_key_dict=caller, return_wildcard_routes=True, include_model_access_groups=True, team_id=team_id
)
return {model["id"] for model in response["data"]}
@pytest.mark.asyncio
async def test_key_dialog_without_a_team_offers_typesafe_models_to_an_admin(
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
dialog_models: Final = await _key_dialog_models(LitellmUserRoles.PROXY_ADMIN, team_id=None)
assert {"gpt-4", "typesafe/*", *_priced_jev_models()} <= dialog_models
@pytest.mark.asyncio
@pytest.mark.parametrize("scope", ["expand", None])
async def test_pickers_offer_typesafe_names_once_and_no_bare_name_for_registered_typesafe_deployments(
monkeypatch: pytest.MonkeyPatch, scope: str | None
) -> None:
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
jev_models: Final = _priced_jev_models()
monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost})
router: Final = Router(
model_list=[
{"model_name": "typesafe/*", "litellm_params": {"model": "typesafe/*", "api_key": "sk-typesafe-fake"}},
{
"model_name": "ticket-router",
"litellm_params": {"model": "typesafe/my-classifier", "api_key": "sk-typesafe-fake"},
},
]
)
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list)
monkeypatch.setattr(proxy_server, "prisma_client", None)
monkeypatch.setattr(proxy_server, "general_settings", {})
monkeypatch.setattr(proxy_server, "user_model", None)
admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN)
response: Final = await proxy_server.model_list(
user_api_key_dict=admin, return_wildcard_routes=True, include_model_access_groups=True, scope=scope
)
listed: Final = [model["id"] for model in response["data"]]
assert {"ticket-router", *jev_models} <= set(listed)
assert listed.count("typesafe/*") == 1
assert set(listed).isdisjoint({"*", "my-classifier"})
@pytest.mark.asyncio
async def test_key_dialog_offers_no_typesafe_models_to_an_internal_user(
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
dialog_models: Final = await _key_dialog_models(LitellmUserRoles.INTERNAL_USER, team_id=None)
assert "gpt-4" in dialog_models
assert dialog_models.isdisjoint({"typesafe/*", *_priced_jev_models()})
@pytest.mark.asyncio
async def test_key_dialog_for_a_team_offers_no_typesafe_models_the_team_lacks(
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
dialog_models: Final = await _key_dialog_models(LitellmUserRoles.PROXY_ADMIN, team_id="team-without-typesafe")
assert dialog_models.isdisjoint({"typesafe/*", *_priced_jev_models()})
@pytest.mark.asyncio
async def test_scope_expand_lists_no_typesafe_models_to_a_client_not_asking_for_allowlist_names(
openai_only_router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN)
response: Final = await proxy_server.model_list(user_api_key_dict=admin, scope="expand")
listed: Final = {model["id"] for model in response["data"]}
assert "gpt-4" in listed
assert listed.isdisjoint({"typesafe/*", *_priced_jev_models()})
@pytest.mark.asyncio
@pytest.mark.parametrize(
("scope", "return_wildcard_routes", "only_model_access_groups"),
[
pytest.param(None, False, True, id="access-group-dropdown"),
pytest.param(None, None, None, id="dashboard-list-without-wildcard-routes"),
pytest.param("expand", None, None, id="expanded-list-without-wildcard-routes"),
pytest.param(None, True, True, id="access-groups-only-with-wildcard-routes"),
],
)
async def test_dashboard_lists_not_asking_for_allowlist_names_offer_no_typesafe_models(
openai_only_router: Router,
monkeypatch: pytest.MonkeyPatch,
scope: str | None,
return_wildcard_routes: bool | None,
only_model_access_groups: bool | None,
) -> None:
monkeypatch.setenv("TYPESAFE_API_KEY", "sk-typesafe-fake")
admin: Final = UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN)
response: Final = await proxy_server.model_list(
user_api_key_dict=admin,
include_model_access_groups=True,
return_wildcard_routes=return_wildcard_routes,
only_model_access_groups=only_model_access_groups,
scope=scope,
)
assert {model["id"] for model in response["data"]}.isdisjoint({"typesafe/*", *_priced_jev_models()})