From c0f6ebcc26829d39d0083eaf32a5e8dc2ef375d5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:44:13 -0700 Subject: [PATCH 01/11] fix(openrouter): list OpenRouter's live catalog for openrouter/* when check_provider_endpoint is on --- litellm/llms/openrouter/common_utils.py | 86 ++++++++++ litellm/utils.py | 4 + .../test_openrouter_common_utils.py | 155 ++++++++++++++++++ tests/unit/proxy/auth/test_model_checks.py | 49 ++++++ 4 files changed, 294 insertions(+) create mode 100644 tests/unit/llms/openrouter/test_openrouter_common_utils.py diff --git a/litellm/llms/openrouter/common_utils.py b/litellm/llms/openrouter/common_utils.py index 96e53a5aae7..701d75b350e 100644 --- a/litellm/llms/openrouter/common_utils.py +++ b/litellm/llms/openrouter/common_utils.py @@ -1,5 +1,91 @@ +import itertools +from collections.abc import Mapping, Sequence +from typing import Final + +import httpx +from pydantic import BaseModel, ConfigDict + +import litellm +from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.base_llm.base_utils import BaseLLMModelInfo from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues + +_DEFAULT_API_BASE: Final = "https://openrouter.ai/api/v1" +_CATALOG_ROUTES: Final = ("models/user", "embeddings/models") +_CATALOG_TIMEOUT: Final = httpx.Timeout(10.0) class OpenRouterException(BaseLLMException): pass + + +class _CatalogModel(BaseModel): + model_config = ConfigDict(frozen=True) + + id: str + + +class _Catalog(BaseModel): + model_config = ConfigDict(frozen=True) + + data: tuple[_CatalogModel, ...] + + +def _catalog_ids(url: str, headers: Mapping[str, str]) -> tuple[str, ...]: + response: Final = litellm.module_level_client.get( # pyright: ignore[reportUnknownMemberType] # legacy HTTP wrapper has untyped query parameters + url=url, headers=headers, timeout=_CATALOG_TIMEOUT + ) + response.raise_for_status() + return tuple(model.id for model in _Catalog.model_validate_json(response.content).data) + + +class OpenRouterModelInfo(BaseLLMModelInfo): + @staticmethod + def get_api_key(api_key: str | None = None) -> str | None: + return ( + api_key + or litellm.api_key + or litellm.openrouter_key + or get_secret_str("OPENROUTER_API_KEY") + or get_secret_str("OR_API_KEY") + ) + + @staticmethod + def get_api_base(api_base: str | None = None) -> str: + return api_base or litellm.api_base or get_secret_str("OPENROUTER_API_BASE") or _DEFAULT_API_BASE + + @staticmethod + def get_base_model(model: str) -> str: + return model.removeprefix("openrouter/") + + def validate_environment( + self, + headers: Mapping[str, str], + model: str, + messages: Sequence[AllMessageValues], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + api_key: str | None = None, + api_base: str | None = None, + ) -> dict[str, str]: # mutable-ok: BaseLLMModelInfo.validate_environment returns a dict + resolved_key: Final = self.get_api_key(api_key) + if resolved_key is None: + return {**headers} + return {**headers, "Authorization": f"Bearer {resolved_key}"} + + def get_models( + self, api_key: str | None = None, api_base: str | None = None + ) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override + resolved_key: Final = self.get_api_key(api_key) + if resolved_key is None: + raise ValueError("Set OPENROUTER_API_KEY or the deployment's api_key to list OpenRouter models") + headers: Final = self.validate_environment( + headers={}, model="", messages=(), optional_params={}, litellm_params={}, api_key=resolved_key + ) + base: Final = self.get_api_base(api_base) + model_ids: Final = itertools.chain.from_iterable( + _catalog_ids(join_api_base(base, route), headers) for route in _CATALOG_ROUTES + ) + return list(dict.fromkeys(f"openrouter/{model_id}" for model_id in model_ids)) diff --git a/litellm/utils.py b/litellm/utils.py index 1e095966a43..0719ff8bcde 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10069,6 +10069,10 @@ class ProviderConfigManager: from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo return AzureFoundryModelInfo(model=model) + elif LlmProviders.OPENROUTER == provider: + from litellm.llms.openrouter.common_utils import OpenRouterModelInfo + + return OpenRouterModelInfo() return None @staticmethod diff --git a/tests/unit/llms/openrouter/test_openrouter_common_utils.py b/tests/unit/llms/openrouter/test_openrouter_common_utils.py new file mode 100644 index 00000000000..10e2630b161 --- /dev/null +++ b/tests/unit/llms/openrouter/test_openrouter_common_utils.py @@ -0,0 +1,155 @@ +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.llms.openrouter.common_utils import OpenRouterModelInfo + +DEFAULT_BASE: Final = "https://openrouter.ai/api/v1" +KEY_SOURCES: Final = ("api_key", "litellm.api_key", "litellm.openrouter_key", "OPENROUTER_API_KEY", "OR_API_KEY") +BASE_SOURCES: Final = ("api_base", "litellm.api_base", "OPENROUTER_API_BASE") + + +@pytest.fixture(autouse=True) +def _no_ambient_openrouter_settings(monkeypatch: pytest.MonkeyPatch): + for env_var in ("OPENROUTER_API_BASE", "OPENROUTER_API_KEY", "OR_API_KEY"): + monkeypatch.delenv(env_var, raising=False) + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setattr(litellm, "openrouter_key", None) + + +def _catalog(*model_ids: str) -> httpx.Response: + return httpx.Response(200, json={"data": [{"id": model_id, "name": model_id} for model_id in model_ids]}) + + +def _configure(monkeypatch: pytest.MonkeyPatch, source: str, value: str) -> None: + if source.startswith("litellm."): + monkeypatch.setattr(litellm, source.removeprefix("litellm."), value) + else: + monkeypatch.setenv(source, value) + + +@respx.mock +def test_get_models_lists_both_catalogs_once_each_under_openrouter_keeping_the_org_namespace(): + respx.get(f"{DEFAULT_BASE}/models/user").mock( + return_value=_catalog("openai/gpt-5.4-mini", "anthropic/claude-haiku-5-5") + ) + respx.get(f"{DEFAULT_BASE}/embeddings/models").mock( + return_value=_catalog("openai/text-embedding-3-small", "openai/gpt-5.4-mini") + ) + + assert OpenRouterModelInfo().get_models(api_key="sk-or-test") == [ + "openrouter/openai/gpt-5.4-mini", + "openrouter/anthropic/claude-haiku-5-5", + "openrouter/openai/text-embedding-3-small", + ] + + +@respx.mock +def test_get_models_keeps_the_api_base_path_and_sends_the_deployment_key(): + chat_route: Final = respx.get("https://gateway.example/openrouter/api/v1/models/user").mock( + return_value=_catalog("openai/gpt-5.4-mini") + ) + embeddings_route: Final = respx.get("https://gateway.example/openrouter/api/v1/embeddings/models").mock( + return_value=_catalog() + ) + + assert OpenRouterModelInfo().get_models( + api_key="sk-or-deployment", api_base="https://gateway.example/openrouter/api/v1/" + ) == ["openrouter/openai/gpt-5.4-mini"] + assert [route.calls.last.request.headers["authorization"] for route in (chat_route, embeddings_route)] == [ + "Bearer sk-or-deployment", + "Bearer sk-or-deployment", + ] + + +@pytest.mark.parametrize("winner", range(len(KEY_SOURCES)), ids=KEY_SOURCES) +@respx.mock +def test_get_models_sends_the_first_configured_key_in_the_chat_paths_order( + monkeypatch: pytest.MonkeyPatch, winner: int +): + for source in KEY_SOURCES[winner + 1 :]: + _configure(monkeypatch, source, f"sk-or-from-{source}") + if winner > 0: + _configure(monkeypatch, KEY_SOURCES[winner], "sk-or-winner") + chat_route: Final = respx.get(f"{DEFAULT_BASE}/models/user").mock(return_value=_catalog("qwen/qwen3-coder")) + respx.get(f"{DEFAULT_BASE}/embeddings/models").mock(return_value=_catalog()) + + assert OpenRouterModelInfo().get_models(api_key="sk-or-winner" if winner == 0 else None) == [ + "openrouter/qwen/qwen3-coder" + ] + assert chat_route.calls.last.request.headers["authorization"] == "Bearer sk-or-winner" + + +@pytest.mark.parametrize("winner", range(len(BASE_SOURCES) + 1), ids=(*BASE_SOURCES, "default")) +@respx.mock +def test_get_models_calls_the_first_configured_base_in_the_chat_paths_order( + monkeypatch: pytest.MonkeyPatch, winner: int +): + for source in BASE_SOURCES[winner + 1 :]: + _configure(monkeypatch, source, f"https://{source.lower().replace('.', '-')}.example/api/v1") + if 0 < winner < len(BASE_SOURCES): + _configure(monkeypatch, BASE_SOURCES[winner], "https://winner.example/api/v1") + expected_base: Final = "https://winner.example/api/v1" if winner < len(BASE_SOURCES) else DEFAULT_BASE + respx.get(f"{expected_base}/models/user").mock(return_value=_catalog("qwen/qwen3-coder")) + respx.get(f"{expected_base}/embeddings/models").mock(return_value=_catalog()) + + assert OpenRouterModelInfo().get_models( + api_key="sk-or-test", api_base="https://winner.example/api/v1" if winner == 0 else None + ) == ["openrouter/qwen/qwen3-coder"] + + +def test_get_models_without_any_key_raises_before_calling_openrouter(): + with respx.mock(assert_all_called=False) as respx_mock: + chat_route: Final = respx_mock.get(f"{DEFAULT_BASE}/models/user").mock( + return_value=_catalog("openai/gpt-5.4-mini") + ) + + with pytest.raises(ValueError, match="OPENROUTER_API_KEY"): + OpenRouterModelInfo().get_models() + + assert not chat_route.called + + +@pytest.mark.parametrize("failing_route", ["models/user", "embeddings/models"]) +def test_get_models_raises_when_either_catalog_answers_an_error(failing_route: str): + with respx.mock(assert_all_called=False) as respx_mock: + respx_mock.get(f"{DEFAULT_BASE}/models/user").mock(return_value=_catalog("openai/gpt-5.4-mini")) + respx_mock.get(f"{DEFAULT_BASE}/embeddings/models").mock(return_value=_catalog()) + respx_mock.get(f"{DEFAULT_BASE}/{failing_route}").mock( + return_value=httpx.Response(401, json={"error": {"code": 401}}) + ) + + with pytest.raises(httpx.HTTPStatusError): + OpenRouterModelInfo().get_models(api_key="sk-or-revoked") + + +def test_validate_environment_adds_the_resolved_key_and_keeps_the_callers_headers(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-env") + headers: Final = {"X-Title": "my-app"} + + assert OpenRouterModelInfo().validate_environment( + headers=headers, model="openrouter/openai/gpt-5.4-mini", messages=(), optional_params={}, litellm_params={} + ) == {"X-Title": "my-app", "Authorization": "Bearer sk-or-env"} + assert headers == {"X-Title": "my-app"} + + +def test_validate_environment_without_any_key_sends_no_authorization(): + assert OpenRouterModelInfo().validate_environment( + headers={"X-Title": "my-app"}, + model="openrouter/openai/gpt-5.4-mini", + messages=(), + optional_params={}, + litellm_params={}, + ) == {"X-Title": "my-app"} + + +@pytest.mark.parametrize( + ("model", "base_model"), + [("openrouter/openai/gpt-5.4-mini", "openai/gpt-5.4-mini"), ("openai/gpt-5.4-mini", "openai/gpt-5.4-mini")], +) +def test_get_base_model_drops_only_the_openrouter_prefix(model: str, base_model: str): + assert OpenRouterModelInfo.get_base_model(model) == base_model diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 83761620cc9..549679e4054 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -1150,6 +1150,55 @@ def test_partial_bedrock_wildcard_lists_nothing_when_no_discovered_id_carries_it assert get_known_models_from_wildcard("bedrock/anthropic.*", deployment) == [] +@pytest.mark.parametrize( + ("wildcard", "deployment_model", "listed_ids"), + [ + ( + "openrouter/*", + "openrouter/*", + [ + "openrouter/openai/gpt-5.4-mini", + "openrouter/anthropic/claude-haiku-5-5", + "openrouter/openai/text-embedding-3-small", + ], + ), + ( + "or-team/*", + "openrouter/*", + ["or-team/openai/gpt-5.4-mini", "or-team/anthropic/claude-haiku-5-5", "or-team/openai/text-embedding-3-small"], + ), + ( + "openrouter/openai/*", + "openrouter/openai/*", + ["openrouter/openai/gpt-5.4-mini", "openrouter/openai/text-embedding-3-small"], + ), + ], +) +@respx.mock +def test_openrouter_wildcard_lists_the_live_catalog_with_each_vendor_namespace( + monkeypatch: pytest.MonkeyPatch, wildcard: str, deployment_model: str, listed_ids: list[str] +) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.delenv("OPENROUTER_API_BASE", raising=False) + respx.get("https://openrouter.ai/api/v1/models/user").mock( + return_value=httpx.Response( + 200, json={"data": [{"id": "openai/gpt-5.4-mini"}, {"id": "anthropic/claude-haiku-5-5"}]} + ) + ) + respx.get("https://openrouter.ai/api/v1/embeddings/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "openai/text-embedding-3-small"}]}) + ) + + assert ( + get_known_models_from_wildcard(wildcard, LiteLLM_Params(model=deployment_model, api_key=f"sk-or-{wildcard}")) + == listed_ids + ) + + @respx.mock def test_custom_prefix_that_starts_a_discovered_id_still_prefixes_every_listed_model( monkeypatch: pytest.MonkeyPatch, From 8d862c61d9bbc50be39be5d1ef008f0be8cd495e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:20:03 -0700 Subject: [PATCH 02/11] fix(openrouter): list models off the event loop and keep only chat-callable catalog ids The model-list builders are sync, and with check_provider_endpoint on they call OpenRouter's catalog with a 10 second timeout and no caching of failures. They ran on the event loop inside /v1/models, /model/info, /model_group/info and the per-user listing, so one slow catalog froze every request on the worker. Those calls now run in a worker thread with asyncio.to_thread File, batch and RAG credential lookups only need each name's deployment, and a wildcard that is itself a deployment sits in the list ahead of its expansions and resolves its own credentials. get_complete_model_list takes expand_deployed_wildcard_routes, which that lookup sets to False, so a deployed wildcard never fetches its provider catalog there. A key wildcard wider than any deployment, like anthropic/* over a deployed anthropic/claude-*, still expands so it finds the matching deployment's credentials. Every other caller keeps the default The pattern router's model lookup tolerates a pattern removed between its two reads of the pattern table, which a deployment delete on the event loop can now do while a listing runs in a thread Listing now reads only OpenRouter's models/user catalog. OpenRouter's own /models list carries no embedding models, the OpenAI-format /v1/models has no mode field so chat pickers cannot tell embeddings apart, /model_group/info labels them mode chat, and one failing catalog emptied the whole list. Ids ending in :batch are dropped because chat completions answers 404 for them, while :free and every other variant stay. An api_base that already ends in /chat/completions, which chat accepts as is, has that suffix stripped before the catalog route is joined --- litellm/llms/openrouter/common_utils.py | 25 +++---- litellm/proxy/auth/model_checks.py | 5 ++ .../openai_files_endpoints/common_utils.py | 1 + litellm/proxy/proxy_server.py | 8 ++- litellm/proxy/utils.py | 6 +- .../router_utils/pattern_match_deployments.py | 4 +- .../test_openrouter_common_utils.py | 62 +++++++++-------- tests/unit/proxy/auth/test_model_checks.py | 21 +----- .../test_files_common_utils.py | 65 +++++++++++++++++ .../proxy/proxy_server/test_routes_models.py | 69 +++++++++++++++++++ .../proxy/utils/helpers/test_model_access.py | 62 ++++++++++++++++- 11 files changed, 262 insertions(+), 66 deletions(-) diff --git a/litellm/llms/openrouter/common_utils.py b/litellm/llms/openrouter/common_utils.py index 701d75b350e..64caf83d745 100644 --- a/litellm/llms/openrouter/common_utils.py +++ b/litellm/llms/openrouter/common_utils.py @@ -1,4 +1,3 @@ -import itertools from collections.abc import Mapping, Sequence from typing import Final @@ -13,7 +12,9 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues _DEFAULT_API_BASE: Final = "https://openrouter.ai/api/v1" -_CATALOG_ROUTES: Final = ("models/user", "embeddings/models") +_CHAT_CATALOG_ROUTE: Final = "models/user" +_CHAT_COMPLETIONS_PATH: Final = "/chat/completions" +_BATCH_ONLY_SUFFIX: Final = ":batch" _CATALOG_TIMEOUT: Final = httpx.Timeout(10.0) @@ -33,12 +34,8 @@ class _Catalog(BaseModel): data: tuple[_CatalogModel, ...] -def _catalog_ids(url: str, headers: Mapping[str, str]) -> tuple[str, ...]: - response: Final = litellm.module_level_client.get( # pyright: ignore[reportUnknownMemberType] # legacy HTTP wrapper has untyped query parameters - url=url, headers=headers, timeout=_CATALOG_TIMEOUT - ) - response.raise_for_status() - return tuple(model.id for model in _Catalog.model_validate_json(response.content).data) +def _chat_catalog_url(api_base: str) -> str: + return join_api_base(api_base.rstrip("/").removesuffix(_CHAT_COMPLETIONS_PATH), _CHAT_CATALOG_ROUTE) class OpenRouterModelInfo(BaseLLMModelInfo): @@ -84,8 +81,12 @@ class OpenRouterModelInfo(BaseLLMModelInfo): headers: Final = self.validate_environment( headers={}, model="", messages=(), optional_params={}, litellm_params={}, api_key=resolved_key ) - base: Final = self.get_api_base(api_base) - model_ids: Final = itertools.chain.from_iterable( - _catalog_ids(join_api_base(base, route), headers) for route in _CATALOG_ROUTES + response: Final = litellm.module_level_client.get( # pyright: ignore[reportUnknownMemberType] # legacy HTTP wrapper has untyped query parameters + url=_chat_catalog_url(self.get_api_base(api_base)), headers=headers, timeout=_CATALOG_TIMEOUT ) - return list(dict.fromkeys(f"openrouter/{model_id}" for model_id in model_ids)) + response.raise_for_status() + return [ + f"openrouter/{model.id}" + for model in _Catalog.model_validate_json(response.content).data + if not model.id.endswith(_BATCH_ONLY_SUFFIX) + ] diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index c4c538083ee..fb97a7c95b7 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -204,6 +204,7 @@ def get_complete_model_list( include_model_access_groups: bool | None = False, only_model_access_groups: bool | None = False, team_id: str | None = None, + expand_deployed_wildcard_routes: bool = True, ) -> list[str]: """Logic for returning complete model list for a given key + team pair""" @@ -249,6 +250,7 @@ def get_complete_model_list( return_wildcard_routes=return_wildcard_routes, llm_router=llm_router, team_id=team_id, + expand_deployed_wildcard_routes=expand_deployed_wildcard_routes, ) complete_model_list: Final = unique_models + all_wildcard_models @@ -539,6 +541,7 @@ def _get_wildcard_models( return_wildcard_routes: bool | None = False, llm_router: Router | None = None, team_id: str | None = None, + expand_deployed_wildcard_routes: bool = True, ) -> list[str]: models_to_remove: Final = set() routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id) @@ -551,6 +554,8 @@ def _get_wildcard_models( models_to_remove.add(model) model_list = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None + if model_list and not expand_deployed_wildcard_routes: + continue if model_list: for router_model in model_list: all_wildcard_models.extend( diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index f7289445d3a..6cbafd53537 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -657,6 +657,7 @@ def get_team_provider_credentials( model_access_groups=model_access_groups, include_model_access_groups=True, team_id=team_id, + expand_deployed_wildcard_routes=False, ) ) ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3b36ea5afce..84b05b88a41 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12398,7 +12398,8 @@ async def model_list( # Get complete model list including wildcard routes if requested from litellm.proxy.auth.model_checks import get_complete_model_list - all_models = get_complete_model_list( + all_models = await asyncio.to_thread( + get_complete_model_list, key_models=[], team_models=[], proxy_model_list=proxy_model_list, @@ -17751,7 +17752,7 @@ async def model_info_v1( alias_models: Final = copy.deepcopy(llm_router.get_model_list_from_model_alias()) all_models.extend(alias_models) - all_models = expand_wildcard_deployments_for_model_info(all_models, llm_router) + all_models = await asyncio.to_thread(expand_wildcard_deployments_for_model_info, all_models, llm_router) allowed_model_names: Final = await _get_v1_model_info_allowed_model_names_with_access_groups( user_api_key_dict=user_api_key_dict, @@ -18086,7 +18087,8 @@ async def model_group_info( LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, ) all_models_str: Final = ( - get_complete_model_list( + await asyncio.to_thread( + get_complete_model_list, key_models=(), team_models=(), proxy_model_list=llm_router.get_model_names(), diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0ddfc5ce62f..cc3c7280160 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -9517,7 +9517,8 @@ async def _agent_access_group_visible_models( if not granted: return frozenset() return frozenset( - get_complete_model_list( + await asyncio.to_thread( + get_complete_model_list, key_models=granted, team_models=(), proxy_model_list=proxy_model_list, @@ -9652,7 +9653,8 @@ async def get_available_models_for_user( ) # Get complete model list - complete_models: Final = get_complete_model_list( + complete_models: Final = await asyncio.to_thread( + get_complete_model_list, key_models=granted_key_models, team_models=granted_team_models, proxy_model_list=proxy_model_list, diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index 4dd217cf59b..2b8d1e9c11b 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -279,7 +279,9 @@ class PatternMatchRouter: PatternMatchRouter.set_deployment_model_name( matched_pattern=pattern_match, litellm_deployment_litellm_model=deployment["litellm_params"]["model"] ) - for deployment in self.patterns[pattern] + for deployment in self.patterns.get( # pyright: ignore[reportUnknownMemberType] # untyped deployment dicts + pattern, () + ) ) diff --git a/tests/unit/llms/openrouter/test_openrouter_common_utils.py b/tests/unit/llms/openrouter/test_openrouter_common_utils.py index 10e2630b161..087c05561b5 100644 --- a/tests/unit/llms/openrouter/test_openrouter_common_utils.py +++ b/tests/unit/llms/openrouter/test_openrouter_common_utils.py @@ -32,38 +32,49 @@ def _configure(monkeypatch: pytest.MonkeyPatch, source: str, value: str) -> None monkeypatch.setenv(source, value) +def test_get_models_lists_only_the_chat_catalog_under_openrouter_keeping_the_org_namespace(): + with respx.mock() as respx_mock: + respx_mock.get(f"{DEFAULT_BASE}/models/user").mock( + return_value=_catalog("openai/gpt-5.4-mini", "anthropic/claude-haiku-5-5") + ) + + assert OpenRouterModelInfo().get_models(api_key="sk-or-test") == [ + "openrouter/openai/gpt-5.4-mini", + "openrouter/anthropic/claude-haiku-5-5", + ] + assert [call.request.url.path for call in respx_mock.calls] == ["/api/v1/models/user"] + + @respx.mock -def test_get_models_lists_both_catalogs_once_each_under_openrouter_keeping_the_org_namespace(): +def test_get_models_drops_batch_only_ids_and_keeps_every_other_variant(): respx.get(f"{DEFAULT_BASE}/models/user").mock( - return_value=_catalog("openai/gpt-5.4-mini", "anthropic/claude-haiku-5-5") - ) - respx.get(f"{DEFAULT_BASE}/embeddings/models").mock( - return_value=_catalog("openai/text-embedding-3-small", "openai/gpt-5.4-mini") + return_value=_catalog("openai/gpt-5.4-mini:batch", "openai/gpt-5.4-mini", "qwen/qwen3-coder:free") ) assert OpenRouterModelInfo().get_models(api_key="sk-or-test") == [ "openrouter/openai/gpt-5.4-mini", - "openrouter/anthropic/claude-haiku-5-5", - "openrouter/openai/text-embedding-3-small", + "openrouter/qwen/qwen3-coder:free", ] +@pytest.mark.parametrize( + "api_base", + [ + "https://gateway.example/openrouter/api/v1/", + "https://gateway.example/openrouter/api/v1/chat/completions", + "https://gateway.example/openrouter/api/v1/chat/completions/", + ], +) @respx.mock -def test_get_models_keeps_the_api_base_path_and_sends_the_deployment_key(): +def test_get_models_keeps_the_api_base_path_and_sends_the_deployment_key(api_base: str): chat_route: Final = respx.get("https://gateway.example/openrouter/api/v1/models/user").mock( return_value=_catalog("openai/gpt-5.4-mini") ) - embeddings_route: Final = respx.get("https://gateway.example/openrouter/api/v1/embeddings/models").mock( - return_value=_catalog() - ) - assert OpenRouterModelInfo().get_models( - api_key="sk-or-deployment", api_base="https://gateway.example/openrouter/api/v1/" - ) == ["openrouter/openai/gpt-5.4-mini"] - assert [route.calls.last.request.headers["authorization"] for route in (chat_route, embeddings_route)] == [ - "Bearer sk-or-deployment", - "Bearer sk-or-deployment", + assert OpenRouterModelInfo().get_models(api_key="sk-or-deployment", api_base=api_base) == [ + "openrouter/openai/gpt-5.4-mini" ] + assert chat_route.calls.last.request.headers["authorization"] == "Bearer sk-or-deployment" @pytest.mark.parametrize("winner", range(len(KEY_SOURCES)), ids=KEY_SOURCES) @@ -76,7 +87,6 @@ def test_get_models_sends_the_first_configured_key_in_the_chat_paths_order( if winner > 0: _configure(monkeypatch, KEY_SOURCES[winner], "sk-or-winner") chat_route: Final = respx.get(f"{DEFAULT_BASE}/models/user").mock(return_value=_catalog("qwen/qwen3-coder")) - respx.get(f"{DEFAULT_BASE}/embeddings/models").mock(return_value=_catalog()) assert OpenRouterModelInfo().get_models(api_key="sk-or-winner" if winner == 0 else None) == [ "openrouter/qwen/qwen3-coder" @@ -95,7 +105,6 @@ def test_get_models_calls_the_first_configured_base_in_the_chat_paths_order( _configure(monkeypatch, BASE_SOURCES[winner], "https://winner.example/api/v1") expected_base: Final = "https://winner.example/api/v1" if winner < len(BASE_SOURCES) else DEFAULT_BASE respx.get(f"{expected_base}/models/user").mock(return_value=_catalog("qwen/qwen3-coder")) - respx.get(f"{expected_base}/embeddings/models").mock(return_value=_catalog()) assert OpenRouterModelInfo().get_models( api_key="sk-or-test", api_base="https://winner.example/api/v1" if winner == 0 else None @@ -114,17 +123,12 @@ def test_get_models_without_any_key_raises_before_calling_openrouter(): assert not chat_route.called -@pytest.mark.parametrize("failing_route", ["models/user", "embeddings/models"]) -def test_get_models_raises_when_either_catalog_answers_an_error(failing_route: str): - with respx.mock(assert_all_called=False) as respx_mock: - respx_mock.get(f"{DEFAULT_BASE}/models/user").mock(return_value=_catalog("openai/gpt-5.4-mini")) - respx_mock.get(f"{DEFAULT_BASE}/embeddings/models").mock(return_value=_catalog()) - respx_mock.get(f"{DEFAULT_BASE}/{failing_route}").mock( - return_value=httpx.Response(401, json={"error": {"code": 401}}) - ) +@respx.mock +def test_get_models_raises_when_the_catalog_answers_an_error(): + respx.get(f"{DEFAULT_BASE}/models/user").mock(return_value=httpx.Response(401, json={"error": {"code": 401}})) - with pytest.raises(httpx.HTTPStatusError): - OpenRouterModelInfo().get_models(api_key="sk-or-revoked") + with pytest.raises(httpx.HTTPStatusError): + OpenRouterModelInfo().get_models(api_key="sk-or-revoked") def test_validate_environment_adds_the_resolved_key_and_keeps_the_callers_headers(monkeypatch: pytest.MonkeyPatch): diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 549679e4054..0f310f8a0ad 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -1156,22 +1156,10 @@ def test_partial_bedrock_wildcard_lists_nothing_when_no_discovered_id_carries_it ( "openrouter/*", "openrouter/*", - [ - "openrouter/openai/gpt-5.4-mini", - "openrouter/anthropic/claude-haiku-5-5", - "openrouter/openai/text-embedding-3-small", - ], - ), - ( - "or-team/*", - "openrouter/*", - ["or-team/openai/gpt-5.4-mini", "or-team/anthropic/claude-haiku-5-5", "or-team/openai/text-embedding-3-small"], - ), - ( - "openrouter/openai/*", - "openrouter/openai/*", - ["openrouter/openai/gpt-5.4-mini", "openrouter/openai/text-embedding-3-small"], + ["openrouter/openai/gpt-5.4-mini", "openrouter/anthropic/claude-haiku-5-5"], ), + ("or-team/*", "openrouter/*", ["or-team/openai/gpt-5.4-mini", "or-team/anthropic/claude-haiku-5-5"]), + ("openrouter/openai/*", "openrouter/openai/*", ["openrouter/openai/gpt-5.4-mini"]), ], ) @respx.mock @@ -1189,9 +1177,6 @@ def test_openrouter_wildcard_lists_the_live_catalog_with_each_vendor_namespace( 200, json={"data": [{"id": "openai/gpt-5.4-mini"}, {"id": "anthropic/claude-haiku-5-5"}]} ) ) - respx.get("https://openrouter.ai/api/v1/embeddings/models").mock( - return_value=httpx.Response(200, json={"data": [{"id": "openai/text-embedding-3-small"}]}) - ) assert ( get_known_models_from_wildcard(wildcard, LiteLLM_Params(model=deployment_model, api_key=f"sk-or-{wildcard}")) diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py index f6c6b51fc93..dad8b1dc616 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py @@ -3,10 +3,13 @@ from types import MappingProxyType from typing import Final from unittest.mock import AsyncMock, MagicMock +import httpx import pytest +import respx from fastapi import HTTPException from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles +import litellm from litellm import Router from litellm.caching.caching import DualCache from litellm.proxy._types import UserAPIKeyAuth @@ -851,3 +854,65 @@ def test_a_team_keeps_its_first_config_file_openai_deployment_over_older_db_ones assert credentials is not None assert credentials["api_key"] == "deployment-z-key" + + +@pytest.mark.parametrize( + ("custom_llm_provider", "openrouter_key", "api_key"), + [("openai", "sk-or-files-openai", "openai-key"), ("openrouter", "sk-or-files-openrouter", "sk-or-files-openrouter")], +) +def test_a_provider_credential_lookup_never_lists_an_openrouter_wildcards_live_catalog( + monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, openrouter_key: str, api_key: str +): + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + router: Final = Router( + model_list=[ + { + "model_name": "openrouter/*", + "litellm_params": { + "model": "openrouter/*", + "api_key": openrouter_key, + "api_base": "https://openrouter.ai/api/v1", + }, + }, + {"model_name": "gpt-5.4-mini", "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "openai-key"}}, + ] + ) + + with respx.mock(assert_all_called=False) as respx_mock: + catalog_route: Final = respx_mock.get("https://openrouter.ai/api/v1/models/user").mock( + return_value=httpx.Response(200, json={"data": [{"id": "openai/gpt-5.4-mini"}]}) + ) + credentials: Final = get_team_provider_credentials( + llm_router=router, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", user_id="test-user"), + custom_llm_provider=custom_llm_provider, + include_accessible_deployments=True, + ) + + assert credentials is not None + assert (credentials["custom_llm_provider"], credentials["api_key"]) == (custom_llm_provider, api_key) + assert catalog_route.call_count == 0 + + +def test_a_key_allowed_a_wider_wildcard_than_any_deployment_gets_the_matching_deployments_credentials( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + router: Final = Router( + model_list=[ + { + "model_name": "anthropic/claude-*", + "litellm_params": {"model": "anthropic/claude-*", "api_key": "anthropic-deployment-key"}, + } + ] + ) + + credentials: Final = get_team_provider_credentials( + llm_router=router, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", user_id="test-user", models=["anthropic/*"]), + custom_llm_provider="anthropic", + include_accessible_deployments=True, + ) + + assert credentials is not None + assert (credentials["custom_llm_provider"], credentials["api_key"]) == ("anthropic", "anthropic-deployment-key") diff --git a/tests/unit/proxy/proxy_server/test_routes_models.py b/tests/unit/proxy/proxy_server/test_routes_models.py index 7be30c61f0c..01aa8722444 100644 --- a/tests/unit/proxy/proxy_server/test_routes_models.py +++ b/tests/unit/proxy/proxy_server/test_routes_models.py @@ -9,12 +9,19 @@ Pins (PR2): from __future__ import annotations +import asyncio +import threading +from dataclasses import dataclass, field +from typing import Final from unittest.mock import MagicMock +import httpx import pytest +import respx import litellm from litellm.proxy import proxy_server +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy import utils as proxy_utils from litellm.proxy.utils import create_model_info_response from litellm.types.router import DeploymentModelListingInfo @@ -514,3 +521,65 @@ def test_codex_service_tiers_resolved_via_internal_team_key(client, auth_as, pat assert response.status_code == 200 (entry,) = response.json()["models"] assert (entry["slug"], [tier["id"] for tier in entry["service_tiers"]]) == ("gpt-4-team", ["ultrafast"]) + + +OPENROUTER_BASE: Final = "https://openrouter.ai/api/v1" + + +@dataclass(frozen=True, slots=True) +class _CatalogHeldUntilTheLoopRuns: + requested: threading.Event = field(default_factory=threading.Event) + released: threading.Event = field(default_factory=threading.Event) + + def answer(self, _request: httpx.Request) -> httpx.Response: + self.requested.set() + if not self.released.wait(timeout=2): + return httpx.Response(504) + return httpx.Response(200, json={"data": [{"id": "openai/gpt-5.4-mini"}]}) + + async def release_from_the_loop(self) -> None: + assert await asyncio.to_thread(self.requested.wait, 5) + self.released.set() + + +@pytest.mark.parametrize("scope", [None, "expand"]) +async def test_v1_models_fetches_a_live_openrouter_catalog_without_blocking_the_event_loop( + monkeypatch: pytest.MonkeyPatch, scope: str | None +): + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "openrouter/*", + "litellm_params": { + "model": "openrouter/*", + "api_key": f"sk-or-v1-models-off-loop-{scope}", + "api_base": OPENROUTER_BASE, + }, + } + ] + ), + ) + catalog: Final = _CatalogHeldUntilTheLoopRuns() + + with respx.mock() as respx_mock: + catalog_route: Final = respx_mock.get(f"{OPENROUTER_BASE}/models/user").mock(side_effect=catalog.answer) + listing: Final = asyncio.ensure_future( + proxy_server.model_list( + user_api_key_dict=UserAPIKeyAuth( + api_key="sk-test-key", user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN + ), + scope=scope, + ) + ) + await catalog.release_from_the_loop() + response: Final = await listing + + assert catalog_route.calls.last.response.status_code == 200 + assert [model["id"] for model in response["data"]] == ["openrouter/openai/gpt-5.4-mini"] diff --git a/tests/unit/proxy/utils/helpers/test_model_access.py b/tests/unit/proxy/utils/helpers/test_model_access.py index ecf1810f34a..82e5dac5592 100644 --- a/tests/unit/proxy/utils/helpers/test_model_access.py +++ b/tests/unit/proxy/utils/helpers/test_model_access.py @@ -1,10 +1,16 @@ +import asyncio +import threading +from dataclasses import dataclass, field +from typing import Final from unittest.mock import MagicMock +import httpx import pytest +import respx from fastapi import HTTPException import litellm -from litellm import ModelResponse +from litellm import ModelResponse, Router from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import ( create_model_info_response, @@ -650,3 +656,57 @@ async def test_listing_is_unchanged_without_an_agent_or_without_attached_groups( resolve_agent_ceiling=_agent_ceiling(None), ) assert (plain_key, agent_without_groups) == (["model-a", "model-b"], ["model-a", "model-b"]) + + +OPENROUTER_BASE: Final = "https://openrouter.ai/api/v1" + + +@dataclass(frozen=True, slots=True) +class _CatalogHeldUntilTheLoopRuns: + requested: threading.Event = field(default_factory=threading.Event) + released: threading.Event = field(default_factory=threading.Event) + + def answer(self, _request: httpx.Request) -> httpx.Response: + self.requested.set() + if not self.released.wait(timeout=2): + return httpx.Response(504) + return httpx.Response(200, json={"data": [{"id": "openai/gpt-5.4-mini"}]}) + + async def release_from_the_loop(self) -> None: + assert await asyncio.to_thread(self.requested.wait, 5) + self.released.set() + + +async def test_get_available_models_for_user_fetches_a_live_catalog_without_blocking_the_event_loop( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + catalog: Final = _CatalogHeldUntilTheLoopRuns() + router: Final = Router( + model_list=[ + { + "model_name": "openrouter/*", + "litellm_params": { + "model": "openrouter/*", + "api_key": "sk-or-available-models-off-loop", + "api_base": OPENROUTER_BASE, + }, + } + ] + ) + + with respx.mock() as respx_mock: + catalog_route: Final = respx_mock.get(f"{OPENROUTER_BASE}/models/user").mock(side_effect=catalog.answer) + listing: Final = asyncio.ensure_future( + get_available_models_for_user( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test-key", user_id="user-1"), + llm_router=router, + general_settings={}, + user_model=None, + ) + ) + await catalog.release_from_the_loop() + models: Final = await listing + + assert catalog_route.calls.last.response.status_code == 200 + assert models == ["openrouter/openai/gpt-5.4-mini"] From a5bbe70588b165ab6b57d0efd769b7ae91ee8717 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:55:00 -0700 Subject: [PATCH 03/11] fix(proxy): keep model listing off the shared thread pool and out of credential lookups Listings run on their own thread pool so many slow catalogs cannot starve the default executor that chat and Responses calls share. File and batch credential lookups match deployed names against a key's wildcard grants instead of listing any provider catalog. The token counter keeps OpenRouter deployments on their previous path, and a catch-all "*" over provider/* lists bare ids that route back through it --- litellm/proxy/auth/model_checks.py | 30 +++++++--- .../openai_files_endpoints/common_utils.py | 54 ++++++++++-------- litellm/proxy/proxy_server.py | 55 ++++++++++-------- litellm/proxy/utils.py | 57 ++++++++++--------- tests/unit/proxy/auth/test_model_checks.py | 1 + .../test_files_common_utils.py | 50 +++++++++++----- .../proxy_server/test_routes_model_info.py | 12 ++-- .../proxy/proxy_server/test_routes_models.py | 13 +++-- tests/unit/proxy/test_proxy_token_counter.py | 33 +++++++++++ .../proxy/utils/helpers/test_model_access.py | 13 +++-- 10 files changed, 210 insertions(+), 108 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index fb97a7c95b7..e4ebf575bef 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -1,11 +1,14 @@ # What is this? ## Common checks for /v1/models and `/model/info` +import asyncio +import contextvars import copy import functools import itertools import re from collections.abc import Callable, Mapping, Sequence -from typing import Final +from concurrent.futures import ThreadPoolExecutor +from typing import Final, TypeVar import litellm from litellm._logging import verbose_proxy_logger @@ -25,6 +28,15 @@ _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) +_MODEL_LISTING_POOL: Final = ThreadPoolExecutor(thread_name_prefix="litellm-model-listing") + +_Listed: Final = TypeVar("_Listed") + + +async def run_model_listing(listing: Callable[[], _Listed]) -> _Listed: + context: Final = contextvars.copy_context() + return await asyncio.get_running_loop().run_in_executor(_MODEL_LISTING_POOL, context.run, listing) + def _check_wildcard_routing(model: str) -> bool: """ @@ -204,7 +216,7 @@ def get_complete_model_list( include_model_access_groups: bool | None = False, only_model_access_groups: bool | None = False, team_id: str | None = None, - expand_deployed_wildcard_routes: bool = True, + expand_wildcard_routes: bool = True, ) -> list[str]: """Logic for returning complete model list for a given key + team pair""" @@ -250,7 +262,7 @@ def get_complete_model_list( return_wildcard_routes=return_wildcard_routes, llm_router=llm_router, team_id=team_id, - expand_deployed_wildcard_routes=expand_deployed_wildcard_routes, + expand_wildcard_routes=expand_wildcard_routes, ) complete_model_list: Final = unique_models + all_wildcard_models @@ -418,7 +430,11 @@ def get_known_models_from_wildcard( wildcard_provider_prefix, wildcard_suffix = wildcard_model_to_expand.split("/", 1) except ValueError: # safely fail return [] - deployment_pattern_naming_listed_ids: Final = _deployment_pattern_naming_listed_ids(wildcard_suffix, litellm_params) + deployment_pattern_naming_listed_ids: Final = ( + wildcard_model_to_expand + if wildcard_model_to_expand != wildcard_model + else _deployment_pattern_naming_listed_ids(wildcard_suffix, litellm_params) + ) # Use provider from litellm_params when available, otherwise from wildcard prefix # (e.g., "openai" from "openai/*" - needed for BYOK where wildcard isn't in router) @@ -541,7 +557,7 @@ def _get_wildcard_models( return_wildcard_routes: bool | None = False, llm_router: Router | None = None, team_id: str | None = None, - expand_deployed_wildcard_routes: bool = True, + expand_wildcard_routes: bool = True, ) -> list[str]: models_to_remove: Final = set() routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id) @@ -552,10 +568,10 @@ def _get_wildcard_models( all_wildcard_models.append(model) models_to_remove.add(model) + if not expand_wildcard_routes: + continue model_list = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None - if model_list and not expand_deployed_wildcard_routes: - continue if model_list: for router_model in model_list: all_wildcard_models.extend( diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 6cbafd53537..3b701e644da 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -1,4 +1,5 @@ import base64 +import itertools import logging import mimetypes import re @@ -541,7 +542,8 @@ def get_team_provider_credentials( listings on the team's own provider account/key instead of a shared global one. 2. Fallback: any deployment the caller is granted access to for this - provider, expanding wildcard routes and the all-proxy-models sentinel. + provider, matching wildcard grants against deployed model names without + listing any provider's catalog, and expanding the all-proxy-models sentinel. Skipped when ``include_accessible_deployments`` is False. Credential lookup is scoped to both the team's allowlist and the key's own @@ -559,6 +561,7 @@ def get_team_provider_credentials( return None from litellm.proxy._types import SpecialModelNames + from litellm.proxy.auth.auth_checks import model_matches_any_wildcard_pattern_in_list from litellm.proxy.auth.model_checks import get_complete_model_list, get_key_models team_id: Final = user_api_key_dict.team_id @@ -644,29 +647,34 @@ def get_team_provider_credentials( grants_all_models: Final = SpecialModelNames.all_proxy_models.value in team_models effective_team_models: Final = [] if grants_all_models else team_models - models_to_try: Final = list( - dict.fromkeys( - get_complete_model_list( - key_models=list(key_model_allowlist), - team_models=effective_team_models, - proxy_model_list=proxy_model_list, - user_model=None, - infer_model_from_keys=False, - return_wildcard_routes=True, - llm_router=llm_router, - model_access_groups=model_access_groups, - include_model_access_groups=True, - team_id=team_id, - expand_deployed_wildcard_routes=False, - ) - ) + granted_models: Final = get_complete_model_list( + key_models=list(key_model_allowlist), + team_models=effective_team_models, + proxy_model_list=proxy_model_list, + user_model=None, + infer_model_from_keys=False, + return_wildcard_routes=True, + llm_router=llm_router, + model_access_groups=model_access_groups, + include_model_access_groups=True, + team_id=team_id, + expand_wildcard_routes=False, + ) + deployments_granted_by_wildcard: Final = ( + model_name + for model_name in proxy_model_list + if model_matches_any_wildcard_pattern_in_list(model=model_name, allowed_model_list=granted_models) + ) + return next( + ( + credentials + for credentials in map( + _provider_credentials, dict.fromkeys(itertools.chain(granted_models, deployments_granted_by_wildcard)) + ) + if credentials is not None + ), + None, ) - for model_name in models_to_try: - credentials = _provider_credentials(model_name) - if credentials is not None: - return credentials - - return None HTTP_HEADER_VALUE: Final = re.compile(r"[\x21-\x7e]+(?:[ \t]+[\x21-\x7e]+)*") diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 84b05b88a41..2442e90f475 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -193,6 +193,7 @@ from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.utils import ( PRICING_OVERRIDES_KEY, CountContextManagement, + LlmProviders, ModelResponse, ModelResponseStream, StreamingChoices, @@ -436,6 +437,7 @@ from litellm.proxy.auth.model_checks import ( get_key_models, get_mcp_server_ids, get_team_models, + run_model_listing, ) from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401, RUF100 # legacy module exports @@ -12398,18 +12400,20 @@ async def model_list( # Get complete model list including wildcard routes if requested from litellm.proxy.auth.model_checks import get_complete_model_list - all_models = await asyncio.to_thread( - get_complete_model_list, - key_models=[], - team_models=[], - proxy_model_list=proxy_model_list, - user_model=None, - infer_model_from_keys=False, - return_wildcard_routes=include_wildcard_routes, - llm_router=llm_router, - model_access_groups=model_access_groups, - include_model_access_groups=include_model_access_groups or False, - only_model_access_groups=only_model_access_groups or False, + all_models = await run_model_listing( + partial( + get_complete_model_list, + key_models=[], + team_models=[], + proxy_model_list=proxy_model_list, + user_model=None, + infer_model_from_keys=False, + return_wildcard_routes=include_wildcard_routes, + llm_router=llm_router, + model_access_groups=model_access_groups, + include_model_access_groups=include_model_access_groups or False, + only_model_access_groups=only_model_access_groups or False, + ) ) expanded_undiscoverable_names: Final = undiscoverable_model_names( @@ -14649,6 +14653,8 @@ from litellm.responses.litellm_completion_transformation.count_request import ( with_provider_file_ids, ) +_PROVIDERS_WITH_LISTING_ONLY_MODEL_INFO: Final = frozenset({LlmProviders.OPENROUTER}) + def _get_provider_token_counter( deployment: dict, model_to_use: str @@ -14676,14 +14682,13 @@ def _get_provider_token_counter( ) # Switch case pattern using existing get_provider_model_info - from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager # Convert string provider to LlmProviders enum llm_provider_enum: Final = LlmProviders(custom_llm_provider) # Add more provider mappings as needed - if llm_provider_enum: + if llm_provider_enum not in _PROVIDERS_WITH_LISTING_ONLY_MODEL_INFO: provider_model_info: Final = ProviderConfigManager.get_provider_model_info( model=full_model, provider=llm_provider_enum ) @@ -17752,7 +17757,7 @@ async def model_info_v1( alias_models: Final = copy.deepcopy(llm_router.get_model_list_from_model_alias()) all_models.extend(alias_models) - all_models = await asyncio.to_thread(expand_wildcard_deployments_for_model_info, all_models, llm_router) + all_models = await run_model_listing(partial(expand_wildcard_deployments_for_model_info, all_models, llm_router)) allowed_model_names: Final = await _get_v1_model_info_allowed_model_names_with_access_groups( user_api_key_dict=user_api_key_dict, @@ -18087,15 +18092,17 @@ async def model_group_info( LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, ) all_models_str: Final = ( - await asyncio.to_thread( - get_complete_model_list, - key_models=(), - team_models=(), - proxy_model_list=llm_router.get_model_names(), - user_model=user_model, - infer_model_from_keys=general_settings.get("infer_model_from_keys", False), - return_wildcard_routes=False, - llm_router=llm_router, + await run_model_listing( + partial( + get_complete_model_list, + key_models=(), + team_models=(), + proxy_model_list=llm_router.get_model_names(), + user_model=user_model, + infer_model_from_keys=general_settings.get("infer_model_from_keys", False), + return_wildcard_routes=False, + llm_router=llm_router, + ) ) if is_proxy_admin else await get_available_models_for_user( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index cc3c7280160..a58f3f7a58f 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -9497,7 +9497,7 @@ async def _agent_access_group_visible_models( ) -> frozenset[str] | None: """Models an agent key may still list once its attached access groups cap it, ``None`` when nothing caps it, so ``/v1/models`` never advertises a model the same key would be denied on.""" - from litellm.proxy.auth.model_checks import get_complete_model_list, get_team_models + from litellm.proxy.auth.model_checks import get_complete_model_list, get_team_models, run_model_listing if not user_api_key_dict.agent_id: return None @@ -9517,18 +9517,20 @@ async def _agent_access_group_visible_models( if not granted: return frozenset() return frozenset( - await asyncio.to_thread( - get_complete_model_list, - key_models=granted, - team_models=(), - proxy_model_list=proxy_model_list, - user_model=None, - infer_model_from_keys=False, - return_wildcard_routes=return_wildcard_routes, - llm_router=llm_router, - model_access_groups=model_access_groups, - include_model_access_groups=include_model_access_groups, - team_id=team_id, + await run_model_listing( + partial( + get_complete_model_list, + key_models=granted, + team_models=(), + proxy_model_list=proxy_model_list, + user_model=None, + infer_model_from_keys=False, + return_wildcard_routes=return_wildcard_routes, + llm_router=llm_router, + model_access_groups=model_access_groups, + include_model_access_groups=include_model_access_groups, + team_id=team_id, + ) ) ) @@ -9573,6 +9575,7 @@ async def get_available_models_for_user( get_complete_model_list, get_key_models, get_team_models, + run_model_listing, ) # Get proxy model list and access groups @@ -9653,19 +9656,21 @@ async def get_available_models_for_user( ) # Get complete model list - complete_models: Final = await asyncio.to_thread( - get_complete_model_list, - key_models=granted_key_models, - team_models=granted_team_models, - proxy_model_list=proxy_model_list, - user_model=user_model, - infer_model_from_keys=general_settings.get("infer_model_from_keys", False), - return_wildcard_routes=return_wildcard_routes, - llm_router=llm_router, - model_access_groups=model_access_groups, - include_model_access_groups=include_model_access_groups, - only_model_access_groups=only_model_access_groups, - team_id=effective_team_id, + complete_models: Final = await run_model_listing( + partial( + get_complete_model_list, + key_models=granted_key_models, + team_models=granted_team_models, + proxy_model_list=proxy_model_list, + user_model=user_model, + infer_model_from_keys=general_settings.get("infer_model_from_keys", False), + return_wildcard_routes=return_wildcard_routes, + llm_router=llm_router, + model_access_groups=model_access_groups, + include_model_access_groups=include_model_access_groups, + only_model_access_groups=only_model_access_groups, + team_id=effective_team_id, + ) ) all_models: Final = ( list( diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 0f310f8a0ad..131a80cd9ce 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -1160,6 +1160,7 @@ def test_partial_bedrock_wildcard_lists_nothing_when_no_discovered_id_carries_it ), ("or-team/*", "openrouter/*", ["or-team/openai/gpt-5.4-mini", "or-team/anthropic/claude-haiku-5-5"]), ("openrouter/openai/*", "openrouter/openai/*", ["openrouter/openai/gpt-5.4-mini"]), + ("*", "openrouter/*", ["openai/gpt-5.4-mini", "anthropic/claude-haiku-5-5"]), ], ) @respx.mock diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py index dad8b1dc616..b6a463bb783 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py @@ -894,25 +894,49 @@ def test_a_provider_credential_lookup_never_lists_an_openrouter_wildcards_live_c assert catalog_route.call_count == 0 -def test_a_key_allowed_a_wider_wildcard_than_any_deployment_gets_the_matching_deployments_credentials( +@pytest.mark.parametrize( + ("key_models", "deployments", "custom_llm_provider", "api_key"), + [ + ( + ["anthropic/*"], + (("anthropic/claude-*", "anthropic/claude-*", "anthropic-narrow-key"),), + "anthropic", + "anthropic-narrow-key", + ), + (["anthropic/claude-3*"], (("anthropic/*", "anthropic/*", "anthropic-wide-key"),), "anthropic", "anthropic-wide-key"), + (["openrouter/*"], (("openrouter/openai/*", "openrouter/openai/*", "sk-or-narrow"),), "openrouter", "sk-or-narrow"), + ( + ["team/*"], + (("team/*", "openrouter/*", "sk-or-team"), ("team/openai/*", "openai/*", "openai-team-key")), + "openai", + "openai-team-key", + ), + ], +) +def test_a_key_wildcard_finds_the_deployment_it_grants_without_listing_any_provider_catalog( monkeypatch: pytest.MonkeyPatch, + key_models: list[str], + deployments: tuple[tuple[str, str, str], ...], + custom_llm_provider: str, + api_key: str, ): - monkeypatch.setattr(litellm, "check_provider_endpoint", False) + monkeypatch.setattr(litellm, "check_provider_endpoint", True) router: Final = Router( model_list=[ - { - "model_name": "anthropic/claude-*", - "litellm_params": {"model": "anthropic/claude-*", "api_key": "anthropic-deployment-key"}, - } + {"model_name": model_name, "litellm_params": {"model": model, "api_key": deployment_key}} + for model_name, model, deployment_key in deployments ] ) - credentials: Final = get_team_provider_credentials( - llm_router=router, - user_api_key_dict=UserAPIKeyAuth(api_key="test-key", user_id="test-user", models=["anthropic/*"]), - custom_llm_provider="anthropic", - include_accessible_deployments=True, - ) + with respx.mock(assert_all_called=False) as respx_mock: + catalog_requests: Final = respx_mock.route().mock(return_value=httpx.Response(200, json={"data": []})) + credentials: Final = get_team_provider_credentials( + llm_router=router, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", user_id="test-user", models=key_models), + custom_llm_provider=custom_llm_provider, + include_accessible_deployments=True, + ) assert credentials is not None - assert (credentials["custom_llm_provider"], credentials["api_key"]) == ("anthropic", "anthropic-deployment-key") + assert (credentials["custom_llm_provider"], credentials["api_key"]) == (custom_llm_provider, api_key) + assert catalog_requests.call_count == 0 diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 9993361d930..45a5c845981 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -550,15 +550,12 @@ async def test_v1_model_info_star_wildcard_filter_keeps_provider_expansion(monke deployment = { "model_name": "*", - "litellm_params": {"model": "openai/*"}, + "litellm_params": {"model": "openai/*", "api_key": "sk-star"}, } - router = MagicMock() - router.get_model_access_groups = MagicMock(return_value={}) - router.get_model_names = MagicMock(return_value=["*"]) - router.get_model_list = MagicMock(return_value=[deployment]) + router = litellm.Router(model_list=[deployment]) monkeypatch.setattr(model_checks, "get_provider_models", fake_get_provider_models) - expanded_deployments = proxy_server.expand_wildcard_deployments_for_model_info([deployment]) + expanded_deployments = proxy_server.expand_wildcard_deployments_for_model_info(router.get_model_list(), router) allowed_model_names = await proxy_server._get_v1_model_info_allowed_model_names_with_access_groups( user_api_key_dict=UserAPIKeyAuth( api_key="sk-test", @@ -575,7 +572,8 @@ async def test_v1_model_info_star_wildcard_filter_keeps_provider_expansion(monke allowed_model_names=allowed_model_names, ) - assert [model["model_name"] for model in result] == ["openai/gpt-4o"] + assert [model["model_name"] for model in result] == ["gpt-4o"] + assert router.wildcard_routed_models("gpt-4o", None) == frozenset({"openai/gpt-4o"}) # --------------------------------------------------------------------------- diff --git a/tests/unit/proxy/proxy_server/test_routes_models.py b/tests/unit/proxy/proxy_server/test_routes_models.py index 01aa8722444..ab8a9ddce13 100644 --- a/tests/unit/proxy/proxy_server/test_routes_models.py +++ b/tests/unit/proxy/proxy_server/test_routes_models.py @@ -11,6 +11,7 @@ from __future__ import annotations import asyncio import threading +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from typing import Final from unittest.mock import MagicMock @@ -543,7 +544,7 @@ class _CatalogHeldUntilTheLoopRuns: @pytest.mark.parametrize("scope", [None, "expand"]) -async def test_v1_models_fetches_a_live_openrouter_catalog_without_blocking_the_event_loop( +def test_v1_models_fetches_a_live_openrouter_catalog_without_holding_the_event_loop_or_its_default_thread_pool( monkeypatch: pytest.MonkeyPatch, scope: str | None ): monkeypatch.setattr(litellm, "check_provider_endpoint", True) @@ -568,8 +569,8 @@ async def test_v1_models_fetches_a_live_openrouter_catalog_without_blocking_the_ ) catalog: Final = _CatalogHeldUntilTheLoopRuns() - with respx.mock() as respx_mock: - catalog_route: Final = respx_mock.get(f"{OPENROUTER_BASE}/models/user").mock(side_effect=catalog.answer) + async def list_models_on_a_one_thread_default_pool() -> dict: + asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(max_workers=1)) listing: Final = asyncio.ensure_future( proxy_server.model_list( user_api_key_dict=UserAPIKeyAuth( @@ -579,7 +580,11 @@ async def test_v1_models_fetches_a_live_openrouter_catalog_without_blocking_the_ ) ) await catalog.release_from_the_loop() - response: Final = await listing + return await listing + + with respx.mock() as respx_mock: + catalog_route: Final = respx_mock.get(f"{OPENROUTER_BASE}/models/user").mock(side_effect=catalog.answer) + response: Final = asyncio.run(list_models_on_a_one_thread_default_pool()) assert catalog_route.calls.last.response.status_code == 200 assert [model["id"] for model in response["data"]] == ["openrouter/openai/gpt-5.4-mini"] diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index fa0c9eda02d..fc0d9a2b697 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -175,6 +175,39 @@ async def test_token_counting_model_not_in_model_list(): assert response.model_used == "special-alias" +@pytest.mark.asyncio +async def test_asking_the_provider_endpoint_leaves_an_openrouter_deployments_count_unchanged( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "or-gpt", + "litellm_params": { + "model": "openai/gpt-4o", + "custom_llm_provider": "openrouter", + "api_key": "sk-or-token-count", + }, + } + ] + ), + ) + request: Final = TokenCountRequest( + model="or-gpt", messages=[{"role": "user", "content": "Count these tokens, ünïcödé 你好 🙂"}] + ) + + via_provider_endpoint: Final = await token_counter(request=request, call_endpoint=True) + counted_locally: Final = await token_counter(request=request, call_endpoint=False) + + assert (via_provider_endpoint.total_tokens, via_provider_endpoint.model_used) == ( + counted_locally.total_tokens, + counted_locally.model_used, + ) + + @pytest.mark.asyncio async def test_gpt_token_counting(): """ diff --git a/tests/unit/proxy/utils/helpers/test_model_access.py b/tests/unit/proxy/utils/helpers/test_model_access.py index 82e5dac5592..94d532dd511 100644 --- a/tests/unit/proxy/utils/helpers/test_model_access.py +++ b/tests/unit/proxy/utils/helpers/test_model_access.py @@ -1,5 +1,6 @@ import asyncio import threading +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from typing import Final from unittest.mock import MagicMock @@ -677,7 +678,7 @@ class _CatalogHeldUntilTheLoopRuns: self.released.set() -async def test_get_available_models_for_user_fetches_a_live_catalog_without_blocking_the_event_loop( +def test_get_available_models_for_user_fetches_a_live_catalog_without_holding_the_event_loop_or_its_default_pool( monkeypatch: pytest.MonkeyPatch, ): monkeypatch.setattr(litellm, "check_provider_endpoint", True) @@ -695,8 +696,8 @@ async def test_get_available_models_for_user_fetches_a_live_catalog_without_bloc ] ) - with respx.mock() as respx_mock: - catalog_route: Final = respx_mock.get(f"{OPENROUTER_BASE}/models/user").mock(side_effect=catalog.answer) + async def list_models_on_a_one_thread_default_pool() -> list[str]: + asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(max_workers=1)) listing: Final = asyncio.ensure_future( get_available_models_for_user( user_api_key_dict=UserAPIKeyAuth(api_key="sk-test-key", user_id="user-1"), @@ -706,7 +707,11 @@ async def test_get_available_models_for_user_fetches_a_live_catalog_without_bloc ) ) await catalog.release_from_the_loop() - models: Final = await listing + return await listing + + with respx.mock() as respx_mock: + catalog_route: Final = respx_mock.get(f"{OPENROUTER_BASE}/models/user").mock(side_effect=catalog.answer) + models: Final = asyncio.run(list_models_on_a_one_thread_default_pool()) assert catalog_route.calls.last.response.status_code == 200 assert models == ["openrouter/openai/gpt-5.4-mini"] From 1d5caf0b1f005f3ea7ad55b76d7bbbd11ec15edf Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:26:33 -0700 Subject: [PATCH 04/11] test(proxy): give the off-loop listing tests a generous deadlock guard and typed ids --- tests/unit/proxy/proxy_server/test_routes_models.py | 13 +++++++------ tests/unit/proxy/utils/helpers/test_model_access.py | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/unit/proxy/proxy_server/test_routes_models.py b/tests/unit/proxy/proxy_server/test_routes_models.py index ab8a9ddce13..200448b1326 100644 --- a/tests/unit/proxy/proxy_server/test_routes_models.py +++ b/tests/unit/proxy/proxy_server/test_routes_models.py @@ -534,12 +534,12 @@ class _CatalogHeldUntilTheLoopRuns: def answer(self, _request: httpx.Request) -> httpx.Response: self.requested.set() - if not self.released.wait(timeout=2): + if not self.released.wait(timeout=30): return httpx.Response(504) return httpx.Response(200, json={"data": [{"id": "openai/gpt-5.4-mini"}]}) async def release_from_the_loop(self) -> None: - assert await asyncio.to_thread(self.requested.wait, 5) + assert await asyncio.to_thread(self.requested.wait, 30) self.released.set() @@ -569,7 +569,7 @@ def test_v1_models_fetches_a_live_openrouter_catalog_without_holding_the_event_l ) catalog: Final = _CatalogHeldUntilTheLoopRuns() - async def list_models_on_a_one_thread_default_pool() -> dict: + async def listed_ids_on_a_one_thread_default_pool() -> list[str]: asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(max_workers=1)) listing: Final = asyncio.ensure_future( proxy_server.model_list( @@ -580,11 +580,12 @@ def test_v1_models_fetches_a_live_openrouter_catalog_without_holding_the_event_l ) ) await catalog.release_from_the_loop() - return await listing + response: Final = await listing + return [str(model["id"]) for model in response["data"]] with respx.mock() as respx_mock: catalog_route: Final = respx_mock.get(f"{OPENROUTER_BASE}/models/user").mock(side_effect=catalog.answer) - response: Final = asyncio.run(list_models_on_a_one_thread_default_pool()) + listed_ids: Final = asyncio.run(listed_ids_on_a_one_thread_default_pool()) assert catalog_route.calls.last.response.status_code == 200 - assert [model["id"] for model in response["data"]] == ["openrouter/openai/gpt-5.4-mini"] + assert listed_ids == ["openrouter/openai/gpt-5.4-mini"] diff --git a/tests/unit/proxy/utils/helpers/test_model_access.py b/tests/unit/proxy/utils/helpers/test_model_access.py index 94d532dd511..9b9b440b11c 100644 --- a/tests/unit/proxy/utils/helpers/test_model_access.py +++ b/tests/unit/proxy/utils/helpers/test_model_access.py @@ -669,12 +669,12 @@ class _CatalogHeldUntilTheLoopRuns: def answer(self, _request: httpx.Request) -> httpx.Response: self.requested.set() - if not self.released.wait(timeout=2): + if not self.released.wait(timeout=30): return httpx.Response(504) return httpx.Response(200, json={"data": [{"id": "openai/gpt-5.4-mini"}]}) async def release_from_the_loop(self) -> None: - assert await asyncio.to_thread(self.requested.wait, 5) + assert await asyncio.to_thread(self.requested.wait, 30) self.released.set() From 99f6fa13641865a33dff25de4f0ae3478ff7467c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:59:45 -0700 Subject: [PATCH 05/11] fix(proxy): keep routed models in wildcard model info, skip key-only OpenRouter catalogs, cache failed listings briefly --- litellm/proxy/auth/model_checks.py | 26 +++++++++++++++-- litellm/utils.py | 8 ++++- tests/unit/proxy/auth/test_model_checks.py | 29 +++++++++++++++++++ .../proxy_server/test_routes_model_info.py | 2 +- tests/unit/test_utils.py | 13 +++++++++ 5 files changed, 74 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index e4ebf575bef..767d7e362b2 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -30,6 +30,8 @@ _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) _MODEL_LISTING_POOL: Final = ThreadPoolExecutor(thread_name_prefix="litellm-model-listing") +_PROVIDERS_LISTED_ONLY_THROUGH_A_DEPLOYMENT: Final = frozenset({LlmProviders.OPENROUTER.value}) + _Listed: Final = TypeVar("_Listed") @@ -495,6 +497,20 @@ def get_known_models_from_wildcard( return list(dict.fromkeys(suffix_appended_wildcard_models)) +@functools.lru_cache(maxsize=256) +def _wildcard_name_regex(name_pattern: str) -> re.Pattern[str]: + return re.compile(PatternMatchRouter().pattern_to_regex(name_pattern)) + + +def _routed_deployment_model(name_pattern: str, listed_name: str, deployment_model: str) -> str: + name_match: Final = _wildcard_name_regex(name_pattern).match(listed_name) + if name_match is None: + return listed_name + return PatternMatchRouter.set_deployment_model_name( + matched_pattern=name_match, litellm_deployment_litellm_model=deployment_model + ) + + def expand_wildcard_deployments_for_model_info( deployments: list[dict[str, object]], llm_router: Router | None = None, @@ -546,12 +562,18 @@ def expand_wildcard_deployments_for_model_info( row["model_name"] = name params = row.get("litellm_params") if isinstance(params, dict): - params["model"] = name + params["model"] = _routed_deployment_model(model_name, name, litellm_model) expanded.append(row) return expanded +def _known_models_without_a_deployment(wildcard_model: str) -> Sequence[str]: + if wildcard_model.partition("/")[0] in _PROVIDERS_LISTED_ONLY_THROUGH_A_DEPLOYMENT: + return () + return get_known_models_from_wildcard(wildcard_model=wildcard_model, litellm_params=None) + + def _get_wildcard_models( unique_models: list[str], return_wildcard_routes: bool | None = False, @@ -582,7 +604,7 @@ def _get_wildcard_models( ) ) else: - all_wildcard_models.extend(get_known_models_from_wildcard(wildcard_model=model, litellm_params=None)) + all_wildcard_models.extend(_known_models_without_a_deployment(model)) for model in models_to_remove: unique_models.remove(model) diff --git a/litellm/utils.py b/litellm/utils.py index 0719ff8bcde..9355a115e30 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8386,15 +8386,18 @@ class AvailableModelsCache(InMemoryCache): custom_llm_provider: str, litellm_params: LiteLLM_Params | None, available_models: list[str], + ttl: float | None = None, ): """Set cached model info""" cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params) - self.set_cache(cache_key, copy.deepcopy(available_models)) + self.set_cache(cache_key, copy.deepcopy(available_models), ttl=ttl) # Global cache instance _model_cache: Final = AvailableModelsCache() +_FAILED_PROVIDER_LISTING_TTL_SECONDS: Final = 30 + def _infer_valid_provider_from_env_vars( custom_llm_provider: str | None = None, @@ -8455,6 +8458,9 @@ def _get_valid_models_from_provider_api( return models except Exception as e: verbose_logger.warning("Error getting valid models: %s", e) + _model_cache.set_cached_model_info( + custom_llm_provider, litellm_params, [], ttl=_FAILED_PROVIDER_LISTING_TTL_SECONDS + ) return [] diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 131a80cd9ce..95cc1a06fa9 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -1185,6 +1185,35 @@ def test_openrouter_wildcard_lists_the_live_catalog_with_each_vendor_namespace( ) +@respx.mock +def test_key_granted_openrouter_wildcard_without_an_openrouter_deployment_lists_no_environment_catalog( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm import Router + from litellm.proxy.auth.model_checks import get_complete_model_list + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-environment-only") + catalog: Final = respx.get("https://openrouter.ai/api/v1/models/user").mock( + return_value=httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]}) + ) + router: Final = Router( + model_list=[{"model_name": "or-team/*", "litellm_params": {"model": "openrouter/*", "api_key": "sk-or-team"}}] + ) + + listed: Final = get_complete_model_list( + key_models=["openrouter/*"], + team_models=[], + proxy_model_list=router.model_names, + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + + assert (listed, catalog.call_count) == ([], 0) + + @respx.mock def test_custom_prefix_that_starts_a_discovered_id_still_prefixes_every_listed_model( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 45a5c845981..74a54105a8f 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -572,7 +572,7 @@ async def test_v1_model_info_star_wildcard_filter_keeps_provider_expansion(monke allowed_model_names=allowed_model_names, ) - assert [model["model_name"] for model in result] == ["gpt-4o"] + assert [(model["model_name"], model["litellm_params"]["model"]) for model in result] == [("gpt-4o", "openai/gpt-4o")] assert router.wildcard_routed_models("gpt-4o", None) == frozenset({"openai/gpt-4o"}) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 38287b25169..0c4a7b1eb93 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -8895,6 +8895,19 @@ def reset_mock_cache() -> None: _model_cache.flush_cache() +@respx.mock +def test_get_valid_models_does_not_ask_a_failing_provider_again_on_the_next_listing(reset_mock_cache: None) -> None: + catalog: Final = respx.get("https://openrouter.ai/api/v1/models/user").mock(return_value=httpx.Response(503)) + deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-during-an-outage") + + listings: Final = tuple( + get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment) + for _ in range(2) + ) + + assert (listings, catalog.call_count) == (([], []), 1) + + @pytest.fixture def isolated_openai_model_sets(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( From adf69a87c4976ad41172bb312dba954a085687c8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:29:40 -0700 Subject: [PATCH 06/11] fix(utils): cache failed model listings under their own key so a late failure never hides a good listing --- litellm/utils.py | 33 +++++++++++++++++++++++++-------- tests/unit/test_utils.py | 22 ++++++++++++++++++++++ 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 9355a115e30..19bffb8c0b6 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8375,22 +8375,39 @@ class AvailableModelsCache(InMemoryCache): cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params) - result: Final = cast(list[str] | None, self.get_cache(cache_key)) + result: Final = self._cached(cache_key) - if result is not None: - return copy.deepcopy(result) - return result + if isinstance(result, Sequence): + return list(result) + if self._cached(_failed_listing_key(cache_key)) is not None: + return [] + return None def set_cached_model_info( self, custom_llm_provider: str, litellm_params: LiteLLM_Params | None, available_models: list[str], - ttl: float | None = None, ): """Set cached model info""" cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params) - self.set_cache(cache_key, copy.deepcopy(available_models), ttl=ttl) + self._store(cache_key, copy.deepcopy(available_models)) + + def set_failed_model_listing( + self, custom_llm_provider: str, litellm_params: LiteLLM_Params | None, ttl_seconds: float + ) -> None: + cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params) + self._store(_failed_listing_key(cache_key), True, ttl=ttl_seconds) + + def _cached(self, key: str) -> Sequence[str] | bool | None: + return cast(Sequence[str] | bool | None, self.get_cache(key)) + + def _store(self, key: str, value: object, ttl: float | None = None) -> None: + self.set_cache(key, value, ttl=ttl) + + +def _failed_listing_key(cache_key: str) -> str: + return f"{cache_key}:failed" # Global cache instance @@ -8458,8 +8475,8 @@ def _get_valid_models_from_provider_api( return models except Exception as e: verbose_logger.warning("Error getting valid models: %s", e) - _model_cache.set_cached_model_info( - custom_llm_provider, litellm_params, [], ttl=_FAILED_PROVIDER_LISTING_TTL_SECONDS + _model_cache.set_failed_model_listing( + custom_llm_provider, litellm_params, ttl_seconds=_FAILED_PROVIDER_LISTING_TTL_SECONDS ) return [] diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 0c4a7b1eb93..55b8f1a4515 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -8908,6 +8908,28 @@ def test_get_valid_models_does_not_ask_a_failing_provider_again_on_the_next_list assert (listings, catalog.call_count) == (([], []), 1) +@respx.mock +def test_get_valid_models_keeps_a_good_listing_when_an_overlapping_listing_fails_after_it( + reset_mock_cache: None, +) -> None: + deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-overlapping-listings") + statuses: Final = iter((503, 200)) + + def answer_after_an_overlapping_listing(request: httpx.Request) -> httpx.Response: + status: Final = next(statuses) + if status == 503: + get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment) + return httpx.Response(503) + return httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]}) + + respx.get("https://openrouter.ai/api/v1/models/user").mock(side_effect=answer_after_an_overlapping_listing) + get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment) + + assert get_valid_models( + check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment + ) == ["openrouter/anthropic/claude-haiku-5-5"] + + @pytest.fixture def isolated_openai_model_sets(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( From aabeffc65286145db06ced924a4f64fd87d5f6db Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:43:27 -0700 Subject: [PATCH 07/11] fix(proxy): remember only timed-out provider listings so a provider that failed fast is listed again --- litellm/utils.py | 7 +++++-- tests/unit/test_utils.py | 28 ++++++++++++++++++++++------ 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index dde5aab764b..e1ec7bfcb6b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8506,12 +8506,15 @@ def _get_valid_models_from_provider_api( _model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) return models - except Exception as e: - verbose_logger.warning("Error getting valid models: %s", e) + except httpx.TimeoutException as e: + verbose_logger.warning("Timed out getting valid models: %s", e) _model_cache.set_failed_model_listing( custom_llm_provider, litellm_params, ttl_seconds=_FAILED_PROVIDER_LISTING_TTL_SECONDS ) return [] + except Exception as e: + verbose_logger.warning("Error getting valid models: %s", e) + return [] def get_valid_models( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index dcc05f19efb..02aaff287cc 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -8898,8 +8898,10 @@ def reset_mock_cache() -> None: @respx.mock -def test_get_valid_models_does_not_ask_a_failing_provider_again_on_the_next_listing(reset_mock_cache: None) -> None: - catalog: Final = respx.get("https://openrouter.ai/api/v1/models/user").mock(return_value=httpx.Response(503)) +def test_get_valid_models_does_not_ask_a_timed_out_provider_again_on_the_next_listing(reset_mock_cache: None) -> None: + catalog: Final = respx.get("https://openrouter.ai/api/v1/models/user").mock( + side_effect=httpx.ReadTimeout("catalog hung") + ) deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-during-an-outage") listings: Final = tuple( @@ -8910,18 +8912,32 @@ def test_get_valid_models_does_not_ask_a_failing_provider_again_on_the_next_list assert (listings, catalog.call_count) == (([], []), 1) +@respx.mock +def test_get_valid_models_lists_a_provider_again_right_after_a_fast_failure(reset_mock_cache: None) -> None: + respx.get("https://openrouter.ai/api/v1/models/user").mock( + side_effect=(httpx.Response(503), httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]})) + ) + deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-recovers") + + listings: Final = tuple( + get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment) + for _ in range(2) + ) + + assert listings == ([], ["openrouter/anthropic/claude-haiku-5-5"]) + + @respx.mock def test_get_valid_models_keeps_a_good_listing_when_an_overlapping_listing_fails_after_it( reset_mock_cache: None, ) -> None: deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-overlapping-listings") - statuses: Final = iter((503, 200)) + first_request: Final = iter((True, False)) def answer_after_an_overlapping_listing(request: httpx.Request) -> httpx.Response: - status: Final = next(statuses) - if status == 503: + if next(first_request): get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment) - return httpx.Response(503) + raise httpx.ReadTimeout("catalog hung", request=request) return httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]}) respx.get("https://openrouter.ai/api/v1/models/user").mock(side_effect=answer_after_an_overlapping_listing) From b786c5514de4da60490b54a5729a0ad8ea6b4035 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:13:42 -0700 Subject: [PATCH 08/11] fix(proxy): ask each provider catalog once per worker, keep deployed names out of catch-all ids, and match file credentials by name Listings that start together for one deployment now wait for the first catalog request. A catch-all wildcard no longer lists a name a deployment serves by name. A key wildcard only finds file and batch credentials on deployments named under it, so a bare name keeps the default credentials as before. --- litellm/proxy/auth/model_checks.py | 13 +++- .../openai_files_endpoints/common_utils.py | 6 +- litellm/utils.py | 25 +++++++ tests/unit/proxy/auth/test_model_checks.py | 69 +++++++++++++++++++ .../test_files_common_utils.py | 23 +++++++ tests/unit/test_utils.py | 20 ++++++ 6 files changed, 153 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 767d7e362b2..6d683bfa94e 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -399,7 +399,18 @@ def _deployment_team_id(deployment: Mapping[str, object]) -> str | None: def _cached_wildcard_routing(llm_router: Router | None) -> Callable[[str, str | None], frozenset[str]] | None: - return None if llm_router is None else functools.cache(llm_router.wildcard_routed_models) + if llm_router is None: + return None + deployed_names: Final = frozenset(llm_router.model_names) + team_deployed_names: Final = frozenset(llm_router.team_model_to_deployment_indices) + wildcard_routed_models: Final = llm_router.wildcard_routed_models + + def routed_models(model: str, team_id: str | None) -> frozenset[str]: + if model in deployed_names or (team_id, model) in team_deployed_names: + return frozenset() + return wildcard_routed_models(model, team_id) + + return functools.cache(routed_models) def _routes_to_model( diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 3b701e644da..e444687ecd1 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -561,7 +561,7 @@ def get_team_provider_credentials( return None from litellm.proxy._types import SpecialModelNames - from litellm.proxy.auth.auth_checks import model_matches_any_wildcard_pattern_in_list + from litellm.proxy.auth.auth_checks import is_model_allowed_by_pattern from litellm.proxy.auth.model_checks import get_complete_model_list, get_key_models team_id: Final = user_api_key_dict.team_id @@ -663,7 +663,9 @@ def get_team_provider_credentials( deployments_granted_by_wildcard: Final = ( model_name for model_name in proxy_model_list - if model_matches_any_wildcard_pattern_in_list(model=model_name, allowed_model_list=granted_models) + if any( + is_model_allowed_by_pattern(model=model_name, allowed_model_pattern=granted) for granted in granted_models + ) ) return next( ( diff --git a/litellm/utils.py b/litellm/utils.py index e1ec7bfcb6b..cda028fabb9 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8432,6 +8432,9 @@ class AvailableModelsCache(InMemoryCache): cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params) self._store(_failed_listing_key(cache_key), True, ttl=ttl_seconds) + def listing_lock(self, custom_llm_provider: str, litellm_params: LiteLLM_Params | None) -> threading.RLock: + return _provider_listing_lock(self._get_cache_key(custom_llm_provider, litellm_params)) + def _cached(self, key: str) -> Sequence[str] | bool | None: return cast(Sequence[str] | bool | None, self.get_cache(key)) @@ -8443,6 +8446,19 @@ def _failed_listing_key(cache_key: str) -> str: return f"{cache_key}:failed" +_PROVIDER_LISTING_LOCK_GUARD: Final = threading.Lock() + + +@lru_cache(maxsize=1024) +def _cached_provider_listing_lock(cache_key: str) -> threading.RLock: + return threading.RLock() + + +def _provider_listing_lock(cache_key: str) -> threading.RLock: + with _PROVIDER_LISTING_LOCK_GUARD: + return _cached_provider_listing_lock(cache_key) + + # Global cache instance _model_cache: Final = AvailableModelsCache() @@ -8494,6 +8510,15 @@ def _get_valid_models_from_provider_api( provider_config: BaseLLMModelInfo, custom_llm_provider: str, litellm_params: LiteLLM_Params | None = None, +) -> list[str]: + with _model_cache.listing_lock(custom_llm_provider, litellm_params): + return _cached_or_listed_provider_models(provider_config, custom_llm_provider, litellm_params) + + +def _cached_or_listed_provider_models( + provider_config: BaseLLMModelInfo, + custom_llm_provider: str, + litellm_params: LiteLLM_Params | None, ) -> list[str]: try: cached_result: Final = _model_cache.get_cached_model_info(custom_llm_provider, litellm_params) diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 95cc1a06fa9..dec17ef2476 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -7,6 +7,8 @@ import httpx import pytest import respx +from litellm import Router + def test_get_team_models_for_all_models_and_team_only_models(): from litellm.proxy.auth.model_checks import get_team_models @@ -1214,6 +1216,73 @@ def test_key_granted_openrouter_wildcard_without_an_openrouter_deployment_lists_ assert (listed, catalog.call_count) == ([], 0) +@contextmanager +def _openai_catalog_behind_a_catch_all( + monkeypatch: pytest.MonkeyPatch, concrete_name: str, concrete_model_info: Mapping[str, str] +) -> Generator[Router, None, None]: + import litellm + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + with respx.mock(assert_all_called=False) as respx_mock: + respx_mock.get("https://api.openai.com/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "gpt-4o"}, {"id": "gpt-4o-mini"}]}) + ) + yield Router( + model_list=[ + {"model_name": "*", "litellm_params": {"model": "openai/*", "api_key": "sk-openai-catch-all"}}, + { + "model_name": concrete_name, + "litellm_params": {"model": "openrouter/openai/gpt-4o", "api_key": "sk-or-gpt-4o"}, + "model_info": dict(concrete_model_info), + }, + ] + ) + + +@pytest.mark.parametrize( + ("concrete_name", "concrete_model_info", "team_id"), + [ + ("gpt-4o", {}, None), + ("model_name_team-x_gpt-4o", {"team_id": "team-x", "team_public_model_name": "gpt-4o"}, "team-x"), + ], +) +def test_catch_all_wildcard_never_lists_a_name_a_concrete_deployment_serves( + monkeypatch: pytest.MonkeyPatch, concrete_name: str, concrete_model_info: Mapping[str, str], team_id: str | None +) -> None: + from litellm.proxy.auth.model_checks import get_complete_model_list + + with _openai_catalog_behind_a_catch_all(monkeypatch, concrete_name, concrete_model_info) as router: + listed: Final = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=router.get_model_names(team_id=team_id), + user_model=None, + infer_model_from_keys=False, + llm_router=router, + team_id=team_id, + ) + + assert sorted(listed) == ["gpt-4o", "gpt-4o-mini"] + + +def test_catch_all_wildcard_model_info_keeps_only_the_concrete_row_for_a_deployed_name( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info + + with _openai_catalog_behind_a_catch_all(monkeypatch, "gpt-4o", {}) as router: + rows: Final = expand_wildcard_deployments_for_model_info( + [dict(deployment) for deployment in router.get_model_list() or ()], llm_router=router + ) + + assert sorted((row["model_name"], row["litellm_params"]["model"]) for row in rows) == [ + ("gpt-4o", "openrouter/openai/gpt-4o"), + ("gpt-4o-mini", "openai/gpt-4o-mini"), + ] + + @respx.mock def test_custom_prefix_that_starts_a_discovered_id_still_prefixes_every_listed_model( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py index b6a463bb783..b48d2bb36cc 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py @@ -940,3 +940,26 @@ def test_a_key_wildcard_finds_the_deployment_it_grants_without_listing_any_provi assert credentials is not None assert (credentials["custom_llm_provider"], credentials["api_key"]) == (custom_llm_provider, api_key) assert catalog_requests.call_count == 0 + + +@pytest.mark.parametrize( + ("deployed_name", "api_key"), [("gpt-4o-mini", None), ("openai/gpt-4o-mini", "openai-deployment-key")] +) +def test_a_provider_wildcard_grant_finds_only_deployments_named_under_it(deployed_name: str, api_key: str | None): + router: Final = Router( + model_list=[ + { + "model_name": deployed_name, + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "openai-deployment-key"}, + } + ] + ) + + credentials: Final = get_team_provider_credentials( + llm_router=router, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", user_id="test-user", models=["openai/*"]), + custom_llm_provider="openai", + include_accessible_deployments=True, + ) + + assert (None if credentials is None else credentials["api_key"]) == api_key diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 02aaff287cc..3b933d64202 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -8948,6 +8948,26 @@ def test_get_valid_models_keeps_a_good_listing_when_an_overlapping_listing_fails ) == ["openrouter/anthropic/claude-haiku-5-5"] +@respx.mock +def test_get_valid_models_asks_a_provider_once_when_many_listings_start_together(reset_mock_cache: None) -> None: + def slow_catalog(request: httpx.Request) -> httpx.Response: + threading.Event().wait(0.2) + return httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]}) + + catalog: Final = respx.get("https://openrouter.ai/api/v1/models/user").mock(side_effect=slow_catalog) + deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-many-listings") + + def list_models(_: int) -> tuple[str, ...]: + return tuple( + get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment) + ) + + with ThreadPoolExecutor(max_workers=8) as pool: + listings: Final = frozenset(pool.map(list_models, range(8))) + + assert (listings, catalog.call_count) == (frozenset({("openrouter/anthropic/claude-haiku-5-5",)}), 1) + + @pytest.fixture def isolated_openai_model_sets(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( From bf76a5380117b8b3a3862689732d6b366635e8e7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:57:21 -0700 Subject: [PATCH 09/11] fix(proxy): keep the listing failure log line the same for timed-out listings --- litellm/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index cda028fabb9..66b8bdf389e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8532,7 +8532,7 @@ def _cached_or_listed_provider_models( _model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) return models except httpx.TimeoutException as e: - verbose_logger.warning("Timed out getting valid models: %s", e) + verbose_logger.warning("Error getting valid models: %s", e) _model_cache.set_failed_model_listing( custom_llm_provider, litellm_params, ttl_seconds=_FAILED_PROVIDER_LISTING_TTL_SECONDS ) From 4b3c273fad89ff341d22c19db6795861e719a8fe Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 09:34:09 -0700 Subject: [PATCH 10/11] refactor(proxy): fold the cached listing lookup back into the provider listing function --- litellm/utils.py | 44 ++++++++++++++++++-------------------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 66b8bdf389e..a7775f4f34c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8512,34 +8512,26 @@ def _get_valid_models_from_provider_api( litellm_params: LiteLLM_Params | None = None, ) -> list[str]: with _model_cache.listing_lock(custom_llm_provider, litellm_params): - return _cached_or_listed_provider_models(provider_config, custom_llm_provider, litellm_params) + try: + cached_result: Final = _model_cache.get_cached_model_info(custom_llm_provider, litellm_params) + if cached_result is not None: + return cached_result + models: Final = provider_config.discover_models( + litellm_params=_provider_listing_params(custom_llm_provider, litellm_params) + ) -def _cached_or_listed_provider_models( - provider_config: BaseLLMModelInfo, - custom_llm_provider: str, - litellm_params: LiteLLM_Params | None, -) -> list[str]: - try: - cached_result: Final = _model_cache.get_cached_model_info(custom_llm_provider, litellm_params) - - if cached_result is not None: - return cached_result - models: Final = provider_config.discover_models( - litellm_params=_provider_listing_params(custom_llm_provider, litellm_params) - ) - - _model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) - return models - except httpx.TimeoutException as e: - verbose_logger.warning("Error getting valid models: %s", e) - _model_cache.set_failed_model_listing( - custom_llm_provider, litellm_params, ttl_seconds=_FAILED_PROVIDER_LISTING_TTL_SECONDS - ) - return [] - except Exception as e: - verbose_logger.warning("Error getting valid models: %s", e) - return [] + _model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) + return models + except httpx.TimeoutException as e: + verbose_logger.warning("Error getting valid models: %s", e) + _model_cache.set_failed_model_listing( + custom_llm_provider, litellm_params, ttl_seconds=_FAILED_PROVIDER_LISTING_TTL_SECONDS + ) + return [] + except Exception as e: + verbose_logger.warning("Error getting valid models: %s", e) + return [] def get_valid_models( From b3f386bb0f807f002f66354b79b3accf95e86ade Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:53:43 -0700 Subject: [PATCH 11/11] fix(proxy): list each model id once and keep a named deployment's row over a wildcard's --- litellm/proxy/auth/model_checks.py | 45 ++++++++++------ tests/unit/proxy/auth/test_model_checks.py | 60 +++++++++++++++++++++- 2 files changed, 88 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 6d683bfa94e..cc3d8aac388 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -267,9 +267,7 @@ def get_complete_model_list( expand_wildcard_routes=expand_wildcard_routes, ) - complete_model_list: Final = unique_models + all_wildcard_models - - return complete_model_list + return list(dict.fromkeys(unique_models + all_wildcard_models)) def _hydrate_litellm_credential_name( @@ -399,18 +397,15 @@ def _deployment_team_id(deployment: Mapping[str, object]) -> str | None: def _cached_wildcard_routing(llm_router: Router | None) -> Callable[[str, str | None], frozenset[str]] | None: + return None if llm_router is None else functools.cache(llm_router.wildcard_routed_models) + + +def _deployed_by_name(llm_router: Router | None) -> Callable[[str, str | None], bool]: if llm_router is None: - return None + return lambda _model, _team_id: False deployed_names: Final = frozenset(llm_router.model_names) team_deployed_names: Final = frozenset(llm_router.team_model_to_deployment_indices) - wildcard_routed_models: Final = llm_router.wildcard_routed_models - - def routed_models(model: str, team_id: str | None) -> frozenset[str]: - if model in deployed_names or (team_id, model) in team_deployed_names: - return frozenset() - return wildcard_routed_models(model, team_id) - - return functools.cache(routed_models) + return lambda model, team_id: model in deployed_names or (team_id, model) in team_deployed_names def _routes_to_model( @@ -522,6 +517,25 @@ def _routed_deployment_model(name_pattern: str, listed_name: str, deployment_mod ) +def _names_listed_under_wildcard( + deployment: Mapping[str, object], + wildcard_pattern: str, + litellm_params: LiteLLM_Params | None, + routed_models: Callable[[str, str | None], frozenset[str]] | None, + deployed_by_name: Callable[[str, str | None], bool], +) -> tuple[str, ...]: + team_id: Final = _deployment_team_id(deployment) + return tuple( + name + for name in get_known_models_from_wildcard( + wildcard_model=wildcard_pattern, + litellm_params=litellm_params, + routes_to_model=_routes_to_model(routed_models, team_id), + ) + if not deployed_by_name(name, team_id) + ) + + def expand_wildcard_deployments_for_model_info( deployments: list[dict[str, object]], llm_router: Router | None = None, @@ -535,6 +549,7 @@ def expand_wildcard_deployments_for_model_info( """ expanded: Final[list[dict[str, object]]] = [] routed_models: Final = _cached_wildcard_routing(llm_router) + deployed_by_name: Final = _deployed_by_name(llm_router) for deployment in deployments: model_name = str(deployment.get("model_name") or "") raw_params = deployment.get("litellm_params") @@ -559,10 +574,8 @@ def expand_wildcard_deployments_for_model_info( except Exception: expanded.append(deployment) continue - expanded_names = get_known_models_from_wildcard( - wildcard_model=wildcard_pattern, - litellm_params=litellm_params, - routes_to_model=_routes_to_model(routed_models, _deployment_team_id(deployment)), + expanded_names = _names_listed_under_wildcard( + deployment, wildcard_pattern, litellm_params, routed_models, deployed_by_name ) if not expanded_names: expanded.append(deployment) diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index dec17ef2476..49289299ea3 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -1248,7 +1248,7 @@ def _openai_catalog_behind_a_catch_all( ("model_name_team-x_gpt-4o", {"team_id": "team-x", "team_public_model_name": "gpt-4o"}, "team-x"), ], ) -def test_catch_all_wildcard_never_lists_a_name_a_concrete_deployment_serves( +def test_catch_all_wildcard_lists_a_name_a_concrete_deployment_serves_once( monkeypatch: pytest.MonkeyPatch, concrete_name: str, concrete_model_info: Mapping[str, str], team_id: str | None ) -> None: from litellm.proxy.auth.model_checks import get_complete_model_list @@ -1283,6 +1283,64 @@ def test_catch_all_wildcard_model_info_keeps_only_the_concrete_row_for_a_deploye ] +@contextmanager +def _openrouter_catalog_beside_a_named_deployment( + monkeypatch: pytest.MonkeyPatch, wildcard_name: str, named: str +) -> Generator[Router, None, None]: + import litellm + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.delenv("OPENROUTER_API_BASE", raising=False) + with respx.mock(assert_all_called=False) as respx_mock: + respx_mock.get("https://openrouter.ai/api/v1/models/user").mock( + return_value=httpx.Response( + 200, json={"data": [{"id": "openai/gpt-5.4-mini"}, {"id": "anthropic/claude-haiku-5-5"}]} + ) + ) + yield Router( + model_list=[ + { + "model_name": wildcard_name, + "litellm_params": {"model": "openrouter/*", "api_key": f"sk-or-wildcard-{wildcard_name}"}, + }, + { + "model_name": named, + "litellm_params": {"model": "openrouter/openai/gpt-5.4-mini", "api_key": "sk-or-named"}, + }, + ] + ) + + +@pytest.mark.parametrize( + ("wildcard_name", "named"), + [("openrouter/*", "openrouter/openai/gpt-5.4-mini"), ("or-team/*", "or-team/openai/gpt-5.4-mini")], +) +def test_provider_wildcard_lists_a_name_a_named_deployment_serves_once( + monkeypatch: pytest.MonkeyPatch, wildcard_name: str, named: str +) -> None: + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list + + with _openrouter_catalog_beside_a_named_deployment(monkeypatch, wildcard_name, named) as router: + listed: Final = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=list(router.model_names), + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + rows: Final = expand_wildcard_deployments_for_model_info( + [dict(deployment) for deployment in router.get_model_list() or ()], llm_router=router + ) + + vendor_prefix: Final = wildcard_name.removesuffix("*") + assert sorted(listed) == [f"{vendor_prefix}anthropic/claude-haiku-5-5", named] + assert sorted((row["model_name"], row["litellm_params"]["api_key"]) for row in rows) == [ + (f"{vendor_prefix}anthropic/claude-haiku-5-5", f"sk-or-wildcard-{wildcard_name}"), + (named, "sk-or-named"), + ] + + @respx.mock def test_custom_prefix_that_starts_a_discovered_id_still_prefixes_every_listed_model( monkeypatch: pytest.MonkeyPatch,