From 8d862c61d9bbc50be39be5d1ef008f0be8cd495e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:20:03 -0700 Subject: [PATCH] fix(openrouter): list models off the event loop and keep only chat-callable catalog ids The model-list builders are sync, and with check_provider_endpoint on they call OpenRouter's catalog with a 10 second timeout and no caching of failures. They ran on the event loop inside /v1/models, /model/info, /model_group/info and the per-user listing, so one slow catalog froze every request on the worker. Those calls now run in a worker thread with asyncio.to_thread File, batch and RAG credential lookups only need each name's deployment, and a wildcard that is itself a deployment sits in the list ahead of its expansions and resolves its own credentials. get_complete_model_list takes expand_deployed_wildcard_routes, which that lookup sets to False, so a deployed wildcard never fetches its provider catalog there. A key wildcard wider than any deployment, like anthropic/* over a deployed anthropic/claude-*, still expands so it finds the matching deployment's credentials. Every other caller keeps the default The pattern router's model lookup tolerates a pattern removed between its two reads of the pattern table, which a deployment delete on the event loop can now do while a listing runs in a thread Listing now reads only OpenRouter's models/user catalog. OpenRouter's own /models list carries no embedding models, the OpenAI-format /v1/models has no mode field so chat pickers cannot tell embeddings apart, /model_group/info labels them mode chat, and one failing catalog emptied the whole list. Ids ending in :batch are dropped because chat completions answers 404 for them, while :free and every other variant stay. An api_base that already ends in /chat/completions, which chat accepts as is, has that suffix stripped before the catalog route is joined --- litellm/llms/openrouter/common_utils.py | 25 +++---- litellm/proxy/auth/model_checks.py | 5 ++ .../openai_files_endpoints/common_utils.py | 1 + litellm/proxy/proxy_server.py | 8 ++- litellm/proxy/utils.py | 6 +- .../router_utils/pattern_match_deployments.py | 4 +- .../test_openrouter_common_utils.py | 62 +++++++++-------- tests/unit/proxy/auth/test_model_checks.py | 21 +----- .../test_files_common_utils.py | 65 +++++++++++++++++ .../proxy/proxy_server/test_routes_models.py | 69 +++++++++++++++++++ .../proxy/utils/helpers/test_model_access.py | 62 ++++++++++++++++- 11 files changed, 262 insertions(+), 66 deletions(-) diff --git a/litellm/llms/openrouter/common_utils.py b/litellm/llms/openrouter/common_utils.py index 701d75b350e..64caf83d745 100644 --- a/litellm/llms/openrouter/common_utils.py +++ b/litellm/llms/openrouter/common_utils.py @@ -1,4 +1,3 @@ -import itertools from collections.abc import Mapping, Sequence from typing import Final @@ -13,7 +12,9 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues _DEFAULT_API_BASE: Final = "https://openrouter.ai/api/v1" -_CATALOG_ROUTES: Final = ("models/user", "embeddings/models") +_CHAT_CATALOG_ROUTE: Final = "models/user" +_CHAT_COMPLETIONS_PATH: Final = "/chat/completions" +_BATCH_ONLY_SUFFIX: Final = ":batch" _CATALOG_TIMEOUT: Final = httpx.Timeout(10.0) @@ -33,12 +34,8 @@ class _Catalog(BaseModel): data: tuple[_CatalogModel, ...] -def _catalog_ids(url: str, headers: Mapping[str, str]) -> tuple[str, ...]: - response: Final = litellm.module_level_client.get( # pyright: ignore[reportUnknownMemberType] # legacy HTTP wrapper has untyped query parameters - url=url, headers=headers, timeout=_CATALOG_TIMEOUT - ) - response.raise_for_status() - return tuple(model.id for model in _Catalog.model_validate_json(response.content).data) +def _chat_catalog_url(api_base: str) -> str: + return join_api_base(api_base.rstrip("/").removesuffix(_CHAT_COMPLETIONS_PATH), _CHAT_CATALOG_ROUTE) class OpenRouterModelInfo(BaseLLMModelInfo): @@ -84,8 +81,12 @@ class OpenRouterModelInfo(BaseLLMModelInfo): headers: Final = self.validate_environment( headers={}, model="", messages=(), optional_params={}, litellm_params={}, api_key=resolved_key ) - base: Final = self.get_api_base(api_base) - model_ids: Final = itertools.chain.from_iterable( - _catalog_ids(join_api_base(base, route), headers) for route in _CATALOG_ROUTES + response: Final = litellm.module_level_client.get( # pyright: ignore[reportUnknownMemberType] # legacy HTTP wrapper has untyped query parameters + url=_chat_catalog_url(self.get_api_base(api_base)), headers=headers, timeout=_CATALOG_TIMEOUT ) - return list(dict.fromkeys(f"openrouter/{model_id}" for model_id in model_ids)) + response.raise_for_status() + return [ + f"openrouter/{model.id}" + for model in _Catalog.model_validate_json(response.content).data + if not model.id.endswith(_BATCH_ONLY_SUFFIX) + ] diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index c4c538083ee..fb97a7c95b7 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -204,6 +204,7 @@ def get_complete_model_list( include_model_access_groups: bool | None = False, only_model_access_groups: bool | None = False, team_id: str | None = None, + expand_deployed_wildcard_routes: bool = True, ) -> list[str]: """Logic for returning complete model list for a given key + team pair""" @@ -249,6 +250,7 @@ def get_complete_model_list( return_wildcard_routes=return_wildcard_routes, llm_router=llm_router, team_id=team_id, + expand_deployed_wildcard_routes=expand_deployed_wildcard_routes, ) complete_model_list: Final = unique_models + all_wildcard_models @@ -539,6 +541,7 @@ def _get_wildcard_models( return_wildcard_routes: bool | None = False, llm_router: Router | None = None, team_id: str | None = None, + expand_deployed_wildcard_routes: bool = True, ) -> list[str]: models_to_remove: Final = set() routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id) @@ -551,6 +554,8 @@ def _get_wildcard_models( models_to_remove.add(model) model_list = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None + if model_list and not expand_deployed_wildcard_routes: + continue if model_list: for router_model in model_list: all_wildcard_models.extend( diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index f7289445d3a..6cbafd53537 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -657,6 +657,7 @@ def get_team_provider_credentials( model_access_groups=model_access_groups, include_model_access_groups=True, team_id=team_id, + expand_deployed_wildcard_routes=False, ) ) ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3b36ea5afce..84b05b88a41 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12398,7 +12398,8 @@ async def model_list( # Get complete model list including wildcard routes if requested from litellm.proxy.auth.model_checks import get_complete_model_list - all_models = get_complete_model_list( + all_models = await asyncio.to_thread( + get_complete_model_list, key_models=[], team_models=[], proxy_model_list=proxy_model_list, @@ -17751,7 +17752,7 @@ async def model_info_v1( alias_models: Final = copy.deepcopy(llm_router.get_model_list_from_model_alias()) all_models.extend(alias_models) - all_models = expand_wildcard_deployments_for_model_info(all_models, llm_router) + all_models = await asyncio.to_thread(expand_wildcard_deployments_for_model_info, all_models, llm_router) allowed_model_names: Final = await _get_v1_model_info_allowed_model_names_with_access_groups( user_api_key_dict=user_api_key_dict, @@ -18086,7 +18087,8 @@ async def model_group_info( LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, ) all_models_str: Final = ( - get_complete_model_list( + await asyncio.to_thread( + get_complete_model_list, key_models=(), team_models=(), proxy_model_list=llm_router.get_model_names(), diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0ddfc5ce62f..cc3c7280160 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -9517,7 +9517,8 @@ async def _agent_access_group_visible_models( if not granted: return frozenset() return frozenset( - get_complete_model_list( + await asyncio.to_thread( + get_complete_model_list, key_models=granted, team_models=(), proxy_model_list=proxy_model_list, @@ -9652,7 +9653,8 @@ async def get_available_models_for_user( ) # Get complete model list - complete_models: Final = get_complete_model_list( + complete_models: Final = await asyncio.to_thread( + get_complete_model_list, key_models=granted_key_models, team_models=granted_team_models, proxy_model_list=proxy_model_list, diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index 4dd217cf59b..2b8d1e9c11b 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -279,7 +279,9 @@ class PatternMatchRouter: PatternMatchRouter.set_deployment_model_name( matched_pattern=pattern_match, litellm_deployment_litellm_model=deployment["litellm_params"]["model"] ) - for deployment in self.patterns[pattern] + for deployment in self.patterns.get( # pyright: ignore[reportUnknownMemberType] # untyped deployment dicts + pattern, () + ) ) diff --git a/tests/unit/llms/openrouter/test_openrouter_common_utils.py b/tests/unit/llms/openrouter/test_openrouter_common_utils.py index 10e2630b161..087c05561b5 100644 --- a/tests/unit/llms/openrouter/test_openrouter_common_utils.py +++ b/tests/unit/llms/openrouter/test_openrouter_common_utils.py @@ -32,38 +32,49 @@ def _configure(monkeypatch: pytest.MonkeyPatch, source: str, value: str) -> None monkeypatch.setenv(source, value) +def test_get_models_lists_only_the_chat_catalog_under_openrouter_keeping_the_org_namespace(): + with respx.mock() as respx_mock: + respx_mock.get(f"{DEFAULT_BASE}/models/user").mock( + return_value=_catalog("openai/gpt-5.4-mini", "anthropic/claude-haiku-5-5") + ) + + assert OpenRouterModelInfo().get_models(api_key="sk-or-test") == [ + "openrouter/openai/gpt-5.4-mini", + "openrouter/anthropic/claude-haiku-5-5", + ] + assert [call.request.url.path for call in respx_mock.calls] == ["/api/v1/models/user"] + + @respx.mock -def test_get_models_lists_both_catalogs_once_each_under_openrouter_keeping_the_org_namespace(): +def test_get_models_drops_batch_only_ids_and_keeps_every_other_variant(): respx.get(f"{DEFAULT_BASE}/models/user").mock( - return_value=_catalog("openai/gpt-5.4-mini", "anthropic/claude-haiku-5-5") - ) - respx.get(f"{DEFAULT_BASE}/embeddings/models").mock( - return_value=_catalog("openai/text-embedding-3-small", "openai/gpt-5.4-mini") + return_value=_catalog("openai/gpt-5.4-mini:batch", "openai/gpt-5.4-mini", "qwen/qwen3-coder:free") ) assert OpenRouterModelInfo().get_models(api_key="sk-or-test") == [ "openrouter/openai/gpt-5.4-mini", - "openrouter/anthropic/claude-haiku-5-5", - "openrouter/openai/text-embedding-3-small", + "openrouter/qwen/qwen3-coder:free", ] +@pytest.mark.parametrize( + "api_base", + [ + "https://gateway.example/openrouter/api/v1/", + "https://gateway.example/openrouter/api/v1/chat/completions", + "https://gateway.example/openrouter/api/v1/chat/completions/", + ], +) @respx.mock -def test_get_models_keeps_the_api_base_path_and_sends_the_deployment_key(): +def test_get_models_keeps_the_api_base_path_and_sends_the_deployment_key(api_base: str): chat_route: Final = respx.get("https://gateway.example/openrouter/api/v1/models/user").mock( return_value=_catalog("openai/gpt-5.4-mini") ) - embeddings_route: Final = respx.get("https://gateway.example/openrouter/api/v1/embeddings/models").mock( - return_value=_catalog() - ) - assert OpenRouterModelInfo().get_models( - api_key="sk-or-deployment", api_base="https://gateway.example/openrouter/api/v1/" - ) == ["openrouter/openai/gpt-5.4-mini"] - assert [route.calls.last.request.headers["authorization"] for route in (chat_route, embeddings_route)] == [ - "Bearer sk-or-deployment", - "Bearer sk-or-deployment", + assert OpenRouterModelInfo().get_models(api_key="sk-or-deployment", api_base=api_base) == [ + "openrouter/openai/gpt-5.4-mini" ] + assert chat_route.calls.last.request.headers["authorization"] == "Bearer sk-or-deployment" @pytest.mark.parametrize("winner", range(len(KEY_SOURCES)), ids=KEY_SOURCES) @@ -76,7 +87,6 @@ def test_get_models_sends_the_first_configured_key_in_the_chat_paths_order( if winner > 0: _configure(monkeypatch, KEY_SOURCES[winner], "sk-or-winner") chat_route: Final = respx.get(f"{DEFAULT_BASE}/models/user").mock(return_value=_catalog("qwen/qwen3-coder")) - respx.get(f"{DEFAULT_BASE}/embeddings/models").mock(return_value=_catalog()) assert OpenRouterModelInfo().get_models(api_key="sk-or-winner" if winner == 0 else None) == [ "openrouter/qwen/qwen3-coder" @@ -95,7 +105,6 @@ def test_get_models_calls_the_first_configured_base_in_the_chat_paths_order( _configure(monkeypatch, BASE_SOURCES[winner], "https://winner.example/api/v1") expected_base: Final = "https://winner.example/api/v1" if winner < len(BASE_SOURCES) else DEFAULT_BASE respx.get(f"{expected_base}/models/user").mock(return_value=_catalog("qwen/qwen3-coder")) - respx.get(f"{expected_base}/embeddings/models").mock(return_value=_catalog()) assert OpenRouterModelInfo().get_models( api_key="sk-or-test", api_base="https://winner.example/api/v1" if winner == 0 else None @@ -114,17 +123,12 @@ def test_get_models_without_any_key_raises_before_calling_openrouter(): assert not chat_route.called -@pytest.mark.parametrize("failing_route", ["models/user", "embeddings/models"]) -def test_get_models_raises_when_either_catalog_answers_an_error(failing_route: str): - with respx.mock(assert_all_called=False) as respx_mock: - respx_mock.get(f"{DEFAULT_BASE}/models/user").mock(return_value=_catalog("openai/gpt-5.4-mini")) - respx_mock.get(f"{DEFAULT_BASE}/embeddings/models").mock(return_value=_catalog()) - respx_mock.get(f"{DEFAULT_BASE}/{failing_route}").mock( - return_value=httpx.Response(401, json={"error": {"code": 401}}) - ) +@respx.mock +def test_get_models_raises_when_the_catalog_answers_an_error(): + respx.get(f"{DEFAULT_BASE}/models/user").mock(return_value=httpx.Response(401, json={"error": {"code": 401}})) - with pytest.raises(httpx.HTTPStatusError): - OpenRouterModelInfo().get_models(api_key="sk-or-revoked") + with pytest.raises(httpx.HTTPStatusError): + OpenRouterModelInfo().get_models(api_key="sk-or-revoked") def test_validate_environment_adds_the_resolved_key_and_keeps_the_callers_headers(monkeypatch: pytest.MonkeyPatch): diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 549679e4054..0f310f8a0ad 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -1156,22 +1156,10 @@ def test_partial_bedrock_wildcard_lists_nothing_when_no_discovered_id_carries_it ( "openrouter/*", "openrouter/*", - [ - "openrouter/openai/gpt-5.4-mini", - "openrouter/anthropic/claude-haiku-5-5", - "openrouter/openai/text-embedding-3-small", - ], - ), - ( - "or-team/*", - "openrouter/*", - ["or-team/openai/gpt-5.4-mini", "or-team/anthropic/claude-haiku-5-5", "or-team/openai/text-embedding-3-small"], - ), - ( - "openrouter/openai/*", - "openrouter/openai/*", - ["openrouter/openai/gpt-5.4-mini", "openrouter/openai/text-embedding-3-small"], + ["openrouter/openai/gpt-5.4-mini", "openrouter/anthropic/claude-haiku-5-5"], ), + ("or-team/*", "openrouter/*", ["or-team/openai/gpt-5.4-mini", "or-team/anthropic/claude-haiku-5-5"]), + ("openrouter/openai/*", "openrouter/openai/*", ["openrouter/openai/gpt-5.4-mini"]), ], ) @respx.mock @@ -1189,9 +1177,6 @@ def test_openrouter_wildcard_lists_the_live_catalog_with_each_vendor_namespace( 200, json={"data": [{"id": "openai/gpt-5.4-mini"}, {"id": "anthropic/claude-haiku-5-5"}]} ) ) - respx.get("https://openrouter.ai/api/v1/embeddings/models").mock( - return_value=httpx.Response(200, json={"data": [{"id": "openai/text-embedding-3-small"}]}) - ) assert ( get_known_models_from_wildcard(wildcard, LiteLLM_Params(model=deployment_model, api_key=f"sk-or-{wildcard}")) diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py index f6c6b51fc93..dad8b1dc616 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py @@ -3,10 +3,13 @@ from types import MappingProxyType from typing import Final from unittest.mock import AsyncMock, MagicMock +import httpx import pytest +import respx from fastapi import HTTPException from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles +import litellm from litellm import Router from litellm.caching.caching import DualCache from litellm.proxy._types import UserAPIKeyAuth @@ -851,3 +854,65 @@ def test_a_team_keeps_its_first_config_file_openai_deployment_over_older_db_ones assert credentials is not None assert credentials["api_key"] == "deployment-z-key" + + +@pytest.mark.parametrize( + ("custom_llm_provider", "openrouter_key", "api_key"), + [("openai", "sk-or-files-openai", "openai-key"), ("openrouter", "sk-or-files-openrouter", "sk-or-files-openrouter")], +) +def test_a_provider_credential_lookup_never_lists_an_openrouter_wildcards_live_catalog( + monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, openrouter_key: str, api_key: str +): + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + router: Final = Router( + model_list=[ + { + "model_name": "openrouter/*", + "litellm_params": { + "model": "openrouter/*", + "api_key": openrouter_key, + "api_base": "https://openrouter.ai/api/v1", + }, + }, + {"model_name": "gpt-5.4-mini", "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "openai-key"}}, + ] + ) + + with respx.mock(assert_all_called=False) as respx_mock: + catalog_route: Final = respx_mock.get("https://openrouter.ai/api/v1/models/user").mock( + return_value=httpx.Response(200, json={"data": [{"id": "openai/gpt-5.4-mini"}]}) + ) + credentials: Final = get_team_provider_credentials( + llm_router=router, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", user_id="test-user"), + custom_llm_provider=custom_llm_provider, + include_accessible_deployments=True, + ) + + assert credentials is not None + assert (credentials["custom_llm_provider"], credentials["api_key"]) == (custom_llm_provider, api_key) + assert catalog_route.call_count == 0 + + +def test_a_key_allowed_a_wider_wildcard_than_any_deployment_gets_the_matching_deployments_credentials( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + router: Final = Router( + model_list=[ + { + "model_name": "anthropic/claude-*", + "litellm_params": {"model": "anthropic/claude-*", "api_key": "anthropic-deployment-key"}, + } + ] + ) + + credentials: Final = get_team_provider_credentials( + llm_router=router, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", user_id="test-user", models=["anthropic/*"]), + custom_llm_provider="anthropic", + include_accessible_deployments=True, + ) + + assert credentials is not None + assert (credentials["custom_llm_provider"], credentials["api_key"]) == ("anthropic", "anthropic-deployment-key") diff --git a/tests/unit/proxy/proxy_server/test_routes_models.py b/tests/unit/proxy/proxy_server/test_routes_models.py index 7be30c61f0c..01aa8722444 100644 --- a/tests/unit/proxy/proxy_server/test_routes_models.py +++ b/tests/unit/proxy/proxy_server/test_routes_models.py @@ -9,12 +9,19 @@ Pins (PR2): from __future__ import annotations +import asyncio +import threading +from dataclasses import dataclass, field +from typing import Final from unittest.mock import MagicMock +import httpx import pytest +import respx import litellm from litellm.proxy import proxy_server +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy import utils as proxy_utils from litellm.proxy.utils import create_model_info_response from litellm.types.router import DeploymentModelListingInfo @@ -514,3 +521,65 @@ def test_codex_service_tiers_resolved_via_internal_team_key(client, auth_as, pat assert response.status_code == 200 (entry,) = response.json()["models"] assert (entry["slug"], [tier["id"] for tier in entry["service_tiers"]]) == ("gpt-4-team", ["ultrafast"]) + + +OPENROUTER_BASE: Final = "https://openrouter.ai/api/v1" + + +@dataclass(frozen=True, slots=True) +class _CatalogHeldUntilTheLoopRuns: + requested: threading.Event = field(default_factory=threading.Event) + released: threading.Event = field(default_factory=threading.Event) + + def answer(self, _request: httpx.Request) -> httpx.Response: + self.requested.set() + if not self.released.wait(timeout=2): + return httpx.Response(504) + return httpx.Response(200, json={"data": [{"id": "openai/gpt-5.4-mini"}]}) + + async def release_from_the_loop(self) -> None: + assert await asyncio.to_thread(self.requested.wait, 5) + self.released.set() + + +@pytest.mark.parametrize("scope", [None, "expand"]) +async def test_v1_models_fetches_a_live_openrouter_catalog_without_blocking_the_event_loop( + monkeypatch: pytest.MonkeyPatch, scope: str | None +): + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "openrouter/*", + "litellm_params": { + "model": "openrouter/*", + "api_key": f"sk-or-v1-models-off-loop-{scope}", + "api_base": OPENROUTER_BASE, + }, + } + ] + ), + ) + catalog: Final = _CatalogHeldUntilTheLoopRuns() + + with respx.mock() as respx_mock: + catalog_route: Final = respx_mock.get(f"{OPENROUTER_BASE}/models/user").mock(side_effect=catalog.answer) + listing: Final = asyncio.ensure_future( + proxy_server.model_list( + user_api_key_dict=UserAPIKeyAuth( + api_key="sk-test-key", user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN + ), + scope=scope, + ) + ) + await catalog.release_from_the_loop() + response: Final = await listing + + assert catalog_route.calls.last.response.status_code == 200 + assert [model["id"] for model in response["data"]] == ["openrouter/openai/gpt-5.4-mini"] diff --git a/tests/unit/proxy/utils/helpers/test_model_access.py b/tests/unit/proxy/utils/helpers/test_model_access.py index ecf1810f34a..82e5dac5592 100644 --- a/tests/unit/proxy/utils/helpers/test_model_access.py +++ b/tests/unit/proxy/utils/helpers/test_model_access.py @@ -1,10 +1,16 @@ +import asyncio +import threading +from dataclasses import dataclass, field +from typing import Final from unittest.mock import MagicMock +import httpx import pytest +import respx from fastapi import HTTPException import litellm -from litellm import ModelResponse +from litellm import ModelResponse, Router from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import ( create_model_info_response, @@ -650,3 +656,57 @@ async def test_listing_is_unchanged_without_an_agent_or_without_attached_groups( resolve_agent_ceiling=_agent_ceiling(None), ) assert (plain_key, agent_without_groups) == (["model-a", "model-b"], ["model-a", "model-b"]) + + +OPENROUTER_BASE: Final = "https://openrouter.ai/api/v1" + + +@dataclass(frozen=True, slots=True) +class _CatalogHeldUntilTheLoopRuns: + requested: threading.Event = field(default_factory=threading.Event) + released: threading.Event = field(default_factory=threading.Event) + + def answer(self, _request: httpx.Request) -> httpx.Response: + self.requested.set() + if not self.released.wait(timeout=2): + return httpx.Response(504) + return httpx.Response(200, json={"data": [{"id": "openai/gpt-5.4-mini"}]}) + + async def release_from_the_loop(self) -> None: + assert await asyncio.to_thread(self.requested.wait, 5) + self.released.set() + + +async def test_get_available_models_for_user_fetches_a_live_catalog_without_blocking_the_event_loop( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + catalog: Final = _CatalogHeldUntilTheLoopRuns() + router: Final = Router( + model_list=[ + { + "model_name": "openrouter/*", + "litellm_params": { + "model": "openrouter/*", + "api_key": "sk-or-available-models-off-loop", + "api_base": OPENROUTER_BASE, + }, + } + ] + ) + + with respx.mock() as respx_mock: + catalog_route: Final = respx_mock.get(f"{OPENROUTER_BASE}/models/user").mock(side_effect=catalog.answer) + listing: Final = asyncio.ensure_future( + get_available_models_for_user( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test-key", user_id="user-1"), + llm_router=router, + general_settings={}, + user_model=None, + ) + ) + await catalog.release_from_the_loop() + models: Final = await listing + + assert catalog_route.calls.last.response.status_code == 200 + assert models == ["openrouter/openai/gpt-5.4-mini"]