scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-06 07:48:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 112

fix(openrouter): list models off the event loop and keep only chat-callable catalog ids

Browse source 
The model-list builders are sync, and with check_provider_endpoint on they call OpenRouter's catalog with a 10 second timeout and no caching of failures. They ran on the event loop inside /v1/models, /model/info, /model_group/info and the per-user listing, so one slow catalog froze every request on the worker. Those calls now run in a worker thread with asyncio.to_thread

File, batch and RAG credential lookups only need each name's deployment, and a wildcard that is itself a deployment sits in the list ahead of its expansions and resolves its own credentials. get_complete_model_list takes expand_deployed_wildcard_routes, which that lookup sets to False, so a deployed wildcard never fetches its provider catalog there. A key wildcard wider than any deployment, like anthropic/* over a deployed anthropic/claude-*, still expands so it finds the matching deployment's credentials. Every other caller keeps the default

The pattern router's model lookup tolerates a pattern removed between its two reads of the pattern table, which a deployment delete on the event loop can now do while a listing runs in a thread

Listing now reads only OpenRouter's models/user catalog. OpenRouter's own /models list carries no embedding models, the OpenAI-format /v1/models has no mode field so chat pickers cannot tell embeddings apart, /model_group/info labels them mode chat, and one failing catalog emptied the whole list. Ids ending in :batch are dropped because chat completions answers 404 for them, while :free and every other variant stay. An api_base that already ends in /chat/completions, which chat accepts as is, has that suffix stripped before the catalog route is joined
This commit is contained in:
mateo-berri 2026-10-28 08:20:03 -07:00
parent c0f6ebcc26
commit 8d862c61d9
11 changed files with 262 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
litellm/llms/openrouter/common_utils.py
View file

@ -1,4 +1,3 @@
import itertools
from collections.abc import Mapping, Sequence
from typing import Final
@ -13,7 +12,9 @@ from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
_DEFAULT_API_BASE: Final = "https://openrouter.ai/api/v1"
_CATALOG_ROUTES: Final = ("models/user", "embeddings/models")
_CHAT_CATALOG_ROUTE: Final = "models/user"
_CHAT_COMPLETIONS_PATH: Final = "/chat/completions"
_BATCH_ONLY_SUFFIX: Final = ":batch"
_CATALOG_TIMEOUT: Final = httpx.Timeout(10.0)
@ -33,12 +34,8 @@ class _Catalog(BaseModel):
data: tuple[_CatalogModel, ...]
def _catalog_ids(url: str, headers: Mapping[str, str]) -> tuple[str, ...]:
response: Final = litellm.module_level_client.get( # pyright: ignore[reportUnknownMemberType] # legacy HTTP wrapper has untyped query parameters
url=url, headers=headers, timeout=_CATALOG_TIMEOUT
)
response.raise_for_status()
return tuple(model.id for model in _Catalog.model_validate_json(response.content).data)
def _chat_catalog_url(api_base: str) -> str:
return join_api_base(api_base.rstrip("/").removesuffix(_CHAT_COMPLETIONS_PATH), _CHAT_CATALOG_ROUTE)
class OpenRouterModelInfo(BaseLLMModelInfo):
@ -84,8 +81,12 @@ class OpenRouterModelInfo(BaseLLMModelInfo):
headers: Final = self.validate_environment(
headers={}, model="", messages=(), optional_params={}, litellm_params={}, api_key=resolved_key
)
base: Final = self.get_api_base(api_base)
model_ids: Final = itertools.chain.from_iterable(
_catalog_ids(join_api_base(base, route), headers) for route in _CATALOG_ROUTES
response: Final = litellm.module_level_client.get( # pyright: ignore[reportUnknownMemberType] # legacy HTTP wrapper has untyped query parameters
url=_chat_catalog_url(self.get_api_base(api_base)), headers=headers, timeout=_CATALOG_TIMEOUT
)
return list(dict.fromkeys(f"openrouter/{model_id}" for model_id in model_ids))
response.raise_for_status()
return [
f"openrouter/{model.id}"
for model in _Catalog.model_validate_json(response.content).data
if not model.id.endswith(_BATCH_ONLY_SUFFIX)
]

5
litellm/proxy/auth/model_checks.py
View file

@ -204,6 +204,7 @@ def get_complete_model_list(
include_model_access_groups: bool | None = False,
only_model_access_groups: bool | None = False,
team_id: str | None = None,
expand_deployed_wildcard_routes: bool = True,
) -> list[str]:
"""Logic for returning complete model list for a given key + team pair"""
@ -249,6 +250,7 @@ def get_complete_model_list(
return_wildcard_routes=return_wildcard_routes,
llm_router=llm_router,
team_id=team_id,
expand_deployed_wildcard_routes=expand_deployed_wildcard_routes,
)
complete_model_list: Final = unique_models + all_wildcard_models
@ -539,6 +541,7 @@ def _get_wildcard_models(
return_wildcard_routes: bool | None = False,
llm_router: Router | None = None,
team_id: str | None = None,
expand_deployed_wildcard_routes: bool = True,
) -> list[str]:
models_to_remove: Final = set()
routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id)
@ -551,6 +554,8 @@ def _get_wildcard_models(
models_to_remove.add(model)
model_list = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None
if model_list and not expand_deployed_wildcard_routes:
continue
if model_list:
for router_model in model_list:
all_wildcard_models.extend(

1
litellm/proxy/openai_files_endpoints/common_utils.py
View file

@ -657,6 +657,7 @@ def get_team_provider_credentials(
model_access_groups=model_access_groups,
include_model_access_groups=True,
team_id=team_id,
expand_deployed_wildcard_routes=False,
)
)
)

8
litellm/proxy/proxy_server.py
View file

@ -12398,7 +12398,8 @@ async def model_list(
# Get complete model list including wildcard routes if requested
from litellm.proxy.auth.model_checks import get_complete_model_list
all_models = get_complete_model_list(
all_models = await asyncio.to_thread(
get_complete_model_list,
key_models=[],
team_models=[],
proxy_model_list=proxy_model_list,
@ -17751,7 +17752,7 @@ async def model_info_v1(
alias_models: Final = copy.deepcopy(llm_router.get_model_list_from_model_alias())
all_models.extend(alias_models)
all_models = expand_wildcard_deployments_for_model_info(all_models, llm_router)
all_models = await asyncio.to_thread(expand_wildcard_deployments_for_model_info, all_models, llm_router)
allowed_model_names: Final = await _get_v1_model_info_allowed_model_names_with_access_groups(
user_api_key_dict=user_api_key_dict,
@ -18086,7 +18087,8 @@ async def model_group_info(
LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
)
all_models_str: Final = (
get_complete_model_list(
await asyncio.to_thread(
get_complete_model_list,
key_models=(),
team_models=(),
proxy_model_list=llm_router.get_model_names(),

6
litellm/proxy/utils.py
View file

@ -9517,7 +9517,8 @@ async def _agent_access_group_visible_models(
if not granted:
return frozenset()
return frozenset(
get_complete_model_list(
await asyncio.to_thread(
get_complete_model_list,
key_models=granted,
team_models=(),
proxy_model_list=proxy_model_list,
@ -9652,7 +9653,8 @@ async def get_available_models_for_user(
)
# Get complete model list
complete_models: Final = get_complete_model_list(
complete_models: Final = await asyncio.to_thread(
get_complete_model_list,
key_models=granted_key_models,
team_models=granted_team_models,
proxy_model_list=proxy_model_list,

4
litellm/router_utils/pattern_match_deployments.py
View file

@ -279,7 +279,9 @@ class PatternMatchRouter:
PatternMatchRouter.set_deployment_model_name(
matched_pattern=pattern_match, litellm_deployment_litellm_model=deployment["litellm_params"]["model"]
)
for deployment in self.patterns[pattern]
for deployment in self.patterns.get( # pyright: ignore[reportUnknownMemberType] # untyped deployment dicts
pattern, ()
)
)

62
tests/unit/llms/openrouter/test_openrouter_common_utils.py
View file

@ -32,38 +32,49 @@ def _configure(monkeypatch: pytest.MonkeyPatch, source: str, value: str) -> None
monkeypatch.setenv(source, value)
def test_get_models_lists_only_the_chat_catalog_under_openrouter_keeping_the_org_namespace():
with respx.mock() as respx_mock:
respx_mock.get(f"{DEFAULT_BASE}/models/user").mock(
return_value=_catalog("openai/gpt-5.4-mini", "anthropic/claude-haiku-5-5")
)
assert OpenRouterModelInfo().get_models(api_key="sk-or-test") == [
"openrouter/openai/gpt-5.4-mini",
"openrouter/anthropic/claude-haiku-5-5",
]
assert [call.request.url.path for call in respx_mock.calls] == ["/api/v1/models/user"]
@respx.mock
def test_get_models_lists_both_catalogs_once_each_under_openrouter_keeping_the_org_namespace():
def test_get_models_drops_batch_only_ids_and_keeps_every_other_variant():
respx.get(f"{DEFAULT_BASE}/models/user").mock(
return_value=_catalog("openai/gpt-5.4-mini", "anthropic/claude-haiku-5-5")
)
respx.get(f"{DEFAULT_BASE}/embeddings/models").mock(
return_value=_catalog("openai/text-embedding-3-small", "openai/gpt-5.4-mini")
return_value=_catalog("openai/gpt-5.4-mini:batch", "openai/gpt-5.4-mini", "qwen/qwen3-coder:free")
)
assert OpenRouterModelInfo().get_models(api_key="sk-or-test") == [
"openrouter/openai/gpt-5.4-mini",
"openrouter/anthropic/claude-haiku-5-5",
"openrouter/openai/text-embedding-3-small",
"openrouter/qwen/qwen3-coder:free",
]
@pytest.mark.parametrize(
"api_base",
[
"https://gateway.example/openrouter/api/v1/",
"https://gateway.example/openrouter/api/v1/chat/completions",
"https://gateway.example/openrouter/api/v1/chat/completions/",
],
)
@respx.mock
def test_get_models_keeps_the_api_base_path_and_sends_the_deployment_key():
def test_get_models_keeps_the_api_base_path_and_sends_the_deployment_key(api_base: str):
chat_route: Final = respx.get("https://gateway.example/openrouter/api/v1/models/user").mock(
return_value=_catalog("openai/gpt-5.4-mini")
)
embeddings_route: Final = respx.get("https://gateway.example/openrouter/api/v1/embeddings/models").mock(
return_value=_catalog()
)
assert OpenRouterModelInfo().get_models(
api_key="sk-or-deployment", api_base="https://gateway.example/openrouter/api/v1/"
) == ["openrouter/openai/gpt-5.4-mini"]
assert [route.calls.last.request.headers["authorization"] for route in (chat_route, embeddings_route)] == [
"Bearer sk-or-deployment",
"Bearer sk-or-deployment",
assert OpenRouterModelInfo().get_models(api_key="sk-or-deployment", api_base=api_base) == [
"openrouter/openai/gpt-5.4-mini"
]
assert chat_route.calls.last.request.headers["authorization"] == "Bearer sk-or-deployment"
@pytest.mark.parametrize("winner", range(len(KEY_SOURCES)), ids=KEY_SOURCES)
@ -76,7 +87,6 @@ def test_get_models_sends_the_first_configured_key_in_the_chat_paths_order(
if winner > 0:
_configure(monkeypatch, KEY_SOURCES[winner], "sk-or-winner")
chat_route: Final = respx.get(f"{DEFAULT_BASE}/models/user").mock(return_value=_catalog("qwen/qwen3-coder"))
respx.get(f"{DEFAULT_BASE}/embeddings/models").mock(return_value=_catalog())
assert OpenRouterModelInfo().get_models(api_key="sk-or-winner" if winner == 0 else None) == [
"openrouter/qwen/qwen3-coder"
@ -95,7 +105,6 @@ def test_get_models_calls_the_first_configured_base_in_the_chat_paths_order(
_configure(monkeypatch, BASE_SOURCES[winner], "https://winner.example/api/v1")
expected_base: Final = "https://winner.example/api/v1" if winner < len(BASE_SOURCES) else DEFAULT_BASE
respx.get(f"{expected_base}/models/user").mock(return_value=_catalog("qwen/qwen3-coder"))
respx.get(f"{expected_base}/embeddings/models").mock(return_value=_catalog())
assert OpenRouterModelInfo().get_models(
api_key="sk-or-test", api_base="https://winner.example/api/v1" if winner == 0 else None
@ -114,17 +123,12 @@ def test_get_models_without_any_key_raises_before_calling_openrouter():
assert not chat_route.called
@pytest.mark.parametrize("failing_route", ["models/user", "embeddings/models"])
def test_get_models_raises_when_either_catalog_answers_an_error(failing_route: str):
with respx.mock(assert_all_called=False) as respx_mock:
respx_mock.get(f"{DEFAULT_BASE}/models/user").mock(return_value=_catalog("openai/gpt-5.4-mini"))
respx_mock.get(f"{DEFAULT_BASE}/embeddings/models").mock(return_value=_catalog())
respx_mock.get(f"{DEFAULT_BASE}/{failing_route}").mock(
return_value=httpx.Response(401, json={"error": {"code": 401}})
)
@respx.mock
def test_get_models_raises_when_the_catalog_answers_an_error():
respx.get(f"{DEFAULT_BASE}/models/user").mock(return_value=httpx.Response(401, json={"error": {"code": 401}}))
with pytest.raises(httpx.HTTPStatusError):
OpenRouterModelInfo().get_models(api_key="sk-or-revoked")
with pytest.raises(httpx.HTTPStatusError):
OpenRouterModelInfo().get_models(api_key="sk-or-revoked")
def test_validate_environment_adds_the_resolved_key_and_keeps_the_callers_headers(monkeypatch: pytest.MonkeyPatch):

21
tests/unit/proxy/auth/test_model_checks.py
View file

@ -1156,22 +1156,10 @@ def test_partial_bedrock_wildcard_lists_nothing_when_no_discovered_id_carries_it
(
"openrouter/*",
"openrouter/*",
[
"openrouter/openai/gpt-5.4-mini",
"openrouter/anthropic/claude-haiku-5-5",
"openrouter/openai/text-embedding-3-small",
],
),
(
"or-team/*",
"openrouter/*",
["or-team/openai/gpt-5.4-mini", "or-team/anthropic/claude-haiku-5-5", "or-team/openai/text-embedding-3-small"],
),
(
"openrouter/openai/*",
"openrouter/openai/*",
["openrouter/openai/gpt-5.4-mini", "openrouter/openai/text-embedding-3-small"],
["openrouter/openai/gpt-5.4-mini", "openrouter/anthropic/claude-haiku-5-5"],
),
("or-team/*", "openrouter/*", ["or-team/openai/gpt-5.4-mini", "or-team/anthropic/claude-haiku-5-5"]),
("openrouter/openai/*", "openrouter/openai/*", ["openrouter/openai/gpt-5.4-mini"]),
],
)
@respx.mock
@ -1189,9 +1177,6 @@ def test_openrouter_wildcard_lists_the_live_catalog_with_each_vendor_namespace(
200, json={"data": [{"id": "openai/gpt-5.4-mini"}, {"id": "anthropic/claude-haiku-5-5"}]}
)
)
respx.get("https://openrouter.ai/api/v1/embeddings/models").mock(
return_value=httpx.Response(200, json={"data": [{"id": "openai/text-embedding-3-small"}]})
)
assert (
get_known_models_from_wildcard(wildcard, LiteLLM_Params(model=deployment_model, api_key=f"sk-or-{wildcard}"))

65
tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py
View file

@ -3,10 +3,13 @@ from types import MappingProxyType
from typing import Final
from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
import respx
from fastapi import HTTPException
from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles
import litellm
from litellm import Router
from litellm.caching.caching import DualCache
from litellm.proxy._types import UserAPIKeyAuth
@ -851,3 +854,65 @@ def test_a_team_keeps_its_first_config_file_openai_deployment_over_older_db_ones
assert credentials is not None
assert credentials["api_key"] == "deployment-z-key"
@pytest.mark.parametrize(
("custom_llm_provider", "openrouter_key", "api_key"),
[("openai", "sk-or-files-openai", "openai-key"), ("openrouter", "sk-or-files-openrouter", "sk-or-files-openrouter")],
)
def test_a_provider_credential_lookup_never_lists_an_openrouter_wildcards_live_catalog(
monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, openrouter_key: str, api_key: str
):
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
router: Final = Router(
model_list=[
{
"model_name": "openrouter/*",
"litellm_params": {
"model": "openrouter/*",
"api_key": openrouter_key,
"api_base": "https://openrouter.ai/api/v1",
},
},
{"model_name": "gpt-5.4-mini", "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "openai-key"}},
]
)
with respx.mock(assert_all_called=False) as respx_mock:
catalog_route: Final = respx_mock.get("https://openrouter.ai/api/v1/models/user").mock(
return_value=httpx.Response(200, json={"data": [{"id": "openai/gpt-5.4-mini"}]})
)
credentials: Final = get_team_provider_credentials(
llm_router=router,
user_api_key_dict=UserAPIKeyAuth(api_key="test-key", user_id="test-user"),
custom_llm_provider=custom_llm_provider,
include_accessible_deployments=True,
)
assert credentials is not None
assert (credentials["custom_llm_provider"], credentials["api_key"]) == (custom_llm_provider, api_key)
assert catalog_route.call_count == 0
def test_a_key_allowed_a_wider_wildcard_than_any_deployment_gets_the_matching_deployments_credentials(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(litellm, "check_provider_endpoint", False)
router: Final = Router(
model_list=[
{
"model_name": "anthropic/claude-*",
"litellm_params": {"model": "anthropic/claude-*", "api_key": "anthropic-deployment-key"},
}
]
)
credentials: Final = get_team_provider_credentials(
llm_router=router,
user_api_key_dict=UserAPIKeyAuth(api_key="test-key", user_id="test-user", models=["anthropic/*"]),
custom_llm_provider="anthropic",
include_accessible_deployments=True,
)
assert credentials is not None
assert (credentials["custom_llm_provider"], credentials["api_key"]) == ("anthropic", "anthropic-deployment-key")

69
tests/unit/proxy/proxy_server/test_routes_models.py
View file

@ -9,12 +9,19 @@ Pins (PR2):
from __future__ import annotations
import asyncio
import threading
from dataclasses import dataclass, field
from typing import Final
from unittest.mock import MagicMock
import httpx
import pytest
import respx
import litellm
from litellm.proxy import proxy_server
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy import utils as proxy_utils
from litellm.proxy.utils import create_model_info_response
from litellm.types.router import DeploymentModelListingInfo
@ -514,3 +521,65 @@ def test_codex_service_tiers_resolved_via_internal_team_key(client, auth_as, pat
assert response.status_code == 200
(entry,) = response.json()["models"]
assert (entry["slug"], [tier["id"] for tier in entry["service_tiers"]]) == ("gpt-4-team", ["ultrafast"])
OPENROUTER_BASE: Final = "https://openrouter.ai/api/v1"
@dataclass(frozen=True, slots=True)
class _CatalogHeldUntilTheLoopRuns:
requested: threading.Event = field(default_factory=threading.Event)
released: threading.Event = field(default_factory=threading.Event)
def answer(self, _request: httpx.Request) -> httpx.Response:
self.requested.set()
if not self.released.wait(timeout=2):
return httpx.Response(504)
return httpx.Response(200, json={"data": [{"id": "openai/gpt-5.4-mini"}]})
async def release_from_the_loop(self) -> None:
assert await asyncio.to_thread(self.requested.wait, 5)
self.released.set()
@pytest.mark.parametrize("scope", [None, "expand"])
async def test_v1_models_fetches_a_live_openrouter_catalog_without_blocking_the_event_loop(
monkeypatch: pytest.MonkeyPatch, scope: str | None
):
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
monkeypatch.setattr(proxy_server, "general_settings", {})
monkeypatch.setattr(proxy_server, "user_model", None)
monkeypatch.setattr(proxy_server, "prisma_client", None)
monkeypatch.setattr(
proxy_server,
"llm_router",
litellm.Router(
model_list=[
{
"model_name": "openrouter/*",
"litellm_params": {
"model": "openrouter/*",
"api_key": f"sk-or-v1-models-off-loop-{scope}",
"api_base": OPENROUTER_BASE,
},
}
]
),
)
catalog: Final = _CatalogHeldUntilTheLoopRuns()
with respx.mock() as respx_mock:
catalog_route: Final = respx_mock.get(f"{OPENROUTER_BASE}/models/user").mock(side_effect=catalog.answer)
listing: Final = asyncio.ensure_future(
proxy_server.model_list(
user_api_key_dict=UserAPIKeyAuth(
api_key="sk-test-key", user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN
),
scope=scope,
)
)
await catalog.release_from_the_loop()
response: Final = await listing
assert catalog_route.calls.last.response.status_code == 200
assert [model["id"] for model in response["data"]] == ["openrouter/openai/gpt-5.4-mini"]

62
tests/unit/proxy/utils/helpers/test_model_access.py
View file

@ -1,10 +1,16 @@
import asyncio
import threading
from dataclasses import dataclass, field
from typing import Final
from unittest.mock import MagicMock
import httpx
import pytest
import respx
from fastapi import HTTPException
import litellm
from litellm import ModelResponse
from litellm import ModelResponse, Router
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.utils import (
create_model_info_response,
@ -650,3 +656,57 @@ async def test_listing_is_unchanged_without_an_agent_or_without_attached_groups(
resolve_agent_ceiling=_agent_ceiling(None),
)
assert (plain_key, agent_without_groups) == (["model-a", "model-b"], ["model-a", "model-b"])
OPENROUTER_BASE: Final = "https://openrouter.ai/api/v1"
@dataclass(frozen=True, slots=True)
class _CatalogHeldUntilTheLoopRuns:
requested: threading.Event = field(default_factory=threading.Event)
released: threading.Event = field(default_factory=threading.Event)
def answer(self, _request: httpx.Request) -> httpx.Response:
self.requested.set()
if not self.released.wait(timeout=2):
return httpx.Response(504)
return httpx.Response(200, json={"data": [{"id": "openai/gpt-5.4-mini"}]})
async def release_from_the_loop(self) -> None:
assert await asyncio.to_thread(self.requested.wait, 5)
self.released.set()
async def test_get_available_models_for_user_fetches_a_live_catalog_without_blocking_the_event_loop(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
catalog: Final = _CatalogHeldUntilTheLoopRuns()
router: Final = Router(
model_list=[
{
"model_name": "openrouter/*",
"litellm_params": {
"model": "openrouter/*",
"api_key": "sk-or-available-models-off-loop",
"api_base": OPENROUTER_BASE,
},
}
]
)
with respx.mock() as respx_mock:
catalog_route: Final = respx_mock.get(f"{OPENROUTER_BASE}/models/user").mock(side_effect=catalog.answer)
listing: Final = asyncio.ensure_future(
get_available_models_for_user(
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test-key", user_id="user-1"),
llm_router=router,
general_settings={},
user_model=None,
)
)
await catalog.release_from_the_loop()
models: Final = await listing
assert catalog_route.calls.last.response.status_code == 200
assert models == ["openrouter/openai/gpt-5.4-mini"]