fix(proxy): run model discovery on its own thread pool and keep credential lookups off the event loop

This commit is contained in:
mateo-berri 2026-10-29 05:57:09 -07:00
parent 42303949e1
commit 22e417363a
11 changed files with 190 additions and 47 deletions

View file

@ -612,6 +612,7 @@ NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int(
"NON_LLM_CONNECTION_TIMEOUT", 15
) # timeout for adjacent services (e.g. jwt auth)
PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS: Final = 60
PROVIDER_MODEL_DISCOVERY_MAX_THREADS: Final = 4
MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000)
MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048)
MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024

View file

@ -1,14 +1,18 @@
# What is this?
## Common checks for /v1/models and `/model/info`
import asyncio
import contextvars
import copy
import functools
import itertools
import re
from collections.abc import Callable, Mapping, Sequence
from typing import Final
from concurrent.futures import ThreadPoolExecutor
from typing import Final, ParamSpec, TypeVar
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import PROVIDER_MODEL_DISCOVERY_MAX_THREADS
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth
@ -22,6 +26,26 @@ from litellm.utils import ProviderConfigManager, get_valid_models
LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders)
_P: Final = ParamSpec("_P")
_R: Final = TypeVar("_R")
_MODEL_DISCOVERY_EXECUTOR: Final = ThreadPoolExecutor(
max_workers=PROVIDER_MODEL_DISCOVERY_MAX_THREADS, thread_name_prefix="litellm-model-discovery"
)
async def run_model_discovery(
discover: Callable[_P, _R],
/,
*args: _P.args,
**kwargs: _P.kwargs, # kwargs-ok: forwards the discovery function's own arguments
) -> _R:
context: Final = contextvars.copy_context()
return await asyncio.get_running_loop().run_in_executor(
_MODEL_DISCOVERY_EXECUTOR, lambda: context.run(discover, *args, **kwargs)
)
_CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields)

View file

@ -489,7 +489,7 @@ async def create_batch(
verbose_proxy_logger.debug("Created batch using model: %s", model_param)
else:
# SCENARIO 3: Fallback to custom_llm_provider (uses env variables)
apply_team_provider_credentials(
await apply_team_provider_credentials(
data=cast(dict, _create_batch_data), # cast-ok: TypedDict is a dict at runtime
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,
@ -772,7 +772,7 @@ async def retrieve_batch(
or get_custom_llm_provider_from_request_query(request=request)
)
custom_llm_provider: Final = requested_provider or "openai"
apply_team_provider_credentials(
await apply_team_provider_credentials(
data=data,
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,
@ -986,7 +986,7 @@ async def list_batches(
or get_custom_llm_provider_from_request_query(request=request)
or "openai"
)
apply_team_provider_credentials(
await apply_team_provider_credentials(
data=data,
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,
@ -1209,7 +1209,7 @@ async def cancel_batch(
# Extract batch_id from data to avoid "multiple values for keyword argument" error
# data was cast from CancelBatchRequest which already contains batch_id
data.pop("batch_id", None)
apply_team_provider_credentials(
await apply_team_provider_credentials(
data=data,
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,

View file

@ -527,7 +527,7 @@ class _PROXY_BatchRateLimiter(CustomLogger):
return True
return False
def _resolve_batch_input_file_fetch_params(
async def _resolve_batch_input_file_fetch_params(
self,
file_id: str,
custom_llm_provider: str,
@ -541,6 +541,7 @@ class _PROXY_BatchRateLimiter(CustomLogger):
without decoding them, ``afile_content`` is called with the encoded ID
and the upstream provider returns 404.
"""
from litellm.proxy.auth.model_checks import run_model_discovery
from litellm.proxy.openai_files_endpoints.common_utils import (
decode_model_from_file_id,
explicit_request_credentials,
@ -591,7 +592,8 @@ class _PROXY_BatchRateLimiter(CustomLogger):
return file_id, fetch_kwargs
team_credentials: Final = (
get_team_provider_credentials(
await run_model_discovery(
get_team_provider_credentials,
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,
custom_llm_provider=custom_llm_provider,
@ -948,7 +950,7 @@ class _PROXY_BatchRateLimiter(CustomLogger):
user_api_key_dict=user_api_key_dict,
)
else:
provider_file_id, fetch_kwargs = self._resolve_batch_input_file_fetch_params(
provider_file_id, fetch_kwargs = await self._resolve_batch_input_file_fetch_params(
file_id=file_id,
custom_llm_provider=custom_llm_provider,
data=data or {},

View file

@ -530,7 +530,7 @@ def get_team_provider_credentials(
custom_llm_provider: str,
*,
include_accessible_deployments: bool = True,
) -> dict | None:
) -> dict[str, object] | None:
"""
Resolve upstream credentials for a provider-scoped file operation
(e.g. GET /v1/files), which doesn't pin a model.
@ -704,7 +704,7 @@ def explicit_request_credentials(data: Mapping[str, object]) -> Mapping[str, str
return MappingProxyType({"api_key": api_key, "api_base": api_base})
def apply_team_provider_credentials(
async def apply_team_provider_credentials(
data: dict, # mutable-ok: credentials are merged into the request payload in place, same contract as prepare_data_with_credentials
llm_router: Optional["Router"],
user_api_key_dict: "UserAPIKeyAuth",
@ -719,7 +719,10 @@ def apply_team_provider_credentials(
``forward_llm_provider_auth_headers``) replaces the deployment's, and the
request's ``api_base`` replaces the deployment's only alongside that key.
"""
credentials: Final = get_team_provider_credentials(
from litellm.proxy.auth.model_checks import run_model_discovery
credentials: Final = await run_model_discovery(
get_team_provider_credentials,
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,
custom_llm_provider=custom_llm_provider,

View file

@ -524,7 +524,7 @@ async def route_create_file(
_create_file_request=_create_file_request,
)
else:
apply_team_provider_credentials(
await apply_team_provider_credentials(
data=cast(dict, _create_file_request), # cast-ok: TypedDict is a plain dict at runtime; merged in place
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,
@ -1131,7 +1131,7 @@ async def get_file_content(
)
if not should_route:
apply_team_provider_credentials(
await apply_team_provider_credentials(
data=data,
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,
@ -1397,7 +1397,7 @@ async def get_file(
# Remove file_id from data to avoid "multiple values for keyword argument" error
# data was initialized with {"file_id": file_id}
data.pop("file_id", None)
apply_team_provider_credentials(
await apply_team_provider_credentials(
data=data,
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,
@ -1616,7 +1616,7 @@ async def delete_file(
)
else:
data.pop("file_id", None)
apply_team_provider_credentials(
await apply_team_provider_credentials(
data=data,
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,
@ -2018,7 +2018,7 @@ async def list_files(
)
else:
resolved_custom_llm_provider: Final = custom_llm_provider or "openai"
apply_team_provider_credentials(
await apply_team_provider_credentials(
data=data,
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,

View file

@ -440,6 +440,7 @@ from litellm.proxy.auth.model_checks import (
get_key_models,
get_mcp_server_ids,
get_team_models,
run_model_discovery,
)
from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy
from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401, RUF100 # legacy module exports
@ -12418,7 +12419,7 @@ async def model_list(
# Get complete model list including wildcard routes if requested
from litellm.proxy.auth.model_checks import get_complete_model_list
all_models = await asyncio.to_thread(
all_models = await run_model_discovery(
get_complete_model_list,
key_models=[],
team_models=[],
@ -17854,7 +17855,7 @@ async def model_info_v1(
alias_models: Final = copy.deepcopy(llm_router.get_model_list_from_model_alias())
all_models.extend(alias_models)
all_models = await asyncio.to_thread(expand_wildcard_deployments_for_model_info, all_models, llm_router)
all_models = await run_model_discovery(expand_wildcard_deployments_for_model_info, all_models, llm_router)
allowed_model_names: Final = await _get_v1_model_info_allowed_model_names_with_access_groups(
user_api_key_dict=user_api_key_dict,
@ -18189,7 +18190,7 @@ async def model_group_info(
LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
)
all_models_str: Final = (
await asyncio.to_thread(
await run_model_discovery(
get_complete_model_list,
key_models=(),
team_models=(),

View file

@ -9499,7 +9499,7 @@ async def _agent_access_group_visible_models(
) -> frozenset[str] | None:
"""Models an agent key may still list once its attached access groups cap it, ``None`` when
nothing caps it, so ``/v1/models`` never advertises a model the same key would be denied on."""
from litellm.proxy.auth.model_checks import get_complete_model_list, get_team_models
from litellm.proxy.auth.model_checks import get_complete_model_list, get_team_models, run_model_discovery
if not user_api_key_dict.agent_id:
return None
@ -9519,7 +9519,7 @@ async def _agent_access_group_visible_models(
if not granted:
return frozenset()
return frozenset(
await asyncio.to_thread(
await run_model_discovery(
get_complete_model_list,
key_models=granted,
team_models=(),
@ -9575,6 +9575,7 @@ async def get_available_models_for_user(
get_complete_model_list,
get_key_models,
get_team_models,
run_model_discovery,
)
# Get proxy model list and access groups
@ -9655,7 +9656,7 @@ async def get_available_models_for_user(
)
# Get complete model list
complete_models: Final = await asyncio.to_thread(
complete_models: Final = await run_model_discovery(
get_complete_model_list,
key_models=granted_key_models,
team_models=granted_team_models,

View file

@ -588,6 +588,59 @@ async def test_a_hung_wildcard_listing_leaves_the_event_loop_free_for_other_requ
assert "hosted_vllm/qwen2.5" in await listing
@respx.mock
def test_a_hung_wildcard_listing_leaves_the_default_executor_free_for_completion_calls(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import asyncio
from concurrent.futures import ThreadPoolExecutor
import litellm
from litellm import Router
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.utils import get_available_models_for_user
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
listing_started: Final = threading.Event()
completion_served: Final = threading.Event()
def listing_that_answers_once_a_completion_is_served(request: httpx.Request) -> httpx.Response:
listing_started.set()
if completion_served.wait(timeout=3):
return httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})
return httpx.Response(503)
respx.get("http://vllm-default-executor.test:8000/v1/models").mock(
side_effect=listing_that_answers_once_a_completion_is_served
)
router: Final = Router(
model_list=[
{
"model_name": "hosted_vllm/*",
"litellm_params": {"model": "hosted_vllm/*", "api_base": "http://vllm-default-executor.test:8000/v1"},
}
]
)
async def list_models_while_a_completion_runs_on_the_default_executor() -> list[str]:
loop: Final = asyncio.get_running_loop()
loop.set_default_executor(ThreadPoolExecutor(max_workers=1))
listing: Final = asyncio.create_task(
get_available_models_for_user(
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", models=[]),
llm_router=router,
general_settings={},
user_model=None,
)
)
with ThreadPoolExecutor(max_workers=1) as waiter:
assert await loop.run_in_executor(waiter, listing_started.wait, 3)
await asyncio.wait_for(loop.run_in_executor(None, completion_served.set), timeout=1)
return await listing
assert "hosted_vllm/qwen2.5" in asyncio.run(list_models_while_a_completion_runs_on_the_default_executor())
@respx.mock
def test_hosted_vllm_wildcard_listing_sends_the_deployment_extra_headers(monkeypatch: pytest.MonkeyPatch) -> None:
import litellm

View file

@ -1274,7 +1274,8 @@ def test_should_not_skip_and_reuses_descriptors_when_limits_present():
assert returned is descriptors
def test_resolve_fetch_params_uses_request_model_credentials():
@pytest.mark.asyncio
async def test_resolve_fetch_params_uses_request_model_credentials():
rate_limiter = _make_rate_limiter()
with (
patch("litellm.proxy.proxy_server.llm_router", MagicMock()),
@ -1288,7 +1289,7 @@ def test_resolve_fetch_params_uses_request_model_credentials():
),
):
provider_file_id, fetch_kwargs = (
rate_limiter._resolve_batch_input_file_fetch_params(
await rate_limiter._resolve_batch_input_file_fetch_params(
file_id="file-plain-openai",
custom_llm_provider="openai",
data={"model": "my-vllm-batch"},
@ -1300,7 +1301,8 @@ def test_resolve_fetch_params_uses_request_model_credentials():
assert fetch_kwargs["api_base"] == "http://vllm:8000/v1"
def test_resolve_fetch_params_fails_open_on_credential_lookup_error():
@pytest.mark.asyncio
async def test_resolve_fetch_params_fails_open_on_credential_lookup_error():
rate_limiter = _make_rate_limiter()
with (
patch("litellm.proxy.proxy_server.llm_router", MagicMock()),
@ -1310,7 +1312,7 @@ def test_resolve_fetch_params_fails_open_on_credential_lookup_error():
),
):
provider_file_id, fetch_kwargs = (
rate_limiter._resolve_batch_input_file_fetch_params(
await rate_limiter._resolve_batch_input_file_fetch_params(
file_id="file-plain-openai",
custom_llm_provider="openai",
data={"model": "my-vllm-batch"},
@ -1320,7 +1322,8 @@ def test_resolve_fetch_params_fails_open_on_credential_lookup_error():
assert fetch_kwargs == {"custom_llm_provider": "openai"}
def test_resolve_fetch_params_model_embedded_fails_open_on_credential_error():
@pytest.mark.asyncio
async def test_resolve_fetch_params_model_embedded_fails_open_on_credential_error():
import base64
rate_limiter = _make_rate_limiter()
@ -1342,7 +1345,7 @@ def test_resolve_fetch_params_model_embedded_fails_open_on_credential_error():
),
):
provider_file_id, fetch_kwargs = (
rate_limiter._resolve_batch_input_file_fetch_params(
await rate_limiter._resolve_batch_input_file_fetch_params(
file_id=encoded_file_id,
custom_llm_provider="openai",
data={},

View file

@ -1,9 +1,13 @@
import asyncio
import threading
from datetime import datetime, timezone
from types import MappingProxyType
from typing import Final
from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
import respx
from fastapi import HTTPException
from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles
@ -764,26 +768,77 @@ def test_a_request_key_that_cannot_be_an_http_header_is_a_bad_request(api_key: s
assert api_key not in str(refused.value.detail)
@pytest.mark.asyncio
@respx.mock
async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_event_loop_free(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import litellm
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
listing_started: Final = threading.Event()
other_request_served: Final = threading.Event()
listing_answered: Final = threading.Event()
def listing_that_answers_once_another_request_is_served(request: httpx.Request) -> httpx.Response:
listing_started.set()
if other_request_served.wait(timeout=3):
listing_answered.set()
return httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})
return httpx.Response(503)
respx.get("http://vllm-credential-lookup.test:8000/v1/models").mock(
side_effect=listing_that_answers_once_another_request_is_served
)
data: Final[dict[str, object]] = {}
lookup: Final = asyncio.create_task(
apply_team_provider_credentials(
data=data,
llm_router=Router(
model_list=[
{
"model_name": "hosted_vllm/*",
"litellm_params": {
"model": "hosted_vllm/*",
"api_base": "http://vllm-credential-lookup.test:8000/v1",
},
}
]
),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
custom_llm_provider="hosted_vllm",
)
)
assert await asyncio.to_thread(listing_started.wait, 3)
other_request_served.set()
await lookup
assert listing_answered.is_set()
assert data.get("api_base") == "http://vllm-credential-lookup.test:8000/v1"
def _apply_team_a_openai_credentials(data: dict[str, object]) -> None:
apply_team_provider_credentials(
data=data,
llm_router=Router(
model_list=[
{
"model_name": "openai/*",
"litellm_params": {
"model": "openai/*",
"api_key": "team-openai-key",
"api_base": "https://team.example/v1",
},
"model_info": {"id": "team-a-openai", "team_id": "team-a"},
}
]
),
user_api_key_dict=UserAPIKeyAuth(
api_key="test-key", user_id="test-user", team_id="team-a", team_models=["openai/*"], models=["gpt-4o"]
),
custom_llm_provider="openai",
asyncio.run(
apply_team_provider_credentials(
data=data,
llm_router=Router(
model_list=[
{
"model_name": "openai/*",
"litellm_params": {
"model": "openai/*",
"api_key": "team-openai-key",
"api_base": "https://team.example/v1",
},
"model_info": {"id": "team-a-openai", "team_id": "team-a"},
}
]
),
user_api_key_dict=UserAPIKeyAuth(
api_key="test-key", user_id="test-user", team_id="team-a", team_models=["openai/*"], models=["gpt-4o"]
),
custom_llm_provider="openai",
)
)