fix(proxy): list wildcard models off the event loop, so a hung provider no longer stalls other requests
Some checks failed
LiteLLM Rust DB / db-queries (push) Waiting to run
LiteLLM Rust / rust-lint (push) Waiting to run
LiteLLM Rust / rust-test (push) Waiting to run
LiteLLM Rust / rust-wheel (push) Waiting to run
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled

This commit is contained in:
mateo-berri 2026-10-28 23:03:45 -07:00
parent 7a575fea15
commit d69da65a6a
5 changed files with 75 additions and 20 deletions

View file

@ -12398,7 +12398,8 @@ async def model_list(
# Get complete model list including wildcard routes if requested
from litellm.proxy.auth.model_checks import get_complete_model_list
all_models = get_complete_model_list(
all_models = await asyncio.to_thread(
get_complete_model_list,
key_models=[],
team_models=[],
proxy_model_list=proxy_model_list,
@ -17751,7 +17752,7 @@ async def model_info_v1(
alias_models: Final = copy.deepcopy(llm_router.get_model_list_from_model_alias())
all_models.extend(alias_models)
all_models = expand_wildcard_deployments_for_model_info(all_models, llm_router)
all_models = await asyncio.to_thread(expand_wildcard_deployments_for_model_info, all_models, llm_router)
allowed_model_names: Final = await _get_v1_model_info_allowed_model_names_with_access_groups(
user_api_key_dict=user_api_key_dict,
@ -18086,7 +18087,8 @@ async def model_group_info(
LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
)
all_models_str: Final = (
get_complete_model_list(
await asyncio.to_thread(
get_complete_model_list,
key_models=(),
team_models=(),
proxy_model_list=llm_router.get_model_names(),

View file

@ -9517,7 +9517,8 @@ async def _agent_access_group_visible_models(
if not granted:
return frozenset()
return frozenset(
get_complete_model_list(
await asyncio.to_thread(
get_complete_model_list,
key_models=granted,
team_models=(),
proxy_model_list=proxy_model_list,
@ -9652,7 +9653,8 @@ async def get_available_models_for_user(
)
# Get complete model list
complete_models: Final = get_complete_model_list(
complete_models: Final = await asyncio.to_thread(
get_complete_model_list,
key_models=granted_key_models,
team_models=granted_team_models,
proxy_model_list=proxy_model_list,

View file

@ -8333,8 +8333,8 @@ _HUGGINGFACE_TOKENIZER_LOAD_FAILURES: Final[_HuggingFaceTokenizerLoadFailureCach
class AvailableModelsCache(InMemoryCache):
def __init__(self, ttl_seconds: int = 300, max_size: int = 1000):
super().__init__(ttl_seconds, max_size)
def __init__(self, ttl_seconds: int = 300, max_size: int = 1000, clock: Callable[[], float] | None = None):
super().__init__(ttl_seconds, max_size, clock=clock)
self._env_hash: str | None = None
def _get_env_hash(self) -> str:
@ -8443,9 +8443,10 @@ def _get_valid_models_from_provider_api(
provider_config: BaseLLMModelInfo,
custom_llm_provider: str,
litellm_params: LiteLLM_Params | None = None,
model_cache: AvailableModelsCache = _model_cache,
) -> list[str]:
try:
cached_result: Final = _model_cache.get_cached_model_info(custom_llm_provider, litellm_params)
cached_result: Final = model_cache.get_cached_model_info(custom_llm_provider, litellm_params)
if cached_result is not None:
return cached_result
@ -8453,12 +8454,12 @@ def _get_valid_models_from_provider_api(
litellm_params=_provider_listing_params(custom_llm_provider, litellm_params)
)
_model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models)
model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models)
return models
except httpx.TimeoutException as e:
verbose_logger.warning("Timed out getting valid models: %s", e)
with suppress(ValueError):
_model_cache.set_cached_model_info(
model_cache.set_cached_model_info(
custom_llm_provider, litellm_params, [], ttl=PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS
)
return []

View file

@ -540,6 +540,54 @@ def test_hosted_vllm_wildcard_listing_does_not_wait_the_completion_timeout_on_a_
assert read_timeout <= NON_LLM_CONNECTION_TIMEOUT
@pytest.mark.asyncio
@respx.mock
async def test_a_hung_wildcard_listing_leaves_the_event_loop_free_for_other_requests(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import asyncio
import litellm
from litellm import Router
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.utils import get_available_models_for_user
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
listing_started: Final = threading.Event()
other_request_served: Final = threading.Event()
def listing_that_answers_once_another_request_is_served(request: httpx.Request) -> httpx.Response:
listing_started.set()
if other_request_served.wait(timeout=3):
return httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})
return httpx.Response(503)
respx.get("http://vllm-stalled.test:8000/v1/models").mock(
side_effect=listing_that_answers_once_another_request_is_served
)
router: Final = Router(
model_list=[
{
"model_name": "hosted_vllm/*",
"litellm_params": {"model": "hosted_vllm/*", "api_base": "http://vllm-stalled.test:8000/v1"},
}
]
)
listing: Final = asyncio.create_task(
get_available_models_for_user(
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", models=[]),
llm_router=router,
general_settings={},
user_model=None,
)
)
assert await asyncio.to_thread(listing_started.wait, 3)
other_request_served.set()
assert "hosted_vllm/qwen2.5" in await listing
@respx.mock
def test_hosted_vllm_wildcard_listing_sends_the_deployment_extra_headers(monkeypatch: pytest.MonkeyPatch) -> None:
import litellm

View file

@ -10582,23 +10582,22 @@ def test_get_valid_models_from_provider_cache_invalidation(monkeypatch):
@respx.mock
def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_ttl(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import time
def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_ttl() -> None:
from litellm.constants import PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS
from litellm.llms.vllm.common_utils import HostedVLLMModelInfo
from litellm.types.router import LiteLLM_Params
from litellm.utils import AvailableModelsCache, _get_valid_models_from_provider_api
models_route: Final = respx.get("http://vllm-hung.test:8000/v1/models").mock(
side_effect=httpx.ReadTimeout("vLLM did not answer")
)
litellm_params: Final = LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-hung.test:8000/v1")
failed_at: Final = time.time()
clock: Final = Mock(return_value=1000.0)
model_cache: Final = AvailableModelsCache(clock=clock)
def list_models() -> list[str]:
return get_valid_models(
check_provider_endpoint=True, custom_llm_provider="hosted_vllm", litellm_params=litellm_params
return _get_valid_models_from_provider_api(
HostedVLLMModelInfo(), "hosted_vllm", litellm_params, model_cache=model_cache
)
assert list_models() == []
@ -10606,13 +10605,15 @@ def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_tt
assert models_route.call_count == 1
models_route.mock(side_effect=None, return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}))
monkeypatch.setattr(time, "time", lambda: failed_at + PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS + 1)
clock.return_value = 1000.0 + PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS - 1
assert list_models() == []
assert models_route.call_count == 1
clock.return_value = 1000.0 + PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS + 1
assert list_models() == ["qwen2.5"]
assert models_route.call_count == 2
@pytest.mark.parametrize(
("host", "first_answer"),
[("vllm-erroring.test", httpx.Response(503)), ("vllm-refusing.test", httpx.ConnectError("connection refused"))],
@ -10638,6 +10639,7 @@ def test_a_provider_model_listing_that_fails_fast_is_tried_again_on_the_next_cal
assert list_models() == ["qwen2.5"]
assert models_route.call_count == 2
def test_delta_tool_calls_sequential_indices():
"""
Test that multiple tool calls without explicit indices receive sequential indices.