diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3b36ea5afce..84b05b88a41 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12398,7 +12398,8 @@ async def model_list( # Get complete model list including wildcard routes if requested from litellm.proxy.auth.model_checks import get_complete_model_list - all_models = get_complete_model_list( + all_models = await asyncio.to_thread( + get_complete_model_list, key_models=[], team_models=[], proxy_model_list=proxy_model_list, @@ -17751,7 +17752,7 @@ async def model_info_v1( alias_models: Final = copy.deepcopy(llm_router.get_model_list_from_model_alias()) all_models.extend(alias_models) - all_models = expand_wildcard_deployments_for_model_info(all_models, llm_router) + all_models = await asyncio.to_thread(expand_wildcard_deployments_for_model_info, all_models, llm_router) allowed_model_names: Final = await _get_v1_model_info_allowed_model_names_with_access_groups( user_api_key_dict=user_api_key_dict, @@ -18086,7 +18087,8 @@ async def model_group_info( LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, ) all_models_str: Final = ( - get_complete_model_list( + await asyncio.to_thread( + get_complete_model_list, key_models=(), team_models=(), proxy_model_list=llm_router.get_model_names(), diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0ddfc5ce62f..cc3c7280160 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -9517,7 +9517,8 @@ async def _agent_access_group_visible_models( if not granted: return frozenset() return frozenset( - get_complete_model_list( + await asyncio.to_thread( + get_complete_model_list, key_models=granted, team_models=(), proxy_model_list=proxy_model_list, @@ -9652,7 +9653,8 @@ async def get_available_models_for_user( ) # Get complete model list - complete_models: Final = get_complete_model_list( + complete_models: Final = await asyncio.to_thread( + get_complete_model_list, key_models=granted_key_models, team_models=granted_team_models, proxy_model_list=proxy_model_list, diff --git a/litellm/utils.py b/litellm/utils.py index d89f405223e..71e3ff882e0 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8333,8 +8333,8 @@ _HUGGINGFACE_TOKENIZER_LOAD_FAILURES: Final[_HuggingFaceTokenizerLoadFailureCach class AvailableModelsCache(InMemoryCache): - def __init__(self, ttl_seconds: int = 300, max_size: int = 1000): - super().__init__(ttl_seconds, max_size) + def __init__(self, ttl_seconds: int = 300, max_size: int = 1000, clock: Callable[[], float] | None = None): + super().__init__(ttl_seconds, max_size, clock=clock) self._env_hash: str | None = None def _get_env_hash(self) -> str: @@ -8443,9 +8443,10 @@ def _get_valid_models_from_provider_api( provider_config: BaseLLMModelInfo, custom_llm_provider: str, litellm_params: LiteLLM_Params | None = None, + model_cache: AvailableModelsCache = _model_cache, ) -> list[str]: try: - cached_result: Final = _model_cache.get_cached_model_info(custom_llm_provider, litellm_params) + cached_result: Final = model_cache.get_cached_model_info(custom_llm_provider, litellm_params) if cached_result is not None: return cached_result @@ -8453,12 +8454,12 @@ def _get_valid_models_from_provider_api( litellm_params=_provider_listing_params(custom_llm_provider, litellm_params) ) - _model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) + model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) return models except httpx.TimeoutException as e: verbose_logger.warning("Timed out getting valid models: %s", e) with suppress(ValueError): - _model_cache.set_cached_model_info( + model_cache.set_cached_model_info( custom_llm_provider, litellm_params, [], ttl=PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS ) return [] diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 8415f593213..fa2f044f0e0 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -540,6 +540,54 @@ def test_hosted_vllm_wildcard_listing_does_not_wait_the_completion_timeout_on_a_ assert read_timeout <= NON_LLM_CONNECTION_TIMEOUT +@pytest.mark.asyncio +@respx.mock +async def test_a_hung_wildcard_listing_leaves_the_event_loop_free_for_other_requests( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import asyncio + + import litellm + from litellm import Router + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.utils import get_available_models_for_user + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + listing_started: Final = threading.Event() + other_request_served: Final = threading.Event() + + def listing_that_answers_once_another_request_is_served(request: httpx.Request) -> httpx.Response: + listing_started.set() + if other_request_served.wait(timeout=3): + return httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + return httpx.Response(503) + + respx.get("http://vllm-stalled.test:8000/v1/models").mock( + side_effect=listing_that_answers_once_another_request_is_served + ) + router: Final = Router( + model_list=[ + { + "model_name": "hosted_vllm/*", + "litellm_params": {"model": "hosted_vllm/*", "api_base": "http://vllm-stalled.test:8000/v1"}, + } + ] + ) + + listing: Final = asyncio.create_task( + get_available_models_for_user( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", models=[]), + llm_router=router, + general_settings={}, + user_model=None, + ) + ) + assert await asyncio.to_thread(listing_started.wait, 3) + other_request_served.set() + + assert "hosted_vllm/qwen2.5" in await listing + + @respx.mock def test_hosted_vllm_wildcard_listing_sends_the_deployment_extra_headers(monkeypatch: pytest.MonkeyPatch) -> None: import litellm diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 04e4d7e2b51..effc2d5e8a2 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10582,23 +10582,22 @@ def test_get_valid_models_from_provider_cache_invalidation(monkeypatch): @respx.mock -def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_ttl( - monkeypatch: pytest.MonkeyPatch, -) -> None: - import time - +def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_ttl() -> None: from litellm.constants import PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS + from litellm.llms.vllm.common_utils import HostedVLLMModelInfo from litellm.types.router import LiteLLM_Params + from litellm.utils import AvailableModelsCache, _get_valid_models_from_provider_api models_route: Final = respx.get("http://vllm-hung.test:8000/v1/models").mock( side_effect=httpx.ReadTimeout("vLLM did not answer") ) litellm_params: Final = LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-hung.test:8000/v1") - failed_at: Final = time.time() + clock: Final = Mock(return_value=1000.0) + model_cache: Final = AvailableModelsCache(clock=clock) def list_models() -> list[str]: - return get_valid_models( - check_provider_endpoint=True, custom_llm_provider="hosted_vllm", litellm_params=litellm_params + return _get_valid_models_from_provider_api( + HostedVLLMModelInfo(), "hosted_vllm", litellm_params, model_cache=model_cache ) assert list_models() == [] @@ -10606,13 +10605,15 @@ def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_tt assert models_route.call_count == 1 models_route.mock(side_effect=None, return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})) - monkeypatch.setattr(time, "time", lambda: failed_at + PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS + 1) + clock.return_value = 1000.0 + PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS - 1 + assert list_models() == [] + assert models_route.call_count == 1 + clock.return_value = 1000.0 + PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS + 1 assert list_models() == ["qwen2.5"] assert models_route.call_count == 2 - @pytest.mark.parametrize( ("host", "first_answer"), [("vllm-erroring.test", httpx.Response(503)), ("vllm-refusing.test", httpx.ConnectError("connection refused"))], @@ -10638,6 +10639,7 @@ def test_a_provider_model_listing_that_fails_fast_is_tried_again_on_the_next_cal assert list_models() == ["qwen2.5"] assert models_route.call_count == 2 + def test_delta_tool_calls_sequential_indices(): """ Test that multiple tool calls without explicit indices receive sequential indices.