From 42ee8014092e8e53eeeaf352f6c2fd51aff9b026 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:30:22 -0700 Subject: [PATCH] fix(proxy): answer model lists from cached listings first and share one listing per server --- litellm/constants.py | 1 + litellm/proxy/auth/model_checks.py | 7 ++- .../openai_files_endpoints/common_utils.py | 17 ++++++ litellm/router.py | 20 +++--- litellm/utils.py | 57 ++++++++++++++--- tests/unit/proxy/auth/test_model_checks.py | 61 +++++++++++++++++++ .../test_files_common_utils.py | 15 +++-- tests/unit/test_utils.py | 37 +++++++++++ 8 files changed, 194 insertions(+), 21 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index d2ac3d62377..d416be0586f 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -613,6 +613,7 @@ NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int( ) # timeout for adjacent services (e.g. jwt auth) PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS: Final = 60 PROVIDER_MODEL_DISCOVERY_MAX_THREADS: Final = 4 +PROVIDER_MODEL_DISCOVERY_LISTING_LOCK_STRIPES: Final = 64 MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000) MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048) MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024 diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 7739cd9941e..163ec2e3141 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -22,7 +22,7 @@ from litellm.router_utils.fallback_event_handlers import get_fallback_model_grou from litellm.router_utils.pattern_match_deployments import PatternMatchRouter from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params from litellm.types.utils import LlmProviders -from litellm.utils import ProviderConfigManager, get_valid_models +from litellm.utils import ProviderConfigManager, answer_from_cached_provider_listings, get_valid_models LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders) @@ -40,6 +40,11 @@ async def run_model_discovery( *args: _P.args, **kwargs: _P.kwargs, # kwargs-ok: forwards the discovery function's own arguments ) -> _R: + cached_answer, needs_provider_listing = answer_from_cached_provider_listings( + functools.partial(discover, *args, **kwargs) + ) + if not needs_provider_listing: + return cached_answer context: Final = contextvars.copy_context() return await asyncio.get_running_loop().run_in_executor( _MODEL_DISCOVERY_EXECUTOR, lambda: context.run(discover, *args, **kwargs) diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 371456f3991..40837919025 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -509,6 +509,18 @@ _UNKNOWN_CREATION_TIME: Final = datetime.max.replace(tzinfo=timezone.utc) class _RouterDeployment(TypedDict): model_name: ReadOnly[NotRequired[str]] model_info: ReadOnly[Mapping[str, object]] + litellm_params: ReadOnly[Mapping[str, object]] + + +def _deployment_serves_provider(llm_router: "Router", deployment: _RouterDeployment, custom_llm_provider: str) -> bool: + litellm_params: Final = deployment["litellm_params"] + model: Final = litellm_params.get("model") + deployment_provider: Final = litellm_params.get("custom_llm_provider") + return ( + isinstance(model, str) + and llm_router.credential_provider(deployment_provider if isinstance(deployment_provider, str) else None, model) + == custom_llm_provider + ) def _team_deployment_rank(model_info: Mapping[str, object], deployment_id: str) -> tuple[datetime, str]: @@ -632,6 +644,11 @@ def get_team_provider_credentials( if not include_accessible_deployments: return None + if not any( + _deployment_serves_provider(llm_router, deployment, custom_llm_provider) + for deployment in llm_router.model_list or [] + ): + return None # 2. Fall back to deployments the caller is allowed to access. The key's # effective allowlist (sentinels and access groups already expanded by diff --git a/litellm/router.py b/litellm/router.py index 02ad6275b82..992996464ff 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -12379,18 +12379,20 @@ class Router: credentials.pop("litellm_credential_name", None) credentials["model"] = deployment.litellm_params.model - - # Add custom_llm_provider - if deployment.litellm_params.custom_llm_provider: - credentials["custom_llm_provider"] = deployment.litellm_params.custom_llm_provider - elif "/" in deployment.litellm_params.model: - # Extract provider from "provider/model" format - credentials["custom_llm_provider"] = deployment.litellm_params.model.split("/")[0] - else: - credentials["custom_llm_provider"] = "openai" # default + credentials["custom_llm_provider"] = self.credential_provider( + deployment.litellm_params.custom_llm_provider, deployment.litellm_params.model + ) return credentials + @staticmethod + def credential_provider(custom_llm_provider: str | None, model: str) -> str: + if custom_llm_provider: + return custom_llm_provider + if "/" in model: + return model.split("/")[0] + return "openai" + @overload def get_router_model_info( self, diff --git a/litellm/utils.py b/litellm/utils.py index 2350e88d11d..bf0c5ca958a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -79,6 +79,7 @@ from litellm.constants import ( MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE, NON_INFERENCE_CALL_TYPES, OPENAI_EMBEDDING_PARAMS, + PROVIDER_MODEL_DISCOVERY_LISTING_LOCK_STRIPES, PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS, PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, TOOL_CHOICE_OBJECT_TOKEN_COUNT, @@ -8371,6 +8372,13 @@ class AvailableModelsCache(InMemoryCache): super().__init__(ttl_seconds, max_size, clock=clock) self._env_hash: str | None = None self._listing_lock = threading.Lock() + self._single_flight_locks: Final = tuple( + threading.RLock() for _ in range(PROVIDER_MODEL_DISCOVERY_LISTING_LOCK_STRIPES) + ) + + def single_flight_lock(self, custom_llm_provider: str, litellm_params: LiteLLM_Params | None) -> threading.RLock: + cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params) + return self._single_flight_locks[hash(cache_key) % len(self._single_flight_locks)] def _get_env_hash(self) -> str: """Create a hash of relevant environment variables""" @@ -8451,6 +8459,22 @@ class AvailableModelsCache(InMemoryCache): # Global cache instance _model_cache: Final = AvailableModelsCache() +_UNCACHED_PROVIDER_LISTING: Final[contextvars.ContextVar[threading.Event | None]] = contextvars.ContextVar( + "uncached_provider_listing", default=None +) +_Answer: Final = TypeVar("_Answer") + + +def answer_from_cached_provider_listings(compute: Callable[[], _Answer]) -> tuple[_Answer, bool]: + uncached_listing: Final = threading.Event() + answer: Final = contextvars.copy_context().run(_answer_marking_uncached_listings, uncached_listing, compute) + return answer, uncached_listing.is_set() + + +def _answer_marking_uncached_listings(uncached_listing: threading.Event, compute: Callable[[], _Answer]) -> _Answer: + _UNCACHED_PROVIDER_LISTING.set(uncached_listing) + return compute() + def _infer_valid_provider_from_env_vars( custom_llm_provider: str | None = None, @@ -8504,21 +8528,40 @@ def _get_valid_models_from_provider_api( if cached_result is not None: return cached_result + uncached_listing: Final = _UNCACHED_PROVIDER_LISTING.get() + if uncached_listing is not None: + uncached_listing.set() + return [] + with model_cache.single_flight_lock(custom_llm_provider, litellm_params): + return _list_provider_models_unless_cached( + provider_config, custom_llm_provider, litellm_params, model_cache + ) + except Exception as e: + verbose_logger.warning("Error getting valid models: %s", e) + return [] + + +def _list_provider_models_unless_cached( + provider_config: BaseLLMModelInfo, + custom_llm_provider: str, + litellm_params: LiteLLM_Params | None, + model_cache: AvailableModelsCache, +) -> list[str]: + cached_while_waiting: Final = model_cache.get_cached_model_info(custom_llm_provider, litellm_params) + if cached_while_waiting is not None: + return cached_while_waiting + try: models: Final = provider_config.discover_models( litellm_params=_provider_listing_params(custom_llm_provider, litellm_params) ) - - model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) - return models except httpx.TimeoutException as e: - verbose_logger.warning("Timed out getting valid models: %s", e) + verbose_logger.warning("Error getting valid models: %s", e) model_cache.cache_timed_out_listing_unless_listed( custom_llm_provider, litellm_params, ttl=PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS ) return model_cache.get_cached_model_info(custom_llm_provider, litellm_params) or [] - except Exception as e: - verbose_logger.warning("Error getting valid models: %s", e) - return [] + model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) + return models def get_valid_models( diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 058ef08aef4..6ed87afc77e 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -588,6 +588,67 @@ async def test_a_hung_wildcard_listing_leaves_the_event_loop_free_for_other_requ assert "hosted_vllm/qwen2.5" in await listing +@pytest.mark.asyncio +@respx.mock +async def test_a_model_list_that_needs_no_listing_answers_while_hung_listings_hold_every_discovery_thread( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import asyncio + + import litellm + from litellm import Router + from litellm.constants import PROVIDER_MODEL_DISCOVERY_MAX_THREADS + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.utils import get_available_models_for_user + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + listing_started: Final = threading.Event() + hung_server_released: Final = threading.Event() + + def hung_listing(request: httpx.Request) -> httpx.Response: + listing_started.set() + hung_server_released.wait(timeout=10) + return httpx.Response(503) + + respx.get("http://vllm-holding-every-thread.test:8000/v1/models").mock(side_effect=hung_listing) + router: Final = Router( + model_list=[ + { + "model_name": "stuck/*", + "litellm_params": { + "model": "hosted_vllm/*", + "api_base": "http://vllm-holding-every-thread.test:8000/v1", + }, + }, + { + "model_name": "qwen-open", + "litellm_params": {"model": "hosted_vllm/qwen2.5", "api_base": "http://vllm-open.test:8000/v1"}, + }, + ] + ) + + def list_models(models: list[str]) -> asyncio.Task[list[str]]: + return asyncio.create_task( + get_available_models_for_user( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", models=models), + llm_router=router, + general_settings={}, + user_model=None, + ) + ) + + try: + admin_listings: Final = tuple(list_models([]) for _ in range(PROVIDER_MODEL_DISCOVERY_MAX_THREADS)) + assert await asyncio.to_thread(listing_started.wait, 3) + await asyncio.sleep(0.1) + + assert await asyncio.wait_for(list_models(["qwen-open"]), timeout=2) == ["qwen-open"] + finally: + hung_server_released.set() + for admin_models in await asyncio.gather(*admin_listings): + assert "qwen-open" in admin_models + + @respx.mock def test_a_hung_wildcard_listing_leaves_the_default_executor_free_for_completion_calls( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py index 6d1fb00308f..15ce9841cde 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py @@ -802,10 +802,11 @@ async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_eve "model": "hosted_vllm/*", "api_base": "http://vllm-credential-lookup.test:8000/v1", }, - } + }, + {"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", "api_key": "openai-key"}}, ] ), - user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", models=["hosted_vllm/*"]), custom_llm_provider="openai", ) ) @@ -834,14 +835,20 @@ async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_eve ("gpt-4o", "openai/gpt-4o"), ("api_base", "http://vllm-route-match.test:8000/v1"), ), + ( + "anthropic", + "http://vllm-beside-no-anthropic-deployment.test:8000/v1", + ("gpt-4o", "openai/gpt-4o"), + ("api_key", None), + ), ], ) -async def test_a_credential_lookup_a_configured_model_answers_never_lists_the_wildcard_server( +async def test_a_credential_lookup_never_lists_a_wildcard_server_whose_models_cannot_change_the_answer( monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, vllm_api_base: str, openai_deployment: tuple[str, str], - applied_credential: tuple[str, str], + applied_credential: tuple[str, str | None], ) -> None: import litellm diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 9e41b61ca5c..75cdf4babce 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10616,6 +10616,43 @@ def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_tt assert models_route.call_count == 2 +@respx.mock +def test_concurrent_listings_of_one_server_share_a_single_request() -> None: + import time + + from litellm.llms.vllm.common_utils import HostedVLLMModelInfo + from litellm.types.router import LiteLLM_Params + from litellm.utils import AvailableModelsCache, _get_valid_models_from_provider_api + + listing_started: Final = threading.Event() + server_released: Final = threading.Event() + + def slow_listing(request: httpx.Request) -> httpx.Response: + listing_started.set() + server_released.wait(timeout=5) + return httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + + models_route: Final = respx.get("http://vllm-slow.test:8000/v1/models").mock(side_effect=slow_listing) + litellm_params: Final = LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-slow.test:8000/v1") + model_cache: Final = AvailableModelsCache(clock=Mock(return_value=1000.0)) + + def list_models() -> list[str]: + return _get_valid_models_from_provider_api( + HostedVLLMModelInfo(), "hosted_vllm", litellm_params, model_cache=model_cache + ) + + with ThreadPoolExecutor(max_workers=4) as callers: + listings: Final = tuple(callers.submit(list_models) for _ in range(4)) + try: + assert listing_started.wait(timeout=3) + time.sleep(0.3) + finally: + server_released.set() + assert [listing.result(timeout=5) for listing in listings] == [["qwen2.5"]] * 4 + + assert models_route.call_count == 1 + + @respx.mock def test_a_listing_that_times_out_keeps_the_models_a_concurrent_listing_cached() -> None: from litellm.llms.vllm.common_utils import HostedVLLMModelInfo