fix(proxy): answer model lists from cached listings first and share one listing per server
Some checks are pending
Cost map pins recheck / rerun-cost-map-guard (push) Waiting to run
LiteLLM Rust DB / db-queries (push) Waiting to run
LiteLLM Rust / rust-lint (push) Waiting to run
LiteLLM Rust / rust-test (push) Waiting to run
LiteLLM Rust / rust-wheel (push) Waiting to run

This commit is contained in:
mateo-berri 2026-10-29 10:30:22 -07:00
parent 83d52c7646
commit 42ee801409
8 changed files with 194 additions and 21 deletions

View file

@ -613,6 +613,7 @@ NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int(
) # timeout for adjacent services (e.g. jwt auth)
PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS: Final = 60
PROVIDER_MODEL_DISCOVERY_MAX_THREADS: Final = 4
PROVIDER_MODEL_DISCOVERY_LISTING_LOCK_STRIPES: Final = 64
MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000)
MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048)
MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024

View file

@ -22,7 +22,7 @@ from litellm.router_utils.fallback_event_handlers import get_fallback_model_grou
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager, get_valid_models
from litellm.utils import ProviderConfigManager, answer_from_cached_provider_listings, get_valid_models
LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders)
@ -40,6 +40,11 @@ async def run_model_discovery(
*args: _P.args,
**kwargs: _P.kwargs, # kwargs-ok: forwards the discovery function's own arguments
) -> _R:
cached_answer, needs_provider_listing = answer_from_cached_provider_listings(
functools.partial(discover, *args, **kwargs)
)
if not needs_provider_listing:
return cached_answer
context: Final = contextvars.copy_context()
return await asyncio.get_running_loop().run_in_executor(
_MODEL_DISCOVERY_EXECUTOR, lambda: context.run(discover, *args, **kwargs)

View file

@ -509,6 +509,18 @@ _UNKNOWN_CREATION_TIME: Final = datetime.max.replace(tzinfo=timezone.utc)
class _RouterDeployment(TypedDict):
model_name: ReadOnly[NotRequired[str]]
model_info: ReadOnly[Mapping[str, object]]
litellm_params: ReadOnly[Mapping[str, object]]
def _deployment_serves_provider(llm_router: "Router", deployment: _RouterDeployment, custom_llm_provider: str) -> bool:
litellm_params: Final = deployment["litellm_params"]
model: Final = litellm_params.get("model")
deployment_provider: Final = litellm_params.get("custom_llm_provider")
return (
isinstance(model, str)
and llm_router.credential_provider(deployment_provider if isinstance(deployment_provider, str) else None, model)
== custom_llm_provider
)
def _team_deployment_rank(model_info: Mapping[str, object], deployment_id: str) -> tuple[datetime, str]:
@ -632,6 +644,11 @@ def get_team_provider_credentials(
if not include_accessible_deployments:
return None
if not any(
_deployment_serves_provider(llm_router, deployment, custom_llm_provider)
for deployment in llm_router.model_list or []
):
return None
# 2. Fall back to deployments the caller is allowed to access. The key's
# effective allowlist (sentinels and access groups already expanded by

View file

@ -12379,18 +12379,20 @@ class Router:
credentials.pop("litellm_credential_name", None)
credentials["model"] = deployment.litellm_params.model
# Add custom_llm_provider
if deployment.litellm_params.custom_llm_provider:
credentials["custom_llm_provider"] = deployment.litellm_params.custom_llm_provider
elif "/" in deployment.litellm_params.model:
# Extract provider from "provider/model" format
credentials["custom_llm_provider"] = deployment.litellm_params.model.split("/")[0]
else:
credentials["custom_llm_provider"] = "openai" # default
credentials["custom_llm_provider"] = self.credential_provider(
deployment.litellm_params.custom_llm_provider, deployment.litellm_params.model
)
return credentials
@staticmethod
def credential_provider(custom_llm_provider: str | None, model: str) -> str:
if custom_llm_provider:
return custom_llm_provider
if "/" in model:
return model.split("/")[0]
return "openai"
@overload
def get_router_model_info(
self,

View file

@ -79,6 +79,7 @@ from litellm.constants import (
MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE,
NON_INFERENCE_CALL_TYPES,
OPENAI_EMBEDDING_PARAMS,
PROVIDER_MODEL_DISCOVERY_LISTING_LOCK_STRIPES,
PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS,
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
TOOL_CHOICE_OBJECT_TOKEN_COUNT,
@ -8371,6 +8372,13 @@ class AvailableModelsCache(InMemoryCache):
super().__init__(ttl_seconds, max_size, clock=clock)
self._env_hash: str | None = None
self._listing_lock = threading.Lock()
self._single_flight_locks: Final = tuple(
threading.RLock() for _ in range(PROVIDER_MODEL_DISCOVERY_LISTING_LOCK_STRIPES)
)
def single_flight_lock(self, custom_llm_provider: str, litellm_params: LiteLLM_Params | None) -> threading.RLock:
cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params)
return self._single_flight_locks[hash(cache_key) % len(self._single_flight_locks)]
def _get_env_hash(self) -> str:
"""Create a hash of relevant environment variables"""
@ -8451,6 +8459,22 @@ class AvailableModelsCache(InMemoryCache):
# Global cache instance
_model_cache: Final = AvailableModelsCache()
_UNCACHED_PROVIDER_LISTING: Final[contextvars.ContextVar[threading.Event | None]] = contextvars.ContextVar(
"uncached_provider_listing", default=None
)
_Answer: Final = TypeVar("_Answer")
def answer_from_cached_provider_listings(compute: Callable[[], _Answer]) -> tuple[_Answer, bool]:
uncached_listing: Final = threading.Event()
answer: Final = contextvars.copy_context().run(_answer_marking_uncached_listings, uncached_listing, compute)
return answer, uncached_listing.is_set()
def _answer_marking_uncached_listings(uncached_listing: threading.Event, compute: Callable[[], _Answer]) -> _Answer:
_UNCACHED_PROVIDER_LISTING.set(uncached_listing)
return compute()
def _infer_valid_provider_from_env_vars(
custom_llm_provider: str | None = None,
@ -8504,21 +8528,40 @@ def _get_valid_models_from_provider_api(
if cached_result is not None:
return cached_result
uncached_listing: Final = _UNCACHED_PROVIDER_LISTING.get()
if uncached_listing is not None:
uncached_listing.set()
return []
with model_cache.single_flight_lock(custom_llm_provider, litellm_params):
return _list_provider_models_unless_cached(
provider_config, custom_llm_provider, litellm_params, model_cache
)
except Exception as e:
verbose_logger.warning("Error getting valid models: %s", e)
return []
def _list_provider_models_unless_cached(
provider_config: BaseLLMModelInfo,
custom_llm_provider: str,
litellm_params: LiteLLM_Params | None,
model_cache: AvailableModelsCache,
) -> list[str]:
cached_while_waiting: Final = model_cache.get_cached_model_info(custom_llm_provider, litellm_params)
if cached_while_waiting is not None:
return cached_while_waiting
try:
models: Final = provider_config.discover_models(
litellm_params=_provider_listing_params(custom_llm_provider, litellm_params)
)
model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models)
return models
except httpx.TimeoutException as e:
verbose_logger.warning("Timed out getting valid models: %s", e)
verbose_logger.warning("Error getting valid models: %s", e)
model_cache.cache_timed_out_listing_unless_listed(
custom_llm_provider, litellm_params, ttl=PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS
)
return model_cache.get_cached_model_info(custom_llm_provider, litellm_params) or []
except Exception as e:
verbose_logger.warning("Error getting valid models: %s", e)
return []
model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models)
return models
def get_valid_models(

View file

@ -588,6 +588,67 @@ async def test_a_hung_wildcard_listing_leaves_the_event_loop_free_for_other_requ
assert "hosted_vllm/qwen2.5" in await listing
@pytest.mark.asyncio
@respx.mock
async def test_a_model_list_that_needs_no_listing_answers_while_hung_listings_hold_every_discovery_thread(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import asyncio
import litellm
from litellm import Router
from litellm.constants import PROVIDER_MODEL_DISCOVERY_MAX_THREADS
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.utils import get_available_models_for_user
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
listing_started: Final = threading.Event()
hung_server_released: Final = threading.Event()
def hung_listing(request: httpx.Request) -> httpx.Response:
listing_started.set()
hung_server_released.wait(timeout=10)
return httpx.Response(503)
respx.get("http://vllm-holding-every-thread.test:8000/v1/models").mock(side_effect=hung_listing)
router: Final = Router(
model_list=[
{
"model_name": "stuck/*",
"litellm_params": {
"model": "hosted_vllm/*",
"api_base": "http://vllm-holding-every-thread.test:8000/v1",
},
},
{
"model_name": "qwen-open",
"litellm_params": {"model": "hosted_vllm/qwen2.5", "api_base": "http://vllm-open.test:8000/v1"},
},
]
)
def list_models(models: list[str]) -> asyncio.Task[list[str]]:
return asyncio.create_task(
get_available_models_for_user(
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", models=models),
llm_router=router,
general_settings={},
user_model=None,
)
)
try:
admin_listings: Final = tuple(list_models([]) for _ in range(PROVIDER_MODEL_DISCOVERY_MAX_THREADS))
assert await asyncio.to_thread(listing_started.wait, 3)
await asyncio.sleep(0.1)
assert await asyncio.wait_for(list_models(["qwen-open"]), timeout=2) == ["qwen-open"]
finally:
hung_server_released.set()
for admin_models in await asyncio.gather(*admin_listings):
assert "qwen-open" in admin_models
@respx.mock
def test_a_hung_wildcard_listing_leaves_the_default_executor_free_for_completion_calls(
monkeypatch: pytest.MonkeyPatch,

View file

@ -802,10 +802,11 @@ async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_eve
"model": "hosted_vllm/*",
"api_base": "http://vllm-credential-lookup.test:8000/v1",
},
}
},
{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", "api_key": "openai-key"}},
]
),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", models=["hosted_vllm/*"]),
custom_llm_provider="openai",
)
)
@ -834,14 +835,20 @@ async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_eve
("gpt-4o", "openai/gpt-4o"),
("api_base", "http://vllm-route-match.test:8000/v1"),
),
(
"anthropic",
"http://vllm-beside-no-anthropic-deployment.test:8000/v1",
("gpt-4o", "openai/gpt-4o"),
("api_key", None),
),
],
)
async def test_a_credential_lookup_a_configured_model_answers_never_lists_the_wildcard_server(
async def test_a_credential_lookup_never_lists_a_wildcard_server_whose_models_cannot_change_the_answer(
monkeypatch: pytest.MonkeyPatch,
custom_llm_provider: str,
vllm_api_base: str,
openai_deployment: tuple[str, str],
applied_credential: tuple[str, str],
applied_credential: tuple[str, str | None],
) -> None:
import litellm

View file

@ -10616,6 +10616,43 @@ def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_tt
assert models_route.call_count == 2
@respx.mock
def test_concurrent_listings_of_one_server_share_a_single_request() -> None:
import time
from litellm.llms.vllm.common_utils import HostedVLLMModelInfo
from litellm.types.router import LiteLLM_Params
from litellm.utils import AvailableModelsCache, _get_valid_models_from_provider_api
listing_started: Final = threading.Event()
server_released: Final = threading.Event()
def slow_listing(request: httpx.Request) -> httpx.Response:
listing_started.set()
server_released.wait(timeout=5)
return httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})
models_route: Final = respx.get("http://vllm-slow.test:8000/v1/models").mock(side_effect=slow_listing)
litellm_params: Final = LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-slow.test:8000/v1")
model_cache: Final = AvailableModelsCache(clock=Mock(return_value=1000.0))
def list_models() -> list[str]:
return _get_valid_models_from_provider_api(
HostedVLLMModelInfo(), "hosted_vllm", litellm_params, model_cache=model_cache
)
with ThreadPoolExecutor(max_workers=4) as callers:
listings: Final = tuple(callers.submit(list_models) for _ in range(4))
try:
assert listing_started.wait(timeout=3)
time.sleep(0.3)
finally:
server_released.set()
assert [listing.result(timeout=5) for listing in listings] == [["qwen2.5"]] * 4
assert models_route.call_count == 1
@respx.mock
def test_a_listing_that_times_out_keeps_the_models_a_concurrent_listing_cached() -> None:
from litellm.llms.vllm.common_utils import HostedVLLMModelInfo