diff --git a/litellm/constants.py b/litellm/constants.py index baa213170ba..d2ac3d62377 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -612,6 +612,7 @@ NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int( "NON_LLM_CONNECTION_TIMEOUT", 15 ) # timeout for adjacent services (e.g. jwt auth) PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS: Final = 60 +PROVIDER_MODEL_DISCOVERY_MAX_THREADS: Final = 4 MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000) MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048) MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024 diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index b74a4f586b0..e6f6d60855a 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -1,14 +1,18 @@ # What is this? ## Common checks for /v1/models and `/model/info` +import asyncio +import contextvars import copy import functools import itertools import re from collections.abc import Callable, Mapping, Sequence -from typing import Final +from concurrent.futures import ThreadPoolExecutor +from typing import Final, ParamSpec, TypeVar import litellm from litellm._logging import verbose_proxy_logger +from litellm.constants import PROVIDER_MODEL_DISCOVERY_MAX_THREADS from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth @@ -22,6 +26,26 @@ from litellm.utils import ProviderConfigManager, get_valid_models LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders) +_P: Final = ParamSpec("_P") +_R: Final = TypeVar("_R") + +_MODEL_DISCOVERY_EXECUTOR: Final = ThreadPoolExecutor( + max_workers=PROVIDER_MODEL_DISCOVERY_MAX_THREADS, thread_name_prefix="litellm-model-discovery" +) + + +async def run_model_discovery( + discover: Callable[_P, _R], + /, + *args: _P.args, + **kwargs: _P.kwargs, # kwargs-ok: forwards the discovery function's own arguments +) -> _R: + context: Final = contextvars.copy_context() + return await asyncio.get_running_loop().run_in_executor( + _MODEL_DISCOVERY_EXECUTOR, lambda: context.run(discover, *args, **kwargs) + ) + + _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index 763737191ef..f650ac3e3c2 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -489,7 +489,7 @@ async def create_batch( verbose_proxy_logger.debug("Created batch using model: %s", model_param) else: # SCENARIO 3: Fallback to custom_llm_provider (uses env variables) - apply_team_provider_credentials( + await apply_team_provider_credentials( data=cast(dict, _create_batch_data), # cast-ok: TypedDict is a dict at runtime llm_router=llm_router, user_api_key_dict=user_api_key_dict, @@ -772,7 +772,7 @@ async def retrieve_batch( or get_custom_llm_provider_from_request_query(request=request) ) custom_llm_provider: Final = requested_provider or "openai" - apply_team_provider_credentials( + await apply_team_provider_credentials( data=data, llm_router=llm_router, user_api_key_dict=user_api_key_dict, @@ -986,7 +986,7 @@ async def list_batches( or get_custom_llm_provider_from_request_query(request=request) or "openai" ) - apply_team_provider_credentials( + await apply_team_provider_credentials( data=data, llm_router=llm_router, user_api_key_dict=user_api_key_dict, @@ -1209,7 +1209,7 @@ async def cancel_batch( # Extract batch_id from data to avoid "multiple values for keyword argument" error # data was cast from CancelBatchRequest which already contains batch_id data.pop("batch_id", None) - apply_team_provider_credentials( + await apply_team_provider_credentials( data=data, llm_router=llm_router, user_api_key_dict=user_api_key_dict, diff --git a/litellm/proxy/hooks/batch_rate_limiter.py b/litellm/proxy/hooks/batch_rate_limiter.py index 94762232e0b..f86338208ad 100644 --- a/litellm/proxy/hooks/batch_rate_limiter.py +++ b/litellm/proxy/hooks/batch_rate_limiter.py @@ -527,7 +527,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): return True return False - def _resolve_batch_input_file_fetch_params( + async def _resolve_batch_input_file_fetch_params( self, file_id: str, custom_llm_provider: str, @@ -541,6 +541,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): without decoding them, ``afile_content`` is called with the encoded ID and the upstream provider returns 404. """ + from litellm.proxy.auth.model_checks import run_model_discovery from litellm.proxy.openai_files_endpoints.common_utils import ( decode_model_from_file_id, explicit_request_credentials, @@ -591,7 +592,8 @@ class _PROXY_BatchRateLimiter(CustomLogger): return file_id, fetch_kwargs team_credentials: Final = ( - get_team_provider_credentials( + await run_model_discovery( + get_team_provider_credentials, llm_router=llm_router, user_api_key_dict=user_api_key_dict, custom_llm_provider=custom_llm_provider, @@ -948,7 +950,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): user_api_key_dict=user_api_key_dict, ) else: - provider_file_id, fetch_kwargs = self._resolve_batch_input_file_fetch_params( + provider_file_id, fetch_kwargs = await self._resolve_batch_input_file_fetch_params( file_id=file_id, custom_llm_provider=custom_llm_provider, data=data or {}, diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index f7289445d3a..907fd11347c 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -530,7 +530,7 @@ def get_team_provider_credentials( custom_llm_provider: str, *, include_accessible_deployments: bool = True, -) -> dict | None: +) -> dict[str, object] | None: """ Resolve upstream credentials for a provider-scoped file operation (e.g. GET /v1/files), which doesn't pin a model. @@ -704,7 +704,7 @@ def explicit_request_credentials(data: Mapping[str, object]) -> Mapping[str, str return MappingProxyType({"api_key": api_key, "api_base": api_base}) -def apply_team_provider_credentials( +async def apply_team_provider_credentials( data: dict, # mutable-ok: credentials are merged into the request payload in place, same contract as prepare_data_with_credentials llm_router: Optional["Router"], user_api_key_dict: "UserAPIKeyAuth", @@ -719,7 +719,10 @@ def apply_team_provider_credentials( ``forward_llm_provider_auth_headers``) replaces the deployment's, and the request's ``api_base`` replaces the deployment's only alongside that key. """ - credentials: Final = get_team_provider_credentials( + from litellm.proxy.auth.model_checks import run_model_discovery + + credentials: Final = await run_model_discovery( + get_team_provider_credentials, llm_router=llm_router, user_api_key_dict=user_api_key_dict, custom_llm_provider=custom_llm_provider, diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 5d63c3db222..73f8fb470ec 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -524,7 +524,7 @@ async def route_create_file( _create_file_request=_create_file_request, ) else: - apply_team_provider_credentials( + await apply_team_provider_credentials( data=cast(dict, _create_file_request), # cast-ok: TypedDict is a plain dict at runtime; merged in place llm_router=llm_router, user_api_key_dict=user_api_key_dict, @@ -1131,7 +1131,7 @@ async def get_file_content( ) if not should_route: - apply_team_provider_credentials( + await apply_team_provider_credentials( data=data, llm_router=llm_router, user_api_key_dict=user_api_key_dict, @@ -1397,7 +1397,7 @@ async def get_file( # Remove file_id from data to avoid "multiple values for keyword argument" error # data was initialized with {"file_id": file_id} data.pop("file_id", None) - apply_team_provider_credentials( + await apply_team_provider_credentials( data=data, llm_router=llm_router, user_api_key_dict=user_api_key_dict, @@ -1616,7 +1616,7 @@ async def delete_file( ) else: data.pop("file_id", None) - apply_team_provider_credentials( + await apply_team_provider_credentials( data=data, llm_router=llm_router, user_api_key_dict=user_api_key_dict, @@ -2018,7 +2018,7 @@ async def list_files( ) else: resolved_custom_llm_provider: Final = custom_llm_provider or "openai" - apply_team_provider_credentials( + await apply_team_provider_credentials( data=data, llm_router=llm_router, user_api_key_dict=user_api_key_dict, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 34fd1efefbb..ada3035baf8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -440,6 +440,7 @@ from litellm.proxy.auth.model_checks import ( get_key_models, get_mcp_server_ids, get_team_models, + run_model_discovery, ) from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401, RUF100 # legacy module exports @@ -12418,7 +12419,7 @@ async def model_list( # Get complete model list including wildcard routes if requested from litellm.proxy.auth.model_checks import get_complete_model_list - all_models = await asyncio.to_thread( + all_models = await run_model_discovery( get_complete_model_list, key_models=[], team_models=[], @@ -17854,7 +17855,7 @@ async def model_info_v1( alias_models: Final = copy.deepcopy(llm_router.get_model_list_from_model_alias()) all_models.extend(alias_models) - all_models = await asyncio.to_thread(expand_wildcard_deployments_for_model_info, all_models, llm_router) + all_models = await run_model_discovery(expand_wildcard_deployments_for_model_info, all_models, llm_router) allowed_model_names: Final = await _get_v1_model_info_allowed_model_names_with_access_groups( user_api_key_dict=user_api_key_dict, @@ -18189,7 +18190,7 @@ async def model_group_info( LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, ) all_models_str: Final = ( - await asyncio.to_thread( + await run_model_discovery( get_complete_model_list, key_models=(), team_models=(), diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index d6916d5be04..a2f0b202dfc 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -9499,7 +9499,7 @@ async def _agent_access_group_visible_models( ) -> frozenset[str] | None: """Models an agent key may still list once its attached access groups cap it, ``None`` when nothing caps it, so ``/v1/models`` never advertises a model the same key would be denied on.""" - from litellm.proxy.auth.model_checks import get_complete_model_list, get_team_models + from litellm.proxy.auth.model_checks import get_complete_model_list, get_team_models, run_model_discovery if not user_api_key_dict.agent_id: return None @@ -9519,7 +9519,7 @@ async def _agent_access_group_visible_models( if not granted: return frozenset() return frozenset( - await asyncio.to_thread( + await run_model_discovery( get_complete_model_list, key_models=granted, team_models=(), @@ -9575,6 +9575,7 @@ async def get_available_models_for_user( get_complete_model_list, get_key_models, get_team_models, + run_model_discovery, ) # Get proxy model list and access groups @@ -9655,7 +9656,7 @@ async def get_available_models_for_user( ) # Get complete model list - complete_models: Final = await asyncio.to_thread( + complete_models: Final = await run_model_discovery( get_complete_model_list, key_models=granted_key_models, team_models=granted_team_models, diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 4033d2a30f5..058ef08aef4 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -588,6 +588,59 @@ async def test_a_hung_wildcard_listing_leaves_the_event_loop_free_for_other_requ assert "hosted_vllm/qwen2.5" in await listing +@respx.mock +def test_a_hung_wildcard_listing_leaves_the_default_executor_free_for_completion_calls( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import asyncio + from concurrent.futures import ThreadPoolExecutor + + import litellm + from litellm import Router + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.utils import get_available_models_for_user + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + listing_started: Final = threading.Event() + completion_served: Final = threading.Event() + + def listing_that_answers_once_a_completion_is_served(request: httpx.Request) -> httpx.Response: + listing_started.set() + if completion_served.wait(timeout=3): + return httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + return httpx.Response(503) + + respx.get("http://vllm-default-executor.test:8000/v1/models").mock( + side_effect=listing_that_answers_once_a_completion_is_served + ) + router: Final = Router( + model_list=[ + { + "model_name": "hosted_vllm/*", + "litellm_params": {"model": "hosted_vllm/*", "api_base": "http://vllm-default-executor.test:8000/v1"}, + } + ] + ) + + async def list_models_while_a_completion_runs_on_the_default_executor() -> list[str]: + loop: Final = asyncio.get_running_loop() + loop.set_default_executor(ThreadPoolExecutor(max_workers=1)) + listing: Final = asyncio.create_task( + get_available_models_for_user( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", models=[]), + llm_router=router, + general_settings={}, + user_model=None, + ) + ) + with ThreadPoolExecutor(max_workers=1) as waiter: + assert await loop.run_in_executor(waiter, listing_started.wait, 3) + await asyncio.wait_for(loop.run_in_executor(None, completion_served.set), timeout=1) + return await listing + + assert "hosted_vllm/qwen2.5" in asyncio.run(list_models_while_a_completion_runs_on_the_default_executor()) + + @respx.mock def test_hosted_vllm_wildcard_listing_sends_the_deployment_extra_headers(monkeypatch: pytest.MonkeyPatch) -> None: import litellm diff --git a/tests/unit/proxy/hooks/test_batch_file_validation.py b/tests/unit/proxy/hooks/test_batch_file_validation.py index 8eeee2e6468..dca7b0c7071 100644 --- a/tests/unit/proxy/hooks/test_batch_file_validation.py +++ b/tests/unit/proxy/hooks/test_batch_file_validation.py @@ -1274,7 +1274,8 @@ def test_should_not_skip_and_reuses_descriptors_when_limits_present(): assert returned is descriptors -def test_resolve_fetch_params_uses_request_model_credentials(): +@pytest.mark.asyncio +async def test_resolve_fetch_params_uses_request_model_credentials(): rate_limiter = _make_rate_limiter() with ( patch("litellm.proxy.proxy_server.llm_router", MagicMock()), @@ -1288,7 +1289,7 @@ def test_resolve_fetch_params_uses_request_model_credentials(): ), ): provider_file_id, fetch_kwargs = ( - rate_limiter._resolve_batch_input_file_fetch_params( + await rate_limiter._resolve_batch_input_file_fetch_params( file_id="file-plain-openai", custom_llm_provider="openai", data={"model": "my-vllm-batch"}, @@ -1300,7 +1301,8 @@ def test_resolve_fetch_params_uses_request_model_credentials(): assert fetch_kwargs["api_base"] == "http://vllm:8000/v1" -def test_resolve_fetch_params_fails_open_on_credential_lookup_error(): +@pytest.mark.asyncio +async def test_resolve_fetch_params_fails_open_on_credential_lookup_error(): rate_limiter = _make_rate_limiter() with ( patch("litellm.proxy.proxy_server.llm_router", MagicMock()), @@ -1310,7 +1312,7 @@ def test_resolve_fetch_params_fails_open_on_credential_lookup_error(): ), ): provider_file_id, fetch_kwargs = ( - rate_limiter._resolve_batch_input_file_fetch_params( + await rate_limiter._resolve_batch_input_file_fetch_params( file_id="file-plain-openai", custom_llm_provider="openai", data={"model": "my-vllm-batch"}, @@ -1320,7 +1322,8 @@ def test_resolve_fetch_params_fails_open_on_credential_lookup_error(): assert fetch_kwargs == {"custom_llm_provider": "openai"} -def test_resolve_fetch_params_model_embedded_fails_open_on_credential_error(): +@pytest.mark.asyncio +async def test_resolve_fetch_params_model_embedded_fails_open_on_credential_error(): import base64 rate_limiter = _make_rate_limiter() @@ -1342,7 +1345,7 @@ def test_resolve_fetch_params_model_embedded_fails_open_on_credential_error(): ), ): provider_file_id, fetch_kwargs = ( - rate_limiter._resolve_batch_input_file_fetch_params( + await rate_limiter._resolve_batch_input_file_fetch_params( file_id=encoded_file_id, custom_llm_provider="openai", data={}, diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py index f6c6b51fc93..dc86253e49a 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py @@ -1,9 +1,13 @@ +import asyncio +import threading from datetime import datetime, timezone from types import MappingProxyType from typing import Final from unittest.mock import AsyncMock, MagicMock +import httpx import pytest +import respx from fastapi import HTTPException from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles @@ -764,26 +768,77 @@ def test_a_request_key_that_cannot_be_an_http_header_is_a_bad_request(api_key: s assert api_key not in str(refused.value.detail) +@pytest.mark.asyncio +@respx.mock +async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_event_loop_free( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + listing_started: Final = threading.Event() + other_request_served: Final = threading.Event() + listing_answered: Final = threading.Event() + + def listing_that_answers_once_another_request_is_served(request: httpx.Request) -> httpx.Response: + listing_started.set() + if other_request_served.wait(timeout=3): + listing_answered.set() + return httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + return httpx.Response(503) + + respx.get("http://vllm-credential-lookup.test:8000/v1/models").mock( + side_effect=listing_that_answers_once_another_request_is_served + ) + data: Final[dict[str, object]] = {} + lookup: Final = asyncio.create_task( + apply_team_provider_credentials( + data=data, + llm_router=Router( + model_list=[ + { + "model_name": "hosted_vllm/*", + "litellm_params": { + "model": "hosted_vllm/*", + "api_base": "http://vllm-credential-lookup.test:8000/v1", + }, + } + ] + ), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + custom_llm_provider="hosted_vllm", + ) + ) + assert await asyncio.to_thread(listing_started.wait, 3) + other_request_served.set() + await lookup + + assert listing_answered.is_set() + assert data.get("api_base") == "http://vllm-credential-lookup.test:8000/v1" + + def _apply_team_a_openai_credentials(data: dict[str, object]) -> None: - apply_team_provider_credentials( - data=data, - llm_router=Router( - model_list=[ - { - "model_name": "openai/*", - "litellm_params": { - "model": "openai/*", - "api_key": "team-openai-key", - "api_base": "https://team.example/v1", - }, - "model_info": {"id": "team-a-openai", "team_id": "team-a"}, - } - ] - ), - user_api_key_dict=UserAPIKeyAuth( - api_key="test-key", user_id="test-user", team_id="team-a", team_models=["openai/*"], models=["gpt-4o"] - ), - custom_llm_provider="openai", + asyncio.run( + apply_team_provider_credentials( + data=data, + llm_router=Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "team-openai-key", + "api_base": "https://team.example/v1", + }, + "model_info": {"id": "team-a-openai", "team_id": "team-a"}, + } + ] + ), + user_api_key_dict=UserAPIKeyAuth( + api_key="test-key", user_id="test-user", team_id="team-a", team_models=["openai/*"], models=["gpt-4o"] + ), + custom_llm_provider="openai", + ) )