mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): list wildcard models off the event loop, so a hung provider no longer stalls other requests
Some checks failed
LiteLLM Rust DB / db-queries (push) Waiting to run
LiteLLM Rust / rust-lint (push) Waiting to run
LiteLLM Rust / rust-test (push) Waiting to run
LiteLLM Rust / rust-wheel (push) Waiting to run
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Some checks failed
LiteLLM Rust DB / db-queries (push) Waiting to run
LiteLLM Rust / rust-lint (push) Waiting to run
LiteLLM Rust / rust-test (push) Waiting to run
LiteLLM Rust / rust-wheel (push) Waiting to run
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
This commit is contained in:
parent
7a575fea15
commit
d69da65a6a
5 changed files with 75 additions and 20 deletions
|
|
@ -12398,7 +12398,8 @@ async def model_list(
|
|||
# Get complete model list including wildcard routes if requested
|
||||
from litellm.proxy.auth.model_checks import get_complete_model_list
|
||||
|
||||
all_models = get_complete_model_list(
|
||||
all_models = await asyncio.to_thread(
|
||||
get_complete_model_list,
|
||||
key_models=[],
|
||||
team_models=[],
|
||||
proxy_model_list=proxy_model_list,
|
||||
|
|
@ -17751,7 +17752,7 @@ async def model_info_v1(
|
|||
alias_models: Final = copy.deepcopy(llm_router.get_model_list_from_model_alias())
|
||||
all_models.extend(alias_models)
|
||||
|
||||
all_models = expand_wildcard_deployments_for_model_info(all_models, llm_router)
|
||||
all_models = await asyncio.to_thread(expand_wildcard_deployments_for_model_info, all_models, llm_router)
|
||||
|
||||
allowed_model_names: Final = await _get_v1_model_info_allowed_model_names_with_access_groups(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
|
|
@ -18086,7 +18087,8 @@ async def model_group_info(
|
|||
LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
|
||||
)
|
||||
all_models_str: Final = (
|
||||
get_complete_model_list(
|
||||
await asyncio.to_thread(
|
||||
get_complete_model_list,
|
||||
key_models=(),
|
||||
team_models=(),
|
||||
proxy_model_list=llm_router.get_model_names(),
|
||||
|
|
|
|||
|
|
@ -9517,7 +9517,8 @@ async def _agent_access_group_visible_models(
|
|||
if not granted:
|
||||
return frozenset()
|
||||
return frozenset(
|
||||
get_complete_model_list(
|
||||
await asyncio.to_thread(
|
||||
get_complete_model_list,
|
||||
key_models=granted,
|
||||
team_models=(),
|
||||
proxy_model_list=proxy_model_list,
|
||||
|
|
@ -9652,7 +9653,8 @@ async def get_available_models_for_user(
|
|||
)
|
||||
|
||||
# Get complete model list
|
||||
complete_models: Final = get_complete_model_list(
|
||||
complete_models: Final = await asyncio.to_thread(
|
||||
get_complete_model_list,
|
||||
key_models=granted_key_models,
|
||||
team_models=granted_team_models,
|
||||
proxy_model_list=proxy_model_list,
|
||||
|
|
|
|||
|
|
@ -8333,8 +8333,8 @@ _HUGGINGFACE_TOKENIZER_LOAD_FAILURES: Final[_HuggingFaceTokenizerLoadFailureCach
|
|||
|
||||
|
||||
class AvailableModelsCache(InMemoryCache):
|
||||
def __init__(self, ttl_seconds: int = 300, max_size: int = 1000):
|
||||
super().__init__(ttl_seconds, max_size)
|
||||
def __init__(self, ttl_seconds: int = 300, max_size: int = 1000, clock: Callable[[], float] | None = None):
|
||||
super().__init__(ttl_seconds, max_size, clock=clock)
|
||||
self._env_hash: str | None = None
|
||||
|
||||
def _get_env_hash(self) -> str:
|
||||
|
|
@ -8443,9 +8443,10 @@ def _get_valid_models_from_provider_api(
|
|||
provider_config: BaseLLMModelInfo,
|
||||
custom_llm_provider: str,
|
||||
litellm_params: LiteLLM_Params | None = None,
|
||||
model_cache: AvailableModelsCache = _model_cache,
|
||||
) -> list[str]:
|
||||
try:
|
||||
cached_result: Final = _model_cache.get_cached_model_info(custom_llm_provider, litellm_params)
|
||||
cached_result: Final = model_cache.get_cached_model_info(custom_llm_provider, litellm_params)
|
||||
|
||||
if cached_result is not None:
|
||||
return cached_result
|
||||
|
|
@ -8453,12 +8454,12 @@ def _get_valid_models_from_provider_api(
|
|||
litellm_params=_provider_listing_params(custom_llm_provider, litellm_params)
|
||||
)
|
||||
|
||||
_model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models)
|
||||
model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models)
|
||||
return models
|
||||
except httpx.TimeoutException as e:
|
||||
verbose_logger.warning("Timed out getting valid models: %s", e)
|
||||
with suppress(ValueError):
|
||||
_model_cache.set_cached_model_info(
|
||||
model_cache.set_cached_model_info(
|
||||
custom_llm_provider, litellm_params, [], ttl=PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS
|
||||
)
|
||||
return []
|
||||
|
|
|
|||
|
|
@ -540,6 +540,54 @@ def test_hosted_vllm_wildcard_listing_does_not_wait_the_completion_timeout_on_a_
|
|||
assert read_timeout <= NON_LLM_CONNECTION_TIMEOUT
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_a_hung_wildcard_listing_leaves_the_event_loop_free_for_other_requests(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import asyncio
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.utils import get_available_models_for_user
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
listing_started: Final = threading.Event()
|
||||
other_request_served: Final = threading.Event()
|
||||
|
||||
def listing_that_answers_once_another_request_is_served(request: httpx.Request) -> httpx.Response:
|
||||
listing_started.set()
|
||||
if other_request_served.wait(timeout=3):
|
||||
return httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})
|
||||
return httpx.Response(503)
|
||||
|
||||
respx.get("http://vllm-stalled.test:8000/v1/models").mock(
|
||||
side_effect=listing_that_answers_once_another_request_is_served
|
||||
)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "hosted_vllm/*",
|
||||
"litellm_params": {"model": "hosted_vllm/*", "api_base": "http://vllm-stalled.test:8000/v1"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
listing: Final = asyncio.create_task(
|
||||
get_available_models_for_user(
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", models=[]),
|
||||
llm_router=router,
|
||||
general_settings={},
|
||||
user_model=None,
|
||||
)
|
||||
)
|
||||
assert await asyncio.to_thread(listing_started.wait, 3)
|
||||
other_request_served.set()
|
||||
|
||||
assert "hosted_vllm/qwen2.5" in await listing
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_hosted_vllm_wildcard_listing_sends_the_deployment_extra_headers(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import litellm
|
||||
|
|
|
|||
|
|
@ -10582,23 +10582,22 @@ def test_get_valid_models_from_provider_cache_invalidation(monkeypatch):
|
|||
|
||||
|
||||
@respx.mock
|
||||
def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_ttl(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import time
|
||||
|
||||
def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_ttl() -> None:
|
||||
from litellm.constants import PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS
|
||||
from litellm.llms.vllm.common_utils import HostedVLLMModelInfo
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
from litellm.utils import AvailableModelsCache, _get_valid_models_from_provider_api
|
||||
|
||||
models_route: Final = respx.get("http://vllm-hung.test:8000/v1/models").mock(
|
||||
side_effect=httpx.ReadTimeout("vLLM did not answer")
|
||||
)
|
||||
litellm_params: Final = LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-hung.test:8000/v1")
|
||||
failed_at: Final = time.time()
|
||||
clock: Final = Mock(return_value=1000.0)
|
||||
model_cache: Final = AvailableModelsCache(clock=clock)
|
||||
|
||||
def list_models() -> list[str]:
|
||||
return get_valid_models(
|
||||
check_provider_endpoint=True, custom_llm_provider="hosted_vllm", litellm_params=litellm_params
|
||||
return _get_valid_models_from_provider_api(
|
||||
HostedVLLMModelInfo(), "hosted_vllm", litellm_params, model_cache=model_cache
|
||||
)
|
||||
|
||||
assert list_models() == []
|
||||
|
|
@ -10606,13 +10605,15 @@ def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_tt
|
|||
assert models_route.call_count == 1
|
||||
|
||||
models_route.mock(side_effect=None, return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}))
|
||||
monkeypatch.setattr(time, "time", lambda: failed_at + PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS + 1)
|
||||
clock.return_value = 1000.0 + PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS - 1
|
||||
assert list_models() == []
|
||||
assert models_route.call_count == 1
|
||||
|
||||
clock.return_value = 1000.0 + PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS + 1
|
||||
assert list_models() == ["qwen2.5"]
|
||||
assert models_route.call_count == 2
|
||||
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("host", "first_answer"),
|
||||
[("vllm-erroring.test", httpx.Response(503)), ("vllm-refusing.test", httpx.ConnectError("connection refused"))],
|
||||
|
|
@ -10638,6 +10639,7 @@ def test_a_provider_model_listing_that_fails_fast_is_tried_again_on_the_next_cal
|
|||
assert list_models() == ["qwen2.5"]
|
||||
assert models_route.call_count == 2
|
||||
|
||||
|
||||
def test_delta_tool_calls_sequential_indices():
|
||||
"""
|
||||
Test that multiple tool calls without explicit indices receive sequential indices.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue