From 7a575fea15f0acaf2e59eeefd73107be6c91598c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:39:53 -0700 Subject: [PATCH] fix(discovery): cache only timed-out model listings, so a fast failure is tried again on the next call --- litellm/constants.py | 2 +- litellm/utils.py | 11 +++++++---- tests/unit/test_utils.py | 34 +++++++++++++++++++++++++++++++--- 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index e458144474b..dc8107f2343 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -610,7 +610,7 @@ DEFAULT_REDIS_MAJOR_VERSION: Final = get_env_int("DEFAULT_REDIS_MAJOR_VERSION", NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int( "NON_LLM_CONNECTION_TIMEOUT", 15 ) # timeout for adjacent services (e.g. jwt auth) -PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS: Final = 60 +PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS: Final = 60 MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000) MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048) MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024 diff --git a/litellm/utils.py b/litellm/utils.py index 50c3d17290e..d89f405223e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -79,7 +79,7 @@ from litellm.constants import ( MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE, NON_INFERENCE_CALL_TYPES, OPENAI_EMBEDDING_PARAMS, - PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS, + PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS, PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, TOOL_CHOICE_OBJECT_TOKEN_COUNT, ) @@ -8455,13 +8455,16 @@ def _get_valid_models_from_provider_api( _model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) return models - except Exception as e: - verbose_logger.warning("Error getting valid models: %s", e) + except httpx.TimeoutException as e: + verbose_logger.warning("Timed out getting valid models: %s", e) with suppress(ValueError): _model_cache.set_cached_model_info( - custom_llm_provider, litellm_params, [], ttl=PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS + custom_llm_provider, litellm_params, [], ttl=PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS ) return [] + except Exception as e: + verbose_logger.warning("Error getting valid models: %s", e) + return [] def get_valid_models( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 7511c7d8b7a..04e4d7e2b51 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10582,10 +10582,12 @@ def test_get_valid_models_from_provider_cache_invalidation(monkeypatch): @respx.mock -def test_a_failed_provider_model_listing_is_retried_only_after_its_failure_ttl(monkeypatch: pytest.MonkeyPatch) -> None: +def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_ttl( + monkeypatch: pytest.MonkeyPatch, +) -> None: import time - from litellm.constants import PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS + from litellm.constants import PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS from litellm.types.router import LiteLLM_Params models_route: Final = respx.get("http://vllm-hung.test:8000/v1/models").mock( @@ -10604,12 +10606,38 @@ def test_a_failed_provider_model_listing_is_retried_only_after_its_failure_ttl(m assert models_route.call_count == 1 models_route.mock(side_effect=None, return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})) - monkeypatch.setattr(time, "time", lambda: failed_at + PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS + 1) + monkeypatch.setattr(time, "time", lambda: failed_at + PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS + 1) assert list_models() == ["qwen2.5"] assert models_route.call_count == 2 + +@pytest.mark.parametrize( + ("host", "first_answer"), + [("vllm-erroring.test", httpx.Response(503)), ("vllm-refusing.test", httpx.ConnectError("connection refused"))], + ids=["server-error", "connection-refused"], +) +@respx.mock +def test_a_provider_model_listing_that_fails_fast_is_tried_again_on_the_next_call( + host: str, first_answer: httpx.Response | httpx.ConnectError +) -> None: + from litellm.types.router import LiteLLM_Params + + models_route: Final = respx.get(f"http://{host}:8000/v1/models").mock( + side_effect=[first_answer, httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})] + ) + litellm_params: Final = LiteLLM_Params(model="hosted_vllm/*", api_base=f"http://{host}:8000/v1") + + def list_models() -> list[str]: + return get_valid_models( + check_provider_endpoint=True, custom_llm_provider="hosted_vllm", litellm_params=litellm_params + ) + + assert list_models() == [] + assert list_models() == ["qwen2.5"] + assert models_route.call_count == 2 + def test_delta_tool_calls_sequential_indices(): """ Test that multiple tool calls without explicit indices receive sequential indices.