fix(discovery): cache only timed-out model listings, so a fast failure is tried again on the next call

This commit is contained in:
mateo-berri 2026-10-28 19:39:53 -07:00
parent 4e109a03e9
commit 7a575fea15
3 changed files with 39 additions and 8 deletions

View file

@ -610,7 +610,7 @@ DEFAULT_REDIS_MAJOR_VERSION: Final = get_env_int("DEFAULT_REDIS_MAJOR_VERSION",
NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int(
"NON_LLM_CONNECTION_TIMEOUT", 15
) # timeout for adjacent services (e.g. jwt auth)
PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS: Final = 60
PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS: Final = 60
MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000)
MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048)
MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024

View file

@ -79,7 +79,7 @@ from litellm.constants import (
MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE,
NON_INFERENCE_CALL_TYPES,
OPENAI_EMBEDDING_PARAMS,
PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS,
PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS,
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
TOOL_CHOICE_OBJECT_TOKEN_COUNT,
)
@ -8455,13 +8455,16 @@ def _get_valid_models_from_provider_api(
_model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models)
return models
except Exception as e:
verbose_logger.warning("Error getting valid models: %s", e)
except httpx.TimeoutException as e:
verbose_logger.warning("Timed out getting valid models: %s", e)
with suppress(ValueError):
_model_cache.set_cached_model_info(
custom_llm_provider, litellm_params, [], ttl=PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS
custom_llm_provider, litellm_params, [], ttl=PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS
)
return []
except Exception as e:
verbose_logger.warning("Error getting valid models: %s", e)
return []
def get_valid_models(

View file

@ -10582,10 +10582,12 @@ def test_get_valid_models_from_provider_cache_invalidation(monkeypatch):
@respx.mock
def test_a_failed_provider_model_listing_is_retried_only_after_its_failure_ttl(monkeypatch: pytest.MonkeyPatch) -> None:
def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_ttl(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import time
from litellm.constants import PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS
from litellm.constants import PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS
from litellm.types.router import LiteLLM_Params
models_route: Final = respx.get("http://vllm-hung.test:8000/v1/models").mock(
@ -10604,12 +10606,38 @@ def test_a_failed_provider_model_listing_is_retried_only_after_its_failure_ttl(m
assert models_route.call_count == 1
models_route.mock(side_effect=None, return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}))
monkeypatch.setattr(time, "time", lambda: failed_at + PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS + 1)
monkeypatch.setattr(time, "time", lambda: failed_at + PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS + 1)
assert list_models() == ["qwen2.5"]
assert models_route.call_count == 2
@pytest.mark.parametrize(
("host", "first_answer"),
[("vllm-erroring.test", httpx.Response(503)), ("vllm-refusing.test", httpx.ConnectError("connection refused"))],
ids=["server-error", "connection-refused"],
)
@respx.mock
def test_a_provider_model_listing_that_fails_fast_is_tried_again_on_the_next_call(
host: str, first_answer: httpx.Response | httpx.ConnectError
) -> None:
from litellm.types.router import LiteLLM_Params
models_route: Final = respx.get(f"http://{host}:8000/v1/models").mock(
side_effect=[first_answer, httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})]
)
litellm_params: Final = LiteLLM_Params(model="hosted_vllm/*", api_base=f"http://{host}:8000/v1")
def list_models() -> list[str]:
return get_valid_models(
check_provider_endpoint=True, custom_llm_provider="hosted_vllm", litellm_params=litellm_params
)
assert list_models() == []
assert list_models() == ["qwen2.5"]
assert models_route.call_count == 2
def test_delta_tool_calls_sequential_indices():
"""
Test that multiple tool calls without explicit indices receive sequential indices.