mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(discovery): cache only timed-out model listings, so a fast failure is tried again on the next call
This commit is contained in:
parent
4e109a03e9
commit
7a575fea15
3 changed files with 39 additions and 8 deletions
|
|
@ -610,7 +610,7 @@ DEFAULT_REDIS_MAJOR_VERSION: Final = get_env_int("DEFAULT_REDIS_MAJOR_VERSION",
|
|||
NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int(
|
||||
"NON_LLM_CONNECTION_TIMEOUT", 15
|
||||
) # timeout for adjacent services (e.g. jwt auth)
|
||||
PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS: Final = 60
|
||||
PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS: Final = 60
|
||||
MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000)
|
||||
MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048)
|
||||
MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ from litellm.constants import (
|
|||
MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE,
|
||||
NON_INFERENCE_CALL_TYPES,
|
||||
OPENAI_EMBEDDING_PARAMS,
|
||||
PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS,
|
||||
PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS,
|
||||
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
|
||||
TOOL_CHOICE_OBJECT_TOKEN_COUNT,
|
||||
)
|
||||
|
|
@ -8455,13 +8455,16 @@ def _get_valid_models_from_provider_api(
|
|||
|
||||
_model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models)
|
||||
return models
|
||||
except Exception as e:
|
||||
verbose_logger.warning("Error getting valid models: %s", e)
|
||||
except httpx.TimeoutException as e:
|
||||
verbose_logger.warning("Timed out getting valid models: %s", e)
|
||||
with suppress(ValueError):
|
||||
_model_cache.set_cached_model_info(
|
||||
custom_llm_provider, litellm_params, [], ttl=PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS
|
||||
custom_llm_provider, litellm_params, [], ttl=PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS
|
||||
)
|
||||
return []
|
||||
except Exception as e:
|
||||
verbose_logger.warning("Error getting valid models: %s", e)
|
||||
return []
|
||||
|
||||
|
||||
def get_valid_models(
|
||||
|
|
|
|||
|
|
@ -10582,10 +10582,12 @@ def test_get_valid_models_from_provider_cache_invalidation(monkeypatch):
|
|||
|
||||
|
||||
@respx.mock
|
||||
def test_a_failed_provider_model_listing_is_retried_only_after_its_failure_ttl(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_ttl(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import time
|
||||
|
||||
from litellm.constants import PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS
|
||||
from litellm.constants import PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
models_route: Final = respx.get("http://vllm-hung.test:8000/v1/models").mock(
|
||||
|
|
@ -10604,12 +10606,38 @@ def test_a_failed_provider_model_listing_is_retried_only_after_its_failure_ttl(m
|
|||
assert models_route.call_count == 1
|
||||
|
||||
models_route.mock(side_effect=None, return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}))
|
||||
monkeypatch.setattr(time, "time", lambda: failed_at + PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS + 1)
|
||||
monkeypatch.setattr(time, "time", lambda: failed_at + PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS + 1)
|
||||
|
||||
assert list_models() == ["qwen2.5"]
|
||||
assert models_route.call_count == 2
|
||||
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("host", "first_answer"),
|
||||
[("vllm-erroring.test", httpx.Response(503)), ("vllm-refusing.test", httpx.ConnectError("connection refused"))],
|
||||
ids=["server-error", "connection-refused"],
|
||||
)
|
||||
@respx.mock
|
||||
def test_a_provider_model_listing_that_fails_fast_is_tried_again_on_the_next_call(
|
||||
host: str, first_answer: httpx.Response | httpx.ConnectError
|
||||
) -> None:
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
models_route: Final = respx.get(f"http://{host}:8000/v1/models").mock(
|
||||
side_effect=[first_answer, httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})]
|
||||
)
|
||||
litellm_params: Final = LiteLLM_Params(model="hosted_vllm/*", api_base=f"http://{host}:8000/v1")
|
||||
|
||||
def list_models() -> list[str]:
|
||||
return get_valid_models(
|
||||
check_provider_endpoint=True, custom_llm_provider="hosted_vllm", litellm_params=litellm_params
|
||||
)
|
||||
|
||||
assert list_models() == []
|
||||
assert list_models() == ["qwen2.5"]
|
||||
assert models_route.call_count == 2
|
||||
|
||||
def test_delta_tool_calls_sequential_indices():
|
||||
"""
|
||||
Test that multiple tool calls without explicit indices receive sequential indices.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue