scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-06 07:48:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 112

fix(proxy): remember only timed-out provider listings so a provider that failed fast is listed again

Browse source
This commit is contained in:
mateo-berri 2026-10-29 02:43:27 -07:00
parent ddda5bd199
commit aabeffc652
2 changed files with 27 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
litellm/utils.py
View file

@ -8506,12 +8506,15 @@ def _get_valid_models_from_provider_api(
_model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models)
return models
except Exception as e:
verbose_logger.warning("Error getting valid models: %s", e)
except httpx.TimeoutException as e:
verbose_logger.warning("Timed out getting valid models: %s", e)
_model_cache.set_failed_model_listing(
custom_llm_provider, litellm_params, ttl_seconds=_FAILED_PROVIDER_LISTING_TTL_SECONDS
)
return []
except Exception as e:
verbose_logger.warning("Error getting valid models: %s", e)
return []
def get_valid_models(

28
tests/unit/test_utils.py
View file

@ -8898,8 +8898,10 @@ def reset_mock_cache() -> None:
@respx.mock
def test_get_valid_models_does_not_ask_a_failing_provider_again_on_the_next_listing(reset_mock_cache: None) -> None:
catalog: Final = respx.get("https://openrouter.ai/api/v1/models/user").mock(return_value=httpx.Response(503))
def test_get_valid_models_does_not_ask_a_timed_out_provider_again_on_the_next_listing(reset_mock_cache: None) -> None:
catalog: Final = respx.get("https://openrouter.ai/api/v1/models/user").mock(
side_effect=httpx.ReadTimeout("catalog hung")
)
deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-during-an-outage")
listings: Final = tuple(
@ -8910,18 +8912,32 @@ def test_get_valid_models_does_not_ask_a_failing_provider_again_on_the_next_list
assert (listings, catalog.call_count) == (([], []), 1)
@respx.mock
def test_get_valid_models_lists_a_provider_again_right_after_a_fast_failure(reset_mock_cache: None) -> None:
respx.get("https://openrouter.ai/api/v1/models/user").mock(
side_effect=(httpx.Response(503), httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]}))
)
deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-recovers")
listings: Final = tuple(
get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment)
for _ in range(2)
)
assert listings == ([], ["openrouter/anthropic/claude-haiku-5-5"])
@respx.mock
def test_get_valid_models_keeps_a_good_listing_when_an_overlapping_listing_fails_after_it(
reset_mock_cache: None,
) -> None:
deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-overlapping-listings")
statuses: Final = iter((503, 200))
first_request: Final = iter((True, False))
def answer_after_an_overlapping_listing(request: httpx.Request) -> httpx.Response:
status: Final = next(statuses)
if status == 503:
if next(first_request):
get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment)
return httpx.Response(503)
raise httpx.ReadTimeout("catalog hung", request=request)
return httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]})
respx.get("https://openrouter.ai/api/v1/models/user").mock(side_effect=answer_after_an_overlapping_listing)