diff --git a/litellm/utils.py b/litellm/utils.py index dde5aab764b..e1ec7bfcb6b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8506,12 +8506,15 @@ def _get_valid_models_from_provider_api( _model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) return models - except Exception as e: - verbose_logger.warning("Error getting valid models: %s", e) + except httpx.TimeoutException as e: + verbose_logger.warning("Timed out getting valid models: %s", e) _model_cache.set_failed_model_listing( custom_llm_provider, litellm_params, ttl_seconds=_FAILED_PROVIDER_LISTING_TTL_SECONDS ) return [] + except Exception as e: + verbose_logger.warning("Error getting valid models: %s", e) + return [] def get_valid_models( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index dcc05f19efb..02aaff287cc 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -8898,8 +8898,10 @@ def reset_mock_cache() -> None: @respx.mock -def test_get_valid_models_does_not_ask_a_failing_provider_again_on_the_next_listing(reset_mock_cache: None) -> None: - catalog: Final = respx.get("https://openrouter.ai/api/v1/models/user").mock(return_value=httpx.Response(503)) +def test_get_valid_models_does_not_ask_a_timed_out_provider_again_on_the_next_listing(reset_mock_cache: None) -> None: + catalog: Final = respx.get("https://openrouter.ai/api/v1/models/user").mock( + side_effect=httpx.ReadTimeout("catalog hung") + ) deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-during-an-outage") listings: Final = tuple( @@ -8910,18 +8912,32 @@ def test_get_valid_models_does_not_ask_a_failing_provider_again_on_the_next_list assert (listings, catalog.call_count) == (([], []), 1) +@respx.mock +def test_get_valid_models_lists_a_provider_again_right_after_a_fast_failure(reset_mock_cache: None) -> None: + respx.get("https://openrouter.ai/api/v1/models/user").mock( + side_effect=(httpx.Response(503), httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]})) + ) + deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-recovers") + + listings: Final = tuple( + get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment) + for _ in range(2) + ) + + assert listings == ([], ["openrouter/anthropic/claude-haiku-5-5"]) + + @respx.mock def test_get_valid_models_keeps_a_good_listing_when_an_overlapping_listing_fails_after_it( reset_mock_cache: None, ) -> None: deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-overlapping-listings") - statuses: Final = iter((503, 200)) + first_request: Final = iter((True, False)) def answer_after_an_overlapping_listing(request: httpx.Request) -> httpx.Response: - status: Final = next(statuses) - if status == 503: + if next(first_request): get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment) - return httpx.Response(503) + raise httpx.ReadTimeout("catalog hung", request=request) return httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]}) respx.get("https://openrouter.ai/api/v1/models/user").mock(side_effect=answer_after_an_overlapping_listing)