fix(router): do not cool down a deployment on a 404 from a response id lookup (#50262)

The Rust core records every attempt failure against the routed deployment, so a client GET, DELETE, cancel or input_items call on a response id that the provider answers with 404 counted toward cooldown. The removed Python core never saw these failures. Exempt them in failure_exits next to the background poll 404 and video lookup exits.

litellm/router_rust/surface/health.py

@@ -279,6 +279,7 @@ class RustHealthCallbacks:
             is_caller_scoped_auth_failure,
             is_caller_timeout_408,
             is_failure_outside_selected_deployment,
+            is_response_lookup_not_found,
             is_video_lookup_not_found,
         )
 
@@ -288,6 +289,7 @@ class RustHealthCallbacks:
             is_failure_outside_selected_deployment(exception if isinstance(exception, BaseException) else None)
             or (isinstance(exception, Exception) and is_background_response_poll_not_found(exception, litellm_params))
             or is_video_lookup_not_found(exception, logged)
+            or is_response_lookup_not_found(exception, logged)
             or is_caller_timeout_408(logged, status)  # pyright: ignore[reportArgumentType]  # the status as raised
             or is_caller_scoped_auth_failure(self._deployment_of(deployment_id), status)  # pyright: ignore[reportArgumentType]  # see above
         )

litellm/router_utils/cooldown_handlers.py

@@ -108,6 +108,26 @@ def is_video_lookup_not_found(exception: object, failure_kwargs: Mapping[str, ob
     return getattr(exception, "status_code", None) == 404 and failure_kwargs.get("call_type") in VIDEO_ID_CALL_TYPES
 
 
+# The calls on an existing response id. ``compact_responses`` is left out: it sends a model and input,
+# so its 404 can mean a missing model rather than a missing response.
+RESPONSE_ID_CALL_TYPES: Final = frozenset(
+    {
+        "aget_responses",
+        "get_responses",
+        "adelete_responses",
+        "delete_responses",
+        "acancel_responses",
+        "cancel_responses",
+        "alist_input_items",
+    }
+)
+
+
+def is_response_lookup_not_found(exception: object, failure_kwargs: Mapping[str, object]) -> bool:
+    """Whether a call on an existing response id failed with a provider 404: the response is not on that account."""
+    return getattr(exception, "status_code", None) == 404 and failure_kwargs.get("call_type") in RESPONSE_ID_CALL_TYPES
+
+
 _EXCEPTION_POLICY_FIELDS: Final[tuple[tuple[type, str], ...]] = (
     # ContentPolicyViolationError subclasses BadRequestError, so it must be checked first.
     (litellm.ContentPolicyViolationError, "ContentPolicyViolationErrorAllowedFails"),

tests/unit/router_utils/test_cooldown_handlers.py

@@ -36,6 +36,7 @@ from litellm.router_utils.cooldown_handlers import (
     async_get_cooldown_deployments,
     cast_exception_status_to_int,
     get_cooldown_deployments,
+    is_response_lookup_not_found,
     mark_advisor_orchestration_failure,
     set_cooldown_deployments,
     should_cooldown_based_on_allowed_fails_policy,
@@ -3565,3 +3566,25 @@ def test_the_refusal_names_each_cooled_deployment_some_cooldown_state_routes_the
     with pytest.raises(RouterRateLimitError) as refusal:
         _select(_tenant_router(EVERY_TENANT, **BOTH_ON), "async", "tenants", copy.deepcopy(request_kwargs))
     assert {deployment["model_id"] for deployment in refusal.value.cooldown_deployments or ()} == reachable
+
+
+@pytest.mark.parametrize(
+    ("status_code", "call_type", "expected"),
+    [
+        (404, "aget_responses", True),
+        (404, "adelete_responses", True),
+        (404, "acancel_responses", True),
+        (404, "alist_input_items", True),
+        (404, "get_responses", True),
+        (404, "aresponses", False),
+        (404, "acompact_responses", False),
+        (404, "acompletion", False),
+        (500, "aget_responses", False),
+        (429, "alist_input_items", False),
+    ],
+)
+def test_is_response_lookup_not_found(status_code: int, call_type: str, expected: bool) -> None:
+    error: Final = litellm.NotFoundError(message="gone", llm_provider="openai", model="gpt-4o-mini")
+    error.status_code = status_code
+
+    assert is_response_lookup_not_found(error, {"call_type": call_type}) is expected

tests/unit/test_router/test_router_provider_endpoints.py

@@ -18,6 +18,7 @@ from litellm.caching.in_memory_cache import InMemoryCache
 from litellm.constants import ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY
 from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
 from litellm.proxy._types import UserAPIKeyAuth
+from litellm.responses.utils import ResponsesAPIRequestUtils
 from litellm.router_utils.router_callbacks.track_deployment_metrics import (
     get_deployment_successes_for_current_minute,
 )
@@ -921,6 +922,43 @@ async def test_video_lookup_404_never_cools_down_the_deployment_that_answered(re
     assert await router.cooldown_cache.async_get_active_cooldowns(["dep-a", "dep-b"], parent_otel_span=None) == []
 
 
+@pytest.mark.parametrize(
+    ("endpoint", "method", "path"),
+    [
+        ("aget_responses", "GET", ""),
+        ("adelete_responses", "DELETE", ""),
+        ("acancel_responses", "POST", "/cancel"),
+        ("alist_input_items", "GET", "/input_items"),
+    ],
+)
+@pytest.mark.asyncio
+async def test_response_lookup_404_never_cools_down_the_deployment_that_answered(
+    respx_mock: respx.MockRouter, endpoint: str, method: str, path: str
+) -> None:
+    respx_mock.route(method=method, url=f"https://api.openai.com/v1/responses/resp_gone{path}").respond(
+        404, json={"error": {"code": "NotFound", "message": "Response not found"}}
+    )
+    router: Final = Router(
+        model_list=[
+            {
+                "model_name": "gpt",
+                "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": api_key},
+                "model_info": {"id": deployment_id},
+            }
+            for api_key, deployment_id in (("key-a", "dep-a"), ("key-b", "dep-b"))
+        ],
+        num_retries=0,
+        allowed_fails=1,
+    )
+    response_id: Final = ResponsesAPIRequestUtils._build_responses_api_response_id("openai", "dep-b", "resp_gone")
+
+    for _ in range(3):
+        with pytest.raises(litellm.NotFoundError):
+            await getattr(router, endpoint)(response_id=response_id)
+
+    assert await router.cooldown_cache.async_get_active_cooldowns(["dep-a", "dep-b"], parent_otel_span=None) == []
+
+
 @pytest.mark.asyncio
 async def test_chat_404_still_cools_down_the_deployment(respx_mock: respx.MockRouter) -> None:
     respx_mock.post("https://api.openai.com/v1/chat/completions").respond(