test(e2e): client hang-up under cancel_on_disconnect never benches the Azure deployment

tests/e2e/coverage_registry/reliability.yaml

@@ -12,6 +12,7 @@
 - {id: reliability.cooldown.429.trips_then_recovers, module: reliability, tier: P0, behavior: cooldown, variant: "429", assertions: [trips_then_recovers], exercised_on: [chat_completions], source: "cooldown_handlers.py:69", rationale: "Cools on 429, avoids hammering exhausted provider"}
 - {id: reliability.cooldown.auth.trips_then_recovers, module: reliability, tier: P1, behavior: cooldown, variant: auth, assertions: [trips_then_recovers], exercised_on: [chat_completions], source: "cooldown_handlers.py:74", rationale: "Cools on 401 auth error"}
 - {id: reliability.cooldown.timeout.trips_then_recovers, module: reliability, tier: P1, behavior: cooldown, variant: timeout, assertions: [trips_then_recovers], exercised_on: [chat_completions], source: "cooldown_handlers.py:77", rationale: "Cools on 408 timeout"}
+- {id: reliability.cooldown.client_disconnect.stays_healthy, module: reliability, tier: P0, behavior: cooldown, variant: client_disconnect, assertions: [stays_healthy], exercised_on: [chat_completions], source: "llms/azure/azure.py:484", fail_before_fix: proven, rationale: "A client hanging up mid-request under cancel_on_disconnect never benches the Azure deployment it was talking to: the cancellation used to surface as a fake 500 that tripped the cooldown and sent every caller behind it to billed fallbacks (GitHub issues #35329 and #42222)"}
 - {id: reliability.routing.simple_shuffle.picks_healthy_deployment, module: reliability, tier: P1, behavior: routing, variant: simple_shuffle, assertions: [picks_healthy_deployment], exercised_on: [chat_completions], source: "router_strategy/simple_shuffle.py", rationale: "Baseline weighted/uniform pick"}
 - {id: reliability.routing.latency_based.picks_lowest_latency, module: reliability, tier: P1, behavior: routing, variant: latency_based, assertions: [picks_lowest_latency], exercised_on: [chat_completions], source: "router_strategy/lowest_latency.py", rationale: "Routes to lowest-latency deployment"}
 - {id: reliability.routing.cost_based.picks_lowest_cost, module: reliability, tier: P1, behavior: routing, variant: cost_based, assertions: [picks_lowest_cost], exercised_on: [chat_completions], source: "router_strategy/lowest_cost.py", rationale: "Spend-aware routing"}

tests/e2e/gateway/stage_mirror_ci_config.yml

@@ -10,6 +10,7 @@ general_settings:
   store_prompts_in_spend_logs: true
   database_connection_pool_limit: 10
   forward_client_headers_to_llm_api: false
+  cancel_on_disconnect: true
   maximum_spend_logs_retention_period: "60d"
   maximum_spend_logs_cleanup_cron: "0 1 * * *"
   proxy_budget_rescheduler_min_time: 15

tests/e2e/models.py

@@ -916,6 +916,23 @@ class RouterSettingsResponse(BaseModel):
     current_values: RouterCurrentValues
 
 
+class ConfigListParams(BaseModel):
+    config_type: Literal["general_settings"]
+
+
+class ConfigField(BaseModel):
+    """One row of GET /config/list: a general_settings field and the value the
+    proxy is running with, the two fields a test preconditions on."""
+
+    model_config = ConfigDict(extra="ignore")
+    field_name: str
+    field_value: JsonValue = None
+
+
+class ConfigFieldList(RootModel[tuple[ConfigField, ...]]):
+    """GET /config/list answers with a bare array of general_settings fields."""
+
+
 class CostMapEntry(BaseModel):
     model_config = ConfigDict(extra="ignore")
     litellm_provider: str | None = None
@@ -1031,6 +1048,7 @@ class ModelInfoBody(BaseModel):
     access_groups: list[str] | None = None
     team_id: str | None = None
     allowed_fails_policy: dict[str, int] | None = None
+    allowed_fails: int | None = None
 
 
 class ModelNewBody(BaseModel):

tests/e2e/proxy_client.py

@@ -47,6 +47,8 @@ from models import (
     AnthropicMessagesResponse,
     ChatBody,
     ChatResponse,
+    ConfigFieldList,
+    ConfigListParams,
     CostMap,
     CostMapEntry,
     CountTokensBody,
@@ -628,6 +630,19 @@ class ProxyClient:
             provider_live=provider_live,
         )
 
+    def general_setting_enabled(self, field_name: str) -> bool:
+        """Whether the proxy is running with the named general_settings flag on, for
+        a test whose behavior only exists under a config flag the stack has to carry."""
+        fields = unwrap(
+            self.transport.get(
+                "/config/list",
+                headers=self.transport.master,
+                params=ConfigListParams(config_type="general_settings"),
+                response_type=ConfigFieldList,
+            )
+        ).root
+        return any(entry.field_name == field_name and entry.field_value is True for entry in fields)
+
     def register_model(
         self, body: ModelNewBody, listed_for: str | None = None, *, provider_live: bool = False
     ) -> str:

tests/e2e/router/reliability_support.py

@@ -42,7 +42,7 @@ REAL_KEY = "os.environ/OPENAI_API_KEY"
 CACHING_MODEL = "anthropic/claude-haiku-4-5"
 CACHING_KEY = "os.environ/ANTHROPIC_API_KEY"
 
-CONTENT_FILTERED_MODEL = "azure/gpt-5.4-nano"
+AZURE_MODEL = "azure/gpt-5.4-nano"
 AZURE_KEY = "os.environ/AZURE_API_KEY"
 AZURE_BASE = "os.environ/AZURE_API_BASE"
 AZURE_API_VERSION = "2024-10-21"
@@ -111,7 +111,7 @@ def create_content_filtered_deployment(proxy: ProxyClient, name: str) -> str:
     return proxy.create_model(
         name,
         LiteLLMParamsBody(
-            model=CONTENT_FILTERED_MODEL,
+            model=AZURE_MODEL,
             api_key=AZURE_KEY,
             api_base=AZURE_BASE,
             api_version=AZURE_API_VERSION,
@@ -120,6 +120,30 @@ def create_content_filtered_deployment(proxy: ProxyClient, name: str) -> str:
     )
 
 
+def create_azure_benched_on_first_failure_deployment(proxy: ProxyClient, name: str, cooldown_time: float) -> str:
+    """The live Azure OpenAI deployment holding all of the group's shuffle weight,
+    benched on its first failure of any class, with the client's own retries off.
+    The 500 the proxy used to book against a call the client hung up on carries no
+    provider body, so litellm maps it to a bare APIError that no named
+    allowed_fails_policy class covers; the deployment-wide allowed_fails=0 is the
+    knob that makes that undeserved bench show on the very next call."""
+    return proxy.register_model(
+        ModelNewBody(
+            model_name=name,
+            litellm_params=LiteLLMParamsBody(
+                model=AZURE_MODEL,
+                api_key=AZURE_KEY,
+                api_base=AZURE_BASE,
+                api_version=AZURE_API_VERSION,
+                max_retries=0,
+                weight=1,
+                cooldown_time=cooldown_time,
+            ),
+            model_info=ModelInfoBody(allowed_fails=0),
+        )
+    )
+
+
 def create_caching_deployment(proxy: ProxyClient, name: str) -> str:
     """Register the Anthropic deployment whose prompt cache the affinity check pins to."""
     return proxy.create_model(name, LiteLLMParamsBody(model=CACHING_MODEL, api_key=CACHING_KEY, weight=1))

tests/e2e/router/test_reliability_cancel_on_disconnect_e2e.py

@@ -0,0 +1,132 @@
+"""Live e2e: a client hanging up mid-request under cancel_on_disconnect never
+benches the deployment it was talking to.
+
+With `general_settings.cancel_on_disconnect: true` the proxy cancels the in-flight
+provider call the moment the client's socket closes. The Azure handler used to
+turn that cancellation into a fake 500, which the router booked as a deployment
+failure: one impatient client benched a healthy deployment and every caller
+behind it paid for fallbacks (GitHub issues #35329 and #42222). This cell pins the
+fix at the seam a customer sees. The group is the cooldown suite's pair: the live
+Azure deployment holding all of the shuffle weight, benched on its first failure
+of any class (the fake 500 carried no provider body, so litellm mapped it to a
+bare APIError no named policy class covers) with a cooldown long enough to
+outlast the test, plus a healthy backup at weight 0 the shuffle can only reach
+once the Azure deployment is benched. One cheap call first proves the Azure
+deployment answers the key and leaves the key's auth path warm. The test then
+asks for a long answer, retries off, and hangs up a few seconds in: the client's
+read timeout closes the socket well after the proxy has handed the call to Azure
+(a cold virtual-key auth can take a couple of seconds on its own, and a hang-up
+that lands before the provider call is in flight cancels nothing the router could
+bench, so a shorter window passes vacuously) and well before the answer is done.
+After a settle window wide enough for a sibling replica to have read any bench
+from Redis, every one of the next calls has to come back 200 from the Azure
+deployment itself, named in x-litellm-model-id; a single answer from the backup
+means the hang-up was booked as a failure.
+
+The test reads `cancel_on_disconnect` back from the proxy first: without the flag
+the hang-up cancels nothing and the cell would pass vacuously.
+"""
+
+from __future__ import annotations
+
+import time
+
+import pytest
+from complexity_router_client import ComplexityRouterClient
+from e2e_config import unique_marker
+from e2e_http import NetworkError, StreamingResponse
+from lifecycle import ResourceManager
+from models import ChatMessage, ChatResponse, ReliabilityChatBody, RouterSettingsOverride
+from reliability_support import (
+    chat_override,
+    create_azure_benched_on_first_failure_deployment,
+    create_zero_weight_backup_deployment,
+    model_id_of,
+)
+
+pytestmark = pytest.mark.e2e
+
+CLIENT_HANGS_UP_AFTER_SECONDS = 8.0
+LONG_ANSWER_MAX_TOKENS = 4096
+BENCH_OUTLASTS_TEST_SECONDS = 300.0
+SETTLE_AFTER_HANGUP_SECONDS = 3.0
+CALLS_AFTER_HANGUP = 6
+
+
+def _say_hi(client: ComplexityRouterClient, key: str, group: str) -> StreamingResponse:
+    return chat_override(
+        client.proxy,
+        key,
+        group,
+        f"say hi {unique_marker()}",
+        override=RouterSettingsOverride(num_retries=0),
+    )
+
+
+def _hang_up_mid_answer(client: ComplexityRouterClient, key: str, group: str) -> None:
+    """Send a request whose answer takes far longer than the client waits, so the
+    read timeout closes the socket while the provider is still generating."""
+    outcome = client.proxy.transport.post(
+        "/chat/completions",
+        headers=client.proxy.transport.bearer(key),
+        json=ReliabilityChatBody(
+            model=group,
+            messages=[
+                ChatMessage(
+                    role="user",
+                    content=f"Write a 3000 word essay on the history of the telegraph. {unique_marker()}",
+                )
+            ],
+            max_tokens=LONG_ANSWER_MAX_TOKENS,
+            router_settings_override=RouterSettingsOverride(num_retries=0),
+        ),
+        response_type=ChatResponse,
+        timeout=CLIENT_HANGS_UP_AFTER_SECONDS,
+    )
+    match outcome:
+        case NetworkError(message=message) if "Read timed out" in message:
+            return
+        case _:
+            pytest.fail(
+                f"the client should have hung up {CLIENT_HANGS_UP_AFTER_SECONDS:.0f}s into a long answer with the "
+                f"call still in flight, but the proxy answered first: {outcome!r}"
+            )
+
+
+class TestReliabilityCancelOnDisconnect:
+    @pytest.mark.covers("reliability.cooldown.client_disconnect.stays_healthy")
+    def test_client_hanging_up_never_benches_the_deployment(
+        self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str
+    ) -> None:
+        assert client.proxy.general_setting_enabled("cancel_on_disconnect"), (
+            "this cell needs general_settings.cancel_on_disconnect: true in the proxy config; without it the "
+            "hang-up cancels nothing and the bench it guards against can never happen"
+        )
+
+        group = f"reliability-cooldown-disconnect-{unique_marker()}"
+        azure = create_azure_benched_on_first_failure_deployment(
+            client.proxy, group, cooldown_time=BENCH_OUTLASTS_TEST_SECONDS
+        )
+        resources.defer(lambda: client.proxy.delete_model(azure))
+        backup = create_zero_weight_backup_deployment(client.proxy, group)
+        resources.defer(lambda: client.proxy.delete_model(backup))
+
+        warm_up = _say_hi(client, scoped_key, group)
+        assert warm_up.status_code == 200 and model_id_of(warm_up) == azure, (
+            f"before any hang-up the Azure deployment {azure} should answer the group, got {warm_up.status_code} "
+            f"from {model_id_of(warm_up)!r}: {warm_up.body[:300]}"
+        )
+
+        _hang_up_mid_answer(client, scoped_key, group)
+        time.sleep(SETTLE_AFTER_HANGUP_SECONDS)
+
+        for call in range(1, CALLS_AFTER_HANGUP + 1):
+            resp = _say_hi(client, scoped_key, group)
+            assert resp.status_code == 200, (
+                f"call {call} after the hang-up should have been a plain 200 from the group, got "
+                f"{resp.status_code}: {resp.body[:300]}"
+            )
+            assert model_id_of(resp) == azure, (
+                f"call {call} after the hang-up should have been served by the Azure deployment {azure}, the proxy "
+                f"named {model_id_of(resp)!r}: the cancelled call was booked as a failure and benched it"
+            )