From c32eea634a702636df41b6df26a41a5642bcf675 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 16 Oct 2026 23:27:51 -0700 Subject: [PATCH] test(integration): send post-kill requests on a fresh connection in two worker-kill tests (#47080) kill() returns once the killed worker's main thread is a zombie, but its other threads can still be exiting and holding its sockets open. A keep-alive connection pooled to that worker still looks healthy, so the next request goes out on it and the closing socket answers with a reset. The old reap wait used to hide this. A new connection can only be accepted by a live worker, so the surviving-worker assertions now run on a fresh client --- .../test_missing_body_param_status.py | 22 +++++++++++-------- ..._mantle_context_overflow_fallbacks_wire.py | 10 +++++---- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/tests/integration/compatibility/test_missing_body_param_status.py b/tests/integration/compatibility/test_missing_body_param_status.py index 42ee063ddc7..dd98ff6dea1 100644 --- a/tests/integration/compatibility/test_missing_body_param_status.py +++ b/tests/integration/compatibility/test_missing_body_param_status.py @@ -1695,15 +1695,19 @@ def test_upstream_pause_and_worker_kill_preserve_required_body_status( record_property("chaos_spend_row_counts", str(tuple(len(rows) for rows in spend_rows))) with owned.replacing_worker(min(workers)) as kill: kill() - post_kill: Final = tuple( - _post(candidate, path, body) for path, body, _expected, _identity in missing_calls[:5] - ) - assert all( - response.status_code == 400 and response.json() == expected - for response, (_path, _body, expected, _identity) in zip(post_kill, missing_calls[:5]) - ), [response.text for response in post_kill] - recovered: Final = _post(candidate, "/v1/chat/completions", _chat_body(chat_model, "recovered", True)) - assert recovered.status_code == 200, recovered.text + with httpx.Client(base_url=candidate.client.base_url, timeout=15, trust_env=False) as fresh: + survivor: Final = Gateway(fresh, candidate.key, candidate.upstream_url) + post_kill: Final = tuple( + _post(survivor, path, body) for path, body, _expected, _identity in missing_calls[:5] + ) + assert all( + response.status_code == 400 and response.json() == expected + for response, (_path, _body, expected, _identity) in zip(post_kill, missing_calls[:5]) + ), [response.text for response in post_kill] + recovered: Final = _post( + survivor, "/v1/chat/completions", _chat_body(chat_model, "recovered", True) + ) + assert recovered.status_code == 200, recovered.text def test_eval_create_and_run_forward_client_metadata(gateway: Gateway) -> None: diff --git a/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py b/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py index 24063d8052a..37aa96c2467 100644 --- a/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py @@ -519,10 +519,12 @@ def test_killing_one_worker_leaves_the_sibling_serving_the_fallback(fb_rig: Rig) workers: Final = eventually(fb_rig.owned.worker_pids, lambda pids: len(pids) >= 2, seconds=30) with fb_rig.owned.replacing_worker(min(workers)) as kill: kill() - responses: Final = tuple( - fb_rig.gateway.request("POST", "/v1/chat/completions", _chat(fb_rig.models["fb"], _prompt("after kill"))) - for _ in range(3) - ) + with httpx.Client(base_url=fb_rig.gateway.client.base_url, timeout=15, trust_env=False) as fresh: + survivor: Final = Gateway(fresh, fb_rig.gateway.key, fb_rig.gateway.upstream_url) + responses: Final = tuple( + survivor.request("POST", "/v1/chat/completions", _chat(fb_rig.models["fb"], _prompt("after kill"))) + for _ in range(3) + ) for response in responses: assert response.status_code == 200, response.text assert _FALLBACK_TEXT in response.text, response.text