From 36143b53f900d8962b1cf72749f85f20e870e961 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:34:27 -0700 Subject: [PATCH 1/3] test(responses): bound the background stream cancel e2e so an upstream stall skips fast test_cancel_streaming_response drained the whole background stream before cancelling, so an OpenAI keepalive stall held the e2e_openai_endpoints job for 301s and failed it on a generic APIError, and on a healthy day it cancelled an already completed response and swallowed the 400 without verifying a cancel. Cancel at the first event carrying a response id, bound admission to 90s, skip naming the stall when only keepalives arrived, and assert status == cancelled --- .../test_e2e_openai_responses_api.py | 70 +++++++++++-------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py b/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py index abae26e02cd..604f1c84d20 100644 --- a/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py +++ b/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py @@ -1,6 +1,10 @@ +import time + import httpx -from openai import OpenAI, BadRequestError, NotFoundError, APIStatusError import pytest +from openai import APIStatusError, BadRequestError, NotFoundError, OpenAI + +BACKGROUND_STREAM_ADMISSION_DEADLINE_SECONDS = 90 def generate_key(): @@ -154,42 +158,46 @@ def test_cancel_response(): def test_cancel_streaming_response(): - try: - client = get_test_client() - from litellm.types.llms.openai import ResponsesAPIResponse + client = get_test_client() + started = time.monotonic() + stream = client.responses.create( + model="gpt-5.5", + input="just respond with the word 'ping'", + stream=True, + background=True, + timeout=BACKGROUND_STREAM_ADMISSION_DEADLINE_SECONDS, + ) - stream = client.responses.create( - model="gpt-5.5", - input="just respond with the word 'ping'", - stream=True, - background=True, - ) - - collected_chunks = [] - response_id = None + keepalive_events = 0 + response_id = None + with stream: for chunk in stream: print("stream chunk=", chunk) - collected_chunks.append(chunk) - # Extract response ID from the first chunk that has it - if ( - response_id is None - and hasattr(chunk, "response") - and hasattr(chunk.response, "id") - ): + if chunk.type == "keepalive": + keepalive_events += 1 + elif getattr(chunk, "response", None) is not None: response_id = chunk.response.id + break + if time.monotonic() - started > BACKGROUND_STREAM_ADMISSION_DEADLINE_SECONDS: + break - assert len(collected_chunks) > 0 + elapsed = time.monotonic() - started + if response_id is None and keepalive_events: + pytest.skip( + f"OpenAI held the background stream in keepalive for {elapsed:.0f}s " + f"({keepalive_events} keepalive events) without creating the response" + ) + assert response_id is not None, f"no response event within {elapsed:.0f}s of streaming a background response" - # cancel the response if we got a response ID - if response_id: - cancel_response = client.responses.cancel(response_id) - print("CANCEL streaming response=", cancel_response) - assert hasattr(cancel_response, "id") - except Exception as e: - if "Cannot cancel a completed response" in str(e): - pass - else: - raise e + try: + cancel_response = client.responses.cancel(response_id) + except BadRequestError as e: + if "Cannot cancel a completed response" not in str(e): + raise + print("response completed before cancel=", e) + return + print("CANCEL streaming response=", cancel_response) + assert cancel_response.status == "cancelled" def test_cancel_invalid_response_id(): From b38516da88ba4eb7ba3fa584e08133881f4d1108 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:37:31 -0700 Subject: [PATCH 2/3] test(responses): make the background stream cancel deterministic A five-token response can complete before the cancel lands, which put the test back on the "Cannot cancel a completed response" path it used to swallow. Ask for a long generation so the cancel always beats completion, and assert the cancelled status unconditionally --- .../test_e2e_openai_responses_api.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py b/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py index 604f1c84d20..0dbecbe2801 100644 --- a/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py +++ b/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py @@ -162,7 +162,7 @@ def test_cancel_streaming_response(): started = time.monotonic() stream = client.responses.create( model="gpt-5.5", - input="just respond with the word 'ping'", + input="count from 1 to 500, one number per line", stream=True, background=True, timeout=BACKGROUND_STREAM_ADMISSION_DEADLINE_SECONDS, @@ -189,13 +189,7 @@ def test_cancel_streaming_response(): ) assert response_id is not None, f"no response event within {elapsed:.0f}s of streaming a background response" - try: - cancel_response = client.responses.cancel(response_id) - except BadRequestError as e: - if "Cannot cancel a completed response" not in str(e): - raise - print("response completed before cancel=", e) - return + cancel_response = client.responses.cancel(response_id) print("CANCEL streaming response=", cancel_response) assert cancel_response.status == "cancelled" From 0904a9223bb4e01c2ad9d6ceb56529fe4637b4b9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:56:56 -0700 Subject: [PATCH 3/3] test(responses): collect the admitted stream events without local mutation --- .../test_e2e_openai_responses_api.py | 46 +++++++++++-------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py b/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py index 0dbecbe2801..7e338dafb86 100644 --- a/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py +++ b/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py @@ -1,10 +1,13 @@ import time +from collections.abc import Iterator +from typing import Final import httpx import pytest -from openai import APIStatusError, BadRequestError, NotFoundError, OpenAI +from openai import APIStatusError, BadRequestError, NotFoundError, OpenAI, Stream +from openai.types.responses import ResponseStreamEvent -BACKGROUND_STREAM_ADMISSION_DEADLINE_SECONDS = 90 +BACKGROUND_STREAM_ADMISSION_DEADLINE_SECONDS: Final = 90 def generate_key(): @@ -157,10 +160,25 @@ def test_cancel_response(): raise e +def admitted_response_id(chunk: ResponseStreamEvent) -> str | None: + response: Final = getattr(chunk, "response", None) + return None if response is None else response.id + + +def events_until_admission(stream: Stream[ResponseStreamEvent], started: float) -> Iterator[ResponseStreamEvent]: + for chunk in stream: + print("stream chunk=", chunk) + yield chunk + if admitted_response_id(chunk) is not None: + return + if time.monotonic() - started > BACKGROUND_STREAM_ADMISSION_DEADLINE_SECONDS: + return + + def test_cancel_streaming_response(): - client = get_test_client() - started = time.monotonic() - stream = client.responses.create( + client: Final = get_test_client() + started: Final = time.monotonic() + stream: Final = client.responses.create( model="gpt-5.5", input="count from 1 to 500, one number per line", stream=True, @@ -168,20 +186,12 @@ def test_cancel_streaming_response(): timeout=BACKGROUND_STREAM_ADMISSION_DEADLINE_SECONDS, ) - keepalive_events = 0 - response_id = None with stream: - for chunk in stream: - print("stream chunk=", chunk) - if chunk.type == "keepalive": - keepalive_events += 1 - elif getattr(chunk, "response", None) is not None: - response_id = chunk.response.id - break - if time.monotonic() - started > BACKGROUND_STREAM_ADMISSION_DEADLINE_SECONDS: - break + events: Final = tuple(events_until_admission(stream, started)) - elapsed = time.monotonic() - started + elapsed: Final = time.monotonic() - started + keepalive_events: Final = sum(1 for chunk in events if chunk.type == "keepalive") + response_id: Final = next((rid for rid in map(admitted_response_id, events) if rid is not None), None) if response_id is None and keepalive_events: pytest.skip( f"OpenAI held the background stream in keepalive for {elapsed:.0f}s " @@ -189,7 +199,7 @@ def test_cancel_streaming_response(): ) assert response_id is not None, f"no response event within {elapsed:.0f}s of streaming a background response" - cancel_response = client.responses.cancel(response_id) + cancel_response: Final = client.responses.cancel(response_id) print("CANCEL streaming response=", cancel_response) assert cancel_response.status == "cancelled"