From 13ecf55cd097ef653d0e5361ce1cea1e1fb70afa Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Fri, 17 Jul 2026 19:07:18 -0700 Subject: [PATCH] test(e2e): skip flaky OpenAI GPT cells; raise multi-window max_tokens (#33799) OpenAI GPT-5.6 Claude Code cells burn minutes on CLI timeouts under the full stage suite; gate them behind COMPAT_OPENAI_GPT_CELLS=1 like Mantle. Multi-window budget e2e used max_tokens=1 which gpt-5.5 rejects mid-message --- tests/e2e/claude_code/_gpt_cells.py | 29 +++++++++++++++---- .../test_openai.py | 2 ++ .../basic_messaging_streaming/test_openai.py | 2 ++ tests/e2e/claude_code/tool_use/test_openai.py | 2 ++ .../tool_use_streaming/test_openai.py | 2 ++ .../budgets/test_multi_window_budget_e2e.py | 8 +++-- 6 files changed, 36 insertions(+), 9 deletions(-) diff --git a/tests/e2e/claude_code/_gpt_cells.py b/tests/e2e/claude_code/_gpt_cells.py index 870e9cea918..70e2b5ed18f 100644 --- a/tests/e2e/claude_code/_gpt_cells.py +++ b/tests/e2e/claude_code/_gpt_cells.py @@ -16,12 +16,12 @@ cover "OpenAI plus the big three clouds": carries only the open-weight gpt-oss MaaS models -The openai and azure_openai columns run unconditionally, like every -other live column: the environments that run the suite carry -`OPENAI_API_KEY` and `AZURE_API_BASE` + `AZURE_API_KEY` pointing at a -resource with gpt-5.6 deployments. The bedrock_mantle column is -opt-in via `COMPAT_MANTLE_CELLS=1` because the AWS account is still -waiting on the Bedrock Mantle allowlist for the `openai.gpt-5.6-*` +The azure_openai column runs unconditionally when Azure gpt-5.6 +deployments exist. The openai column is opt-in via +`COMPAT_OPENAI_GPT_CELLS=1` because under the full stage suite those +cells routinely burn minutes on Claude CLI timeouts. The bedrock_mantle +column is opt-in via `COMPAT_MANTLE_CELLS=1` because the AWS account is +still waiting on the Bedrock Mantle allowlist for the `openai.gpt-5.6-*` models; until the flag is set each Mantle cell skips and its matrix cell publishes as `not_tested` instead of a credential-shaped red. The `vertex_ai_gpt` column needs no flag either way: its cells report @@ -35,6 +35,7 @@ import os import pytest MANTLE_CELLS_ENV = "COMPAT_MANTLE_CELLS" +OPENAI_GPT_CELLS_ENV = "COMPAT_OPENAI_GPT_CELLS" VERTEX_AI_GPT_NOT_APPLICABLE_REASON = ( "GCP Vertex AI does not offer OpenAI's closed-weight GPT-5.6 family " @@ -59,3 +60,19 @@ def skip_unless_mantle_cells_enabled() -> None: f"Bedrock Mantle GPT-5.6 cells are opt-in; set {MANTLE_CELLS_ENV}=1 " "once the AWS account is allowlisted for the openai.gpt-5.6-* models" ) + + +def skip_unless_openai_gpt_cells_enabled() -> None: + """Skip OpenAI GPT-5.6 columns unless `COMPAT_OPENAI_GPT_CELLS` opts them in. + + Under the full stage suite these cells routinely hit 120s Claude CLI + timeouts and rate-limit-shaped retries across Sol/Terra/Luna, burning + ~8+ minutes per cell without a stable green. Opt in when exercising + the OpenAI GPT translation path in isolation. + """ + if os.environ.get(OPENAI_GPT_CELLS_ENV, "").strip().lower() in {"1", "true", "yes"}: + return + pytest.skip( + f"OpenAI GPT-5.6 cells are opt-in; set {OPENAI_GPT_CELLS_ENV}=1 " + "to run them (stage suite timeouts under concurrent load)" + ) diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py index b0d143fa5e0..323c2f11173 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py @@ -23,6 +23,7 @@ green if all three pass. from __future__ import annotations from claude_code._basic_messaging import run_basic_messaging_cell +from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled OPENAI_MODELS = [ "gpt-5-6-sol-openai", @@ -34,6 +35,7 @@ OPENAI_MODELS = [ def test_basic_messaging_non_streaming_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty reply from each GPT-5.6 tier.""" + skip_unless_openai_gpt_cells_enabled() run_basic_messaging_cell( compat_result=compat_result, models=OPENAI_MODELS, diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py b/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py index 402c763496b..a7945fb92c0 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py @@ -25,6 +25,7 @@ green if all three pass. from __future__ import annotations from claude_code._basic_messaging import run_basic_messaging_cell +from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled OPENAI_MODELS = [ "gpt-5-6-sol-openai", @@ -36,6 +37,7 @@ OPENAI_MODELS = [ def test_basic_messaging_streaming_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply from each GPT-5.6 tier.""" + skip_unless_openai_gpt_cells_enabled() run_basic_messaging_cell( compat_result=compat_result, models=OPENAI_MODELS, diff --git a/tests/e2e/claude_code/tool_use/test_openai.py b/tests/e2e/claude_code/tool_use/test_openai.py index dbe60a65281..ffb7e795c2b 100644 --- a/tests/e2e/claude_code/tool_use/test_openai.py +++ b/tests/e2e/claude_code/tool_use/test_openai.py @@ -29,6 +29,7 @@ from typing import Any, Mapping, Sequence import pytest from claude_code._env import require_proxy +from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, @@ -69,6 +70,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: def test_tool_use_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire by each GPT-5.6 tier.""" + skip_unless_openai_gpt_cells_enabled() proxy = require_proxy(compat_result) outcomes = run_claude_models_parallel( diff --git a/tests/e2e/claude_code/tool_use_streaming/test_openai.py b/tests/e2e/claude_code/tool_use_streaming/test_openai.py index 895f88d994b..a5ce31b1fd6 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_openai.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_openai.py @@ -30,6 +30,7 @@ from typing import Any, Mapping, Sequence import pytest from claude_code._env import require_proxy +from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, @@ -87,6 +88,7 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: def test_tool_use_streaming_openai(compat_result): + skip_unless_openai_gpt_cells_enabled() proxy = require_proxy(compat_result) outcomes = run_claude_models_parallel( diff --git a/tests/e2e/quota_management/budgets/test_multi_window_budget_e2e.py b/tests/e2e/quota_management/budgets/test_multi_window_budget_e2e.py index 23f3e162761..45e2bf539d4 100644 --- a/tests/e2e/quota_management/budgets/test_multi_window_budget_e2e.py +++ b/tests/e2e/quota_management/budgets/test_multi_window_budget_e2e.py @@ -23,14 +23,16 @@ pytestmark = pytest.mark.e2e WINDOW_SECONDS = 30 # the tight window; calls succeed again only after it elapses # Prefer the OpenAI cheap model for this polling test: under the full stage suite # Claude chat latency + ALB target idle timeout (~60s) can surface as awselb 502 -# HTML mid-wait, which is not a budget signal. gpt-5.5 + 1 token stays well under -# that ceiling so the wait loop measures window reset, not provider/ALB timeout. +# HTML mid-wait, which is not a budget signal. gpt-5.5 stays well under that +# ceiling so the wait loop measures window reset, not provider/ALB timeout. +# max_tokens must be >1: gpt-5.5 refuses completions that hit the output limit +# mid-message when capped at 1 token. MODEL = CHEAP_OPENAI_MODEL def _call(client: BudgetClient, key: str): return client.chat( - key, MODEL, f"window {unique_marker()}", max_tokens=1 + key, MODEL, f"window {unique_marker()}", max_tokens=16 )