test(e2e): skip flaky OpenAI GPT cells; raise multi-window max_tokens (#33799)

OpenAI GPT-5.6 Claude Code cells burn minutes on CLI timeouts under the
full stage suite; gate them behind COMPAT_OPENAI_GPT_CELLS=1 like Mantle.
Multi-window budget e2e used max_tokens=1 which gpt-5.5 rejects mid-message
This commit is contained in:
mubashir1osmani 2026-07-17 19:07:18 -07:00 committed by GitHub
parent 6a26a3aee7
commit 13ecf55cd0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 36 additions and 9 deletions

View file

@ -16,12 +16,12 @@ cover "OpenAI plus the big three clouds":
carries only the open-weight
gpt-oss MaaS models
The openai and azure_openai columns run unconditionally, like every
other live column: the environments that run the suite carry
`OPENAI_API_KEY` and `AZURE_API_BASE` + `AZURE_API_KEY` pointing at a
resource with gpt-5.6 deployments. The bedrock_mantle column is
opt-in via `COMPAT_MANTLE_CELLS=1` because the AWS account is still
waiting on the Bedrock Mantle allowlist for the `openai.gpt-5.6-*`
The azure_openai column runs unconditionally when Azure gpt-5.6
deployments exist. The openai column is opt-in via
`COMPAT_OPENAI_GPT_CELLS=1` because under the full stage suite those
cells routinely burn minutes on Claude CLI timeouts. The bedrock_mantle
column is opt-in via `COMPAT_MANTLE_CELLS=1` because the AWS account is
still waiting on the Bedrock Mantle allowlist for the `openai.gpt-5.6-*`
models; until the flag is set each Mantle cell skips and its matrix
cell publishes as `not_tested` instead of a credential-shaped red.
The `vertex_ai_gpt` column needs no flag either way: its cells report
@ -35,6 +35,7 @@ import os
import pytest
MANTLE_CELLS_ENV = "COMPAT_MANTLE_CELLS"
OPENAI_GPT_CELLS_ENV = "COMPAT_OPENAI_GPT_CELLS"
VERTEX_AI_GPT_NOT_APPLICABLE_REASON = (
"GCP Vertex AI does not offer OpenAI's closed-weight GPT-5.6 family "
@ -59,3 +60,19 @@ def skip_unless_mantle_cells_enabled() -> None:
f"Bedrock Mantle GPT-5.6 cells are opt-in; set {MANTLE_CELLS_ENV}=1 "
"once the AWS account is allowlisted for the openai.gpt-5.6-* models"
)
def skip_unless_openai_gpt_cells_enabled() -> None:
"""Skip OpenAI GPT-5.6 columns unless `COMPAT_OPENAI_GPT_CELLS` opts them in.
Under the full stage suite these cells routinely hit 120s Claude CLI
timeouts and rate-limit-shaped retries across Sol/Terra/Luna, burning
~8+ minutes per cell without a stable green. Opt in when exercising
the OpenAI GPT translation path in isolation.
"""
if os.environ.get(OPENAI_GPT_CELLS_ENV, "").strip().lower() in {"1", "true", "yes"}:
return
pytest.skip(
f"OpenAI GPT-5.6 cells are opt-in; set {OPENAI_GPT_CELLS_ENV}=1 "
"to run them (stage suite timeouts under concurrent load)"
)

View file

@ -23,6 +23,7 @@ green if all three pass.
from __future__ import annotations
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled
OPENAI_MODELS = [
"gpt-5-6-sol-openai",
@ -34,6 +35,7 @@ OPENAI_MODELS = [
def test_basic_messaging_non_streaming_openai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty reply from each GPT-5.6 tier."""
skip_unless_openai_gpt_cells_enabled()
run_basic_messaging_cell(
compat_result=compat_result,
models=OPENAI_MODELS,

View file

@ -25,6 +25,7 @@ green if all three pass.
from __future__ import annotations
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled
OPENAI_MODELS = [
"gpt-5-6-sol-openai",
@ -36,6 +37,7 @@ OPENAI_MODELS = [
def test_basic_messaging_streaming_openai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty streamed reply from each GPT-5.6 tier."""
skip_unless_openai_gpt_cells_enabled()
run_basic_messaging_cell(
compat_result=compat_result,
models=OPENAI_MODELS,

View file

@ -29,6 +29,7 @@ from typing import Any, Mapping, Sequence
import pytest
from claude_code._env import require_proxy
from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled
from claude_code.cli_driver import (
ClaudeCLIError,
failure_diagnostic,
@ -69,6 +70,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
def test_tool_use_openai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
tool call was emitted on the wire by each GPT-5.6 tier."""
skip_unless_openai_gpt_cells_enabled()
proxy = require_proxy(compat_result)
outcomes = run_claude_models_parallel(

View file

@ -30,6 +30,7 @@ from typing import Any, Mapping, Sequence
import pytest
from claude_code._env import require_proxy
from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled
from claude_code.cli_driver import (
ClaudeCLIError,
failure_diagnostic,
@ -87,6 +88,7 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
def test_tool_use_streaming_openai(compat_result):
skip_unless_openai_gpt_cells_enabled()
proxy = require_proxy(compat_result)
outcomes = run_claude_models_parallel(

View file

@ -23,14 +23,16 @@ pytestmark = pytest.mark.e2e
WINDOW_SECONDS = 30 # the tight window; calls succeed again only after it elapses
# Prefer the OpenAI cheap model for this polling test: under the full stage suite
# Claude chat latency + ALB target idle timeout (~60s) can surface as awselb 502
# HTML mid-wait, which is not a budget signal. gpt-5.5 + 1 token stays well under
# that ceiling so the wait loop measures window reset, not provider/ALB timeout.
# HTML mid-wait, which is not a budget signal. gpt-5.5 stays well under that
# ceiling so the wait loop measures window reset, not provider/ALB timeout.
# max_tokens must be >1: gpt-5.5 refuses completions that hit the output limit
# mid-message when capped at 1 token.
MODEL = CHEAP_OPENAI_MODEL
def _call(client: BudgetClient, key: str):
return client.chat(
key, MODEL, f"window {unique_marker()}", max_tokens=1
key, MODEL, f"window {unique_marker()}", max_tokens=16
)