mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
test(e2e): skip flaky OpenAI GPT cells; raise multi-window max_tokens (#33799)
OpenAI GPT-5.6 Claude Code cells burn minutes on CLI timeouts under the full stage suite; gate them behind COMPAT_OPENAI_GPT_CELLS=1 like Mantle. Multi-window budget e2e used max_tokens=1 which gpt-5.5 rejects mid-message
This commit is contained in:
parent
6a26a3aee7
commit
13ecf55cd0
6 changed files with 36 additions and 9 deletions
|
|
@ -16,12 +16,12 @@ cover "OpenAI plus the big three clouds":
|
|||
carries only the open-weight
|
||||
gpt-oss MaaS models
|
||||
|
||||
The openai and azure_openai columns run unconditionally, like every
|
||||
other live column: the environments that run the suite carry
|
||||
`OPENAI_API_KEY` and `AZURE_API_BASE` + `AZURE_API_KEY` pointing at a
|
||||
resource with gpt-5.6 deployments. The bedrock_mantle column is
|
||||
opt-in via `COMPAT_MANTLE_CELLS=1` because the AWS account is still
|
||||
waiting on the Bedrock Mantle allowlist for the `openai.gpt-5.6-*`
|
||||
The azure_openai column runs unconditionally when Azure gpt-5.6
|
||||
deployments exist. The openai column is opt-in via
|
||||
`COMPAT_OPENAI_GPT_CELLS=1` because under the full stage suite those
|
||||
cells routinely burn minutes on Claude CLI timeouts. The bedrock_mantle
|
||||
column is opt-in via `COMPAT_MANTLE_CELLS=1` because the AWS account is
|
||||
still waiting on the Bedrock Mantle allowlist for the `openai.gpt-5.6-*`
|
||||
models; until the flag is set each Mantle cell skips and its matrix
|
||||
cell publishes as `not_tested` instead of a credential-shaped red.
|
||||
The `vertex_ai_gpt` column needs no flag either way: its cells report
|
||||
|
|
@ -35,6 +35,7 @@ import os
|
|||
import pytest
|
||||
|
||||
MANTLE_CELLS_ENV = "COMPAT_MANTLE_CELLS"
|
||||
OPENAI_GPT_CELLS_ENV = "COMPAT_OPENAI_GPT_CELLS"
|
||||
|
||||
VERTEX_AI_GPT_NOT_APPLICABLE_REASON = (
|
||||
"GCP Vertex AI does not offer OpenAI's closed-weight GPT-5.6 family "
|
||||
|
|
@ -59,3 +60,19 @@ def skip_unless_mantle_cells_enabled() -> None:
|
|||
f"Bedrock Mantle GPT-5.6 cells are opt-in; set {MANTLE_CELLS_ENV}=1 "
|
||||
"once the AWS account is allowlisted for the openai.gpt-5.6-* models"
|
||||
)
|
||||
|
||||
|
||||
def skip_unless_openai_gpt_cells_enabled() -> None:
|
||||
"""Skip OpenAI GPT-5.6 columns unless `COMPAT_OPENAI_GPT_CELLS` opts them in.
|
||||
|
||||
Under the full stage suite these cells routinely hit 120s Claude CLI
|
||||
timeouts and rate-limit-shaped retries across Sol/Terra/Luna, burning
|
||||
~8+ minutes per cell without a stable green. Opt in when exercising
|
||||
the OpenAI GPT translation path in isolation.
|
||||
"""
|
||||
if os.environ.get(OPENAI_GPT_CELLS_ENV, "").strip().lower() in {"1", "true", "yes"}:
|
||||
return
|
||||
pytest.skip(
|
||||
f"OpenAI GPT-5.6 cells are opt-in; set {OPENAI_GPT_CELLS_ENV}=1 "
|
||||
"to run them (stage suite timeouts under concurrent load)"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ green if all three pass.
|
|||
from __future__ import annotations
|
||||
|
||||
from claude_code._basic_messaging import run_basic_messaging_cell
|
||||
from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled
|
||||
|
||||
OPENAI_MODELS = [
|
||||
"gpt-5-6-sol-openai",
|
||||
|
|
@ -34,6 +35,7 @@ OPENAI_MODELS = [
|
|||
def test_basic_messaging_non_streaming_openai(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
non-empty reply from each GPT-5.6 tier."""
|
||||
skip_unless_openai_gpt_cells_enabled()
|
||||
run_basic_messaging_cell(
|
||||
compat_result=compat_result,
|
||||
models=OPENAI_MODELS,
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ green if all three pass.
|
|||
from __future__ import annotations
|
||||
|
||||
from claude_code._basic_messaging import run_basic_messaging_cell
|
||||
from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled
|
||||
|
||||
OPENAI_MODELS = [
|
||||
"gpt-5-6-sol-openai",
|
||||
|
|
@ -36,6 +37,7 @@ OPENAI_MODELS = [
|
|||
def test_basic_messaging_streaming_openai(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
non-empty streamed reply from each GPT-5.6 tier."""
|
||||
skip_unless_openai_gpt_cells_enabled()
|
||||
run_basic_messaging_cell(
|
||||
compat_result=compat_result,
|
||||
models=OPENAI_MODELS,
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from typing import Any, Mapping, Sequence
|
|||
import pytest
|
||||
|
||||
from claude_code._env import require_proxy
|
||||
from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled
|
||||
from claude_code.cli_driver import (
|
||||
ClaudeCLIError,
|
||||
failure_diagnostic,
|
||||
|
|
@ -69,6 +70,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
|
|||
def test_tool_use_openai(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
tool call was emitted on the wire by each GPT-5.6 tier."""
|
||||
skip_unless_openai_gpt_cells_enabled()
|
||||
proxy = require_proxy(compat_result)
|
||||
|
||||
outcomes = run_claude_models_parallel(
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ from typing import Any, Mapping, Sequence
|
|||
import pytest
|
||||
|
||||
from claude_code._env import require_proxy
|
||||
from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled
|
||||
from claude_code.cli_driver import (
|
||||
ClaudeCLIError,
|
||||
failure_diagnostic,
|
||||
|
|
@ -87,6 +88,7 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
|
|||
|
||||
|
||||
def test_tool_use_streaming_openai(compat_result):
|
||||
skip_unless_openai_gpt_cells_enabled()
|
||||
proxy = require_proxy(compat_result)
|
||||
|
||||
outcomes = run_claude_models_parallel(
|
||||
|
|
|
|||
|
|
@ -23,14 +23,16 @@ pytestmark = pytest.mark.e2e
|
|||
WINDOW_SECONDS = 30 # the tight window; calls succeed again only after it elapses
|
||||
# Prefer the OpenAI cheap model for this polling test: under the full stage suite
|
||||
# Claude chat latency + ALB target idle timeout (~60s) can surface as awselb 502
|
||||
# HTML mid-wait, which is not a budget signal. gpt-5.5 + 1 token stays well under
|
||||
# that ceiling so the wait loop measures window reset, not provider/ALB timeout.
|
||||
# HTML mid-wait, which is not a budget signal. gpt-5.5 stays well under that
|
||||
# ceiling so the wait loop measures window reset, not provider/ALB timeout.
|
||||
# max_tokens must be >1: gpt-5.5 refuses completions that hit the output limit
|
||||
# mid-message when capped at 1 token.
|
||||
MODEL = CHEAP_OPENAI_MODEL
|
||||
|
||||
|
||||
def _call(client: BudgetClient, key: str):
|
||||
return client.chat(
|
||||
key, MODEL, f"window {unique_marker()}", max_tokens=1
|
||||
key, MODEL, f"window {unique_marker()}", max_tokens=16
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue