test(e2e/claude_code): run openai and azure GPT cells unconditionally, gate only bedrock_mantle

The openai and azure_openai columns have working credentials in every suite runner, so only the bedrock_mantle column still needs an opt-in flag while the AWS account waits on the Mantle allowlist; COMPAT_GPT_CELLS becomes COMPAT_MANTLE_CELLS. The six tool_use cells now resolve the proxy through claude_code._env like the basic_messaging cells instead of hardcoding LITELLM_PROXY_BASE_URL/LITELLM_PROXY_API_KEY.
This commit is contained in:
mateo-berri 2026-07-16 17:25:39 -07:00
parent f93a84b01e
commit bb04a1ed15
13 changed files with 60 additions and 193 deletions

View file

@ -16,15 +16,16 @@ cover "OpenAI plus the big three clouds":
carries only the open-weight
gpt-oss MaaS models
Live GPT cells are opt-in via `COMPAT_GPT_CELLS=1`. The cron VM that
runs the scheduled suite and publishes the matrix must be provisioned
with the GPT-route credentials (`OPENAI_API_KEY` with available
quota, `AZURE_API_BASE` + `AZURE_API_KEY` with gpt-5.6
deployments, and Bedrock Mantle model access) before these cells can
pass, so until the flag is set each live cell skips and its matrix
The openai and azure_openai columns run unconditionally, like every
other live column: the environments that run the suite carry
`OPENAI_API_KEY` and `AZURE_API_BASE` + `AZURE_API_KEY` pointing at a
resource with gpt-5.6 deployments. The bedrock_mantle column is
opt-in via `COMPAT_MANTLE_CELLS=1` because the AWS account is still
waiting on the Bedrock Mantle allowlist for the `openai.gpt-5.6-*`
models; until the flag is set each Mantle cell skips and its matrix
cell publishes as `not_tested` instead of a credential-shaped red.
The `vertex_ai_gpt` column ignores the flag: its cells report a
static `not_applicable` and never touch the network.
The `vertex_ai_gpt` column needs no flag either way: its cells report
a static `not_applicable` and never touch the network.
"""
from __future__ import annotations
@ -33,7 +34,7 @@ import os
import pytest
GPT_CELLS_ENV = "COMPAT_GPT_CELLS"
MANTLE_CELLS_ENV = "COMPAT_MANTLE_CELLS"
VERTEX_AI_GPT_NOT_APPLICABLE_REASON = (
"GCP Vertex AI does not offer OpenAI's closed-weight GPT-5.6 family "
@ -43,18 +44,18 @@ VERTEX_AI_GPT_NOT_APPLICABLE_REASON = (
)
def skip_unless_gpt_cells_enabled() -> None:
"""Skip the calling test unless `COMPAT_GPT_CELLS` opts GPT cells in.
def skip_unless_mantle_cells_enabled() -> None:
"""Skip the calling test unless `COMPAT_MANTLE_CELLS` opts the
Bedrock Mantle cells in.
A skipped cell is recorded as `not_tested` in the published matrix
(see the skip handling in `tests/e2e/claude_code/conftest.py`),
which is the honest state for an environment that has no GPT-route
credentials yet.
which is the honest state while the AWS account has no Mantle
access to the GPT-5.6 models yet.
"""
if os.environ.get(GPT_CELLS_ENV, "").strip().lower() in {"1", "true", "yes"}:
if os.environ.get(MANTLE_CELLS_ENV, "").strip().lower() in {"1", "true", "yes"}:
return
pytest.skip(
f"GPT-5.6 cells are opt-in; set {GPT_CELLS_ENV}=1 once the proxy has "
"OpenAI / Azure OpenAI / Bedrock Mantle credentials for the "
"gpt-5-6-* aliases"
f"Bedrock Mantle GPT-5.6 cells are opt-in; set {MANTLE_CELLS_ENV}=1 "
"once the AWS account is allowlisted for the openai.gpt-5.6-* models"
)

View file

@ -20,14 +20,12 @@ The (feature, provider) for this cell is inferred from the file path by
feature_id provider
Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes
green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see
`claude_code._gpt_cells`).
green if all three pass.
"""
from __future__ import annotations
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
AZURE_OPENAI_MODELS = [
"gpt-5-6-sol-azure-openai",
@ -39,7 +37,6 @@ AZURE_OPENAI_MODELS = [
def test_basic_messaging_non_streaming_azure_openai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty reply from each GPT-5.6 tier."""
skip_unless_gpt_cells_enabled()
run_basic_messaging_cell(
compat_result=compat_result,
models=AZURE_OPENAI_MODELS,

View file

@ -19,14 +19,14 @@ The (feature, provider) for this cell is inferred from the file path by
feature_id provider
Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes
green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see
`claude_code._gpt_cells`).
green if all three pass. Mantle cells are opt-in via
COMPAT_MANTLE_CELLS=1 (see `claude_code._gpt_cells`).
"""
from __future__ import annotations
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
from claude_code._gpt_cells import skip_unless_mantle_cells_enabled
BEDROCK_MANTLE_MODELS = [
"gpt-5-6-sol-bedrock-mantle",
@ -38,7 +38,7 @@ BEDROCK_MANTLE_MODELS = [
def test_basic_messaging_non_streaming_bedrock_mantle(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty reply from each GPT-5.6 tier."""
skip_unless_gpt_cells_enabled()
skip_unless_mantle_cells_enabled()
run_basic_messaging_cell(
compat_result=compat_result,
models=BEDROCK_MANTLE_MODELS,

View file

@ -17,14 +17,12 @@ The (feature, provider) for this cell is inferred from the file path by
feature_id provider
Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes
green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see
`claude_code._gpt_cells`).
green if all three pass.
"""
from __future__ import annotations
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
OPENAI_MODELS = [
"gpt-5-6-sol-openai",
@ -36,7 +34,6 @@ OPENAI_MODELS = [
def test_basic_messaging_non_streaming_openai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty reply from each GPT-5.6 tier."""
skip_unless_gpt_cells_enabled()
run_basic_messaging_cell(
compat_result=compat_result,
models=OPENAI_MODELS,

View file

@ -19,14 +19,12 @@ The (feature, provider) for this cell is inferred from the file path by
feature_id provider
Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes
green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see
`claude_code._gpt_cells`).
green if all three pass.
"""
from __future__ import annotations
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
AZURE_OPENAI_MODELS = [
"gpt-5-6-sol-azure-openai",
@ -38,7 +36,6 @@ AZURE_OPENAI_MODELS = [
def test_basic_messaging_streaming_azure_openai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty streamed reply from each GPT-5.6 tier."""
skip_unless_gpt_cells_enabled()
run_basic_messaging_cell(
compat_result=compat_result,
models=AZURE_OPENAI_MODELS,

View file

@ -19,14 +19,14 @@ The (feature, provider) for this cell is inferred from the file path by
feature_id provider
Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes
green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see
`claude_code._gpt_cells`).
green if all three pass. Mantle cells are opt-in via
COMPAT_MANTLE_CELLS=1 (see `claude_code._gpt_cells`).
"""
from __future__ import annotations
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
from claude_code._gpt_cells import skip_unless_mantle_cells_enabled
BEDROCK_MANTLE_MODELS = [
"gpt-5-6-sol-bedrock-mantle",
@ -38,7 +38,7 @@ BEDROCK_MANTLE_MODELS = [
def test_basic_messaging_streaming_bedrock_mantle(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty streamed reply from each GPT-5.6 tier."""
skip_unless_gpt_cells_enabled()
skip_unless_mantle_cells_enabled()
run_basic_messaging_cell(
compat_result=compat_result,
models=BEDROCK_MANTLE_MODELS,

View file

@ -19,14 +19,12 @@ The (feature, provider) for this cell is inferred from the file path by
feature_id provider
Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes
green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see
`claude_code._gpt_cells`).
green if all three pass.
"""
from __future__ import annotations
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
OPENAI_MODELS = [
"gpt-5-6-sol-openai",
@ -38,7 +36,6 @@ OPENAI_MODELS = [
def test_basic_messaging_streaming_openai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty streamed reply from each GPT-5.6 tier."""
skip_unless_gpt_cells_enabled()
run_basic_messaging_cell(
compat_result=compat_result,
models=OPENAI_MODELS,

View file

@ -15,9 +15,6 @@ Bash is restricted to the exact command `echo pong` plus
`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the
security rationale.
GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see
`claude_code._gpt_cells`).
The (feature, provider) for this cell is inferred from the file path by
`tests/e2e/claude_code/conftest.py`:
@ -28,21 +25,17 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
import pytest
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
failure_diagnostic,
run_claude_models_parallel,
)
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
AZURE_OPENAI_MODELS = [
"gpt-5-6-sol-azure-openai",
"gpt-5-6-terra-azure-openai",
@ -77,28 +70,13 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
def test_tool_use_azure_openai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
tool call was emitted on the wire by each GPT-5.6 tier."""
skip_unless_gpt_cells_enabled()
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
compat_result.set(
{
"status": "fail",
"error": (
f"missing required env: set {PROXY_BASE_URL_ENV} and "
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
),
}
)
pytest.fail(
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
)
proxy = require_proxy(compat_result)
outcomes = run_claude_models_parallel(
models=AZURE_OPENAI_MODELS,
prompt=TOOL_USE_PROMPT,
base_url=base_url,
api_key=api_key,
base_url=proxy.base_url,
api_key=proxy.api_key,
extra_args=TOOL_USE_ARGS,
)

View file

@ -16,7 +16,7 @@ Bash is restricted to the exact command `echo pong` plus
`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the
security rationale.
GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see
Mantle cells are opt-in via COMPAT_MANTLE_CELLS=1 (see
`claude_code._gpt_cells`).
The (feature, provider) for this cell is inferred from the file path by
@ -29,21 +29,18 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
import pytest
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
from claude_code._env import require_proxy
from claude_code._gpt_cells import skip_unless_mantle_cells_enabled
from claude_code.cli_driver import (
ClaudeCLIError,
failure_diagnostic,
run_claude_models_parallel,
)
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
BEDROCK_MANTLE_MODELS = [
"gpt-5-6-sol-bedrock-mantle",
"gpt-5-6-terra-bedrock-mantle",
@ -78,28 +75,14 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
def test_tool_use_bedrock_mantle(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
tool call was emitted on the wire by each GPT-5.6 tier."""
skip_unless_gpt_cells_enabled()
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
compat_result.set(
{
"status": "fail",
"error": (
f"missing required env: set {PROXY_BASE_URL_ENV} and "
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
),
}
)
pytest.fail(
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
)
skip_unless_mantle_cells_enabled()
proxy = require_proxy(compat_result)
outcomes = run_claude_models_parallel(
models=BEDROCK_MANTLE_MODELS,
prompt=TOOL_USE_PROMPT,
base_url=base_url,
api_key=api_key,
base_url=proxy.base_url,
api_key=proxy.api_key,
extra_args=TOOL_USE_ARGS,
)

View file

@ -14,9 +14,6 @@ Bash is restricted to the exact command `echo pong` plus
`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the
security rationale.
GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see
`claude_code._gpt_cells`).
The (feature, provider) for this cell is inferred from the file path by
`tests/e2e/claude_code/conftest.py`:
@ -27,21 +24,17 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
import pytest
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
failure_diagnostic,
run_claude_models_parallel,
)
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
OPENAI_MODELS = [
"gpt-5-6-sol-openai",
"gpt-5-6-terra-openai",
@ -76,28 +69,13 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
def test_tool_use_openai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
tool call was emitted on the wire by each GPT-5.6 tier."""
skip_unless_gpt_cells_enabled()
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
compat_result.set(
{
"status": "fail",
"error": (
f"missing required env: set {PROXY_BASE_URL_ENV} and "
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
),
}
)
pytest.fail(
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
)
proxy = require_proxy(compat_result)
outcomes = run_claude_models_parallel(
models=OPENAI_MODELS,
prompt=TOOL_USE_PROMPT,
base_url=base_url,
api_key=api_key,
base_url=proxy.base_url,
api_key=proxy.api_key,
extra_args=TOOL_USE_ARGS,
)

View file

@ -17,9 +17,6 @@ Bash is restricted to the exact command `echo pong` plus
`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the
security rationale.
GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see
`claude_code._gpt_cells`).
The (feature, provider) for this cell is inferred from the file path by
`tests/e2e/claude_code/conftest.py`:
@ -30,21 +27,17 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
import pytest
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
failure_diagnostic,
run_claude_models_parallel,
)
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
AZURE_OPENAI_MODELS = [
"gpt-5-6-sol-azure-openai",
"gpt-5-6-terra-azure-openai",
@ -96,28 +89,13 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
def test_tool_use_streaming_azure_openai(compat_result):
skip_unless_gpt_cells_enabled()
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
compat_result.set(
{
"status": "fail",
"error": (
f"missing required env: set {PROXY_BASE_URL_ENV} and "
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
),
}
)
pytest.fail(
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
)
proxy = require_proxy(compat_result)
outcomes = run_claude_models_parallel(
models=AZURE_OPENAI_MODELS,
prompt=TOOL_USE_PROMPT,
base_url=base_url,
api_key=api_key,
base_url=proxy.base_url,
api_key=proxy.api_key,
extra_args=TOOL_USE_ARGS,
)

View file

@ -17,7 +17,7 @@ Bash is restricted to the exact command `echo pong` plus
`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the
security rationale.
GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see
Mantle cells are opt-in via COMPAT_MANTLE_CELLS=1 (see
`claude_code._gpt_cells`).
The (feature, provider) for this cell is inferred from the file path by
@ -30,21 +30,18 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
import pytest
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
from claude_code._env import require_proxy
from claude_code._gpt_cells import skip_unless_mantle_cells_enabled
from claude_code.cli_driver import (
ClaudeCLIError,
failure_diagnostic,
run_claude_models_parallel,
)
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
BEDROCK_MANTLE_MODELS = [
"gpt-5-6-sol-bedrock-mantle",
"gpt-5-6-terra-bedrock-mantle",
@ -96,28 +93,14 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
def test_tool_use_streaming_bedrock_mantle(compat_result):
skip_unless_gpt_cells_enabled()
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
compat_result.set(
{
"status": "fail",
"error": (
f"missing required env: set {PROXY_BASE_URL_ENV} and "
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
),
}
)
pytest.fail(
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
)
skip_unless_mantle_cells_enabled()
proxy = require_proxy(compat_result)
outcomes = run_claude_models_parallel(
models=BEDROCK_MANTLE_MODELS,
prompt=TOOL_USE_PROMPT,
base_url=base_url,
api_key=api_key,
base_url=proxy.base_url,
api_key=proxy.api_key,
extra_args=TOOL_USE_ARGS,
)

View file

@ -15,9 +15,6 @@ Bash is restricted to the exact command `echo pong` plus
`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the
security rationale.
GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see
`claude_code._gpt_cells`).
The (feature, provider) for this cell is inferred from the file path by
`tests/e2e/claude_code/conftest.py`:
@ -28,21 +25,17 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
import pytest
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
failure_diagnostic,
run_claude_models_parallel,
)
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
OPENAI_MODELS = [
"gpt-5-6-sol-openai",
"gpt-5-6-terra-openai",
@ -94,28 +87,13 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
def test_tool_use_streaming_openai(compat_result):
skip_unless_gpt_cells_enabled()
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
compat_result.set(
{
"status": "fail",
"error": (
f"missing required env: set {PROXY_BASE_URL_ENV} and "
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
),
}
)
pytest.fail(
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
)
proxy = require_proxy(compat_result)
outcomes = run_claude_models_parallel(
models=OPENAI_MODELS,
prompt=TOOL_USE_PROMPT,
base_url=base_url,
api_key=api_key,
base_url=proxy.base_url,
api_key=proxy.api_key,
extra_args=TOOL_USE_ARGS,
)