mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-10 03:28:53 +00:00
test(e2e/claude_code): run openai and azure GPT cells unconditionally, gate only bedrock_mantle
The openai and azure_openai columns have working credentials in every suite runner, so only the bedrock_mantle column still needs an opt-in flag while the AWS account waits on the Mantle allowlist; COMPAT_GPT_CELLS becomes COMPAT_MANTLE_CELLS. The six tool_use cells now resolve the proxy through claude_code._env like the basic_messaging cells instead of hardcoding LITELLM_PROXY_BASE_URL/LITELLM_PROXY_API_KEY.
This commit is contained in:
parent
f93a84b01e
commit
bb04a1ed15
13 changed files with 60 additions and 193 deletions
|
|
@ -16,15 +16,16 @@ cover "OpenAI plus the big three clouds":
|
|||
carries only the open-weight
|
||||
gpt-oss MaaS models
|
||||
|
||||
Live GPT cells are opt-in via `COMPAT_GPT_CELLS=1`. The cron VM that
|
||||
runs the scheduled suite and publishes the matrix must be provisioned
|
||||
with the GPT-route credentials (`OPENAI_API_KEY` with available
|
||||
quota, `AZURE_API_BASE` + `AZURE_API_KEY` with gpt-5.6
|
||||
deployments, and Bedrock Mantle model access) before these cells can
|
||||
pass, so until the flag is set each live cell skips and its matrix
|
||||
The openai and azure_openai columns run unconditionally, like every
|
||||
other live column: the environments that run the suite carry
|
||||
`OPENAI_API_KEY` and `AZURE_API_BASE` + `AZURE_API_KEY` pointing at a
|
||||
resource with gpt-5.6 deployments. The bedrock_mantle column is
|
||||
opt-in via `COMPAT_MANTLE_CELLS=1` because the AWS account is still
|
||||
waiting on the Bedrock Mantle allowlist for the `openai.gpt-5.6-*`
|
||||
models; until the flag is set each Mantle cell skips and its matrix
|
||||
cell publishes as `not_tested` instead of a credential-shaped red.
|
||||
The `vertex_ai_gpt` column ignores the flag: its cells report a
|
||||
static `not_applicable` and never touch the network.
|
||||
The `vertex_ai_gpt` column needs no flag either way: its cells report
|
||||
a static `not_applicable` and never touch the network.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -33,7 +34,7 @@ import os
|
|||
|
||||
import pytest
|
||||
|
||||
GPT_CELLS_ENV = "COMPAT_GPT_CELLS"
|
||||
MANTLE_CELLS_ENV = "COMPAT_MANTLE_CELLS"
|
||||
|
||||
VERTEX_AI_GPT_NOT_APPLICABLE_REASON = (
|
||||
"GCP Vertex AI does not offer OpenAI's closed-weight GPT-5.6 family "
|
||||
|
|
@ -43,18 +44,18 @@ VERTEX_AI_GPT_NOT_APPLICABLE_REASON = (
|
|||
)
|
||||
|
||||
|
||||
def skip_unless_gpt_cells_enabled() -> None:
|
||||
"""Skip the calling test unless `COMPAT_GPT_CELLS` opts GPT cells in.
|
||||
def skip_unless_mantle_cells_enabled() -> None:
|
||||
"""Skip the calling test unless `COMPAT_MANTLE_CELLS` opts the
|
||||
Bedrock Mantle cells in.
|
||||
|
||||
A skipped cell is recorded as `not_tested` in the published matrix
|
||||
(see the skip handling in `tests/e2e/claude_code/conftest.py`),
|
||||
which is the honest state for an environment that has no GPT-route
|
||||
credentials yet.
|
||||
which is the honest state while the AWS account has no Mantle
|
||||
access to the GPT-5.6 models yet.
|
||||
"""
|
||||
if os.environ.get(GPT_CELLS_ENV, "").strip().lower() in {"1", "true", "yes"}:
|
||||
if os.environ.get(MANTLE_CELLS_ENV, "").strip().lower() in {"1", "true", "yes"}:
|
||||
return
|
||||
pytest.skip(
|
||||
f"GPT-5.6 cells are opt-in; set {GPT_CELLS_ENV}=1 once the proxy has "
|
||||
"OpenAI / Azure OpenAI / Bedrock Mantle credentials for the "
|
||||
"gpt-5-6-* aliases"
|
||||
f"Bedrock Mantle GPT-5.6 cells are opt-in; set {MANTLE_CELLS_ENV}=1 "
|
||||
"once the AWS account is allowlisted for the openai.gpt-5.6-* models"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -20,14 +20,12 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
feature_id provider
|
||||
|
||||
Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes
|
||||
green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see
|
||||
`claude_code._gpt_cells`).
|
||||
green if all three pass.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from claude_code._basic_messaging import run_basic_messaging_cell
|
||||
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
|
||||
|
||||
AZURE_OPENAI_MODELS = [
|
||||
"gpt-5-6-sol-azure-openai",
|
||||
|
|
@ -39,7 +37,6 @@ AZURE_OPENAI_MODELS = [
|
|||
def test_basic_messaging_non_streaming_azure_openai(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
non-empty reply from each GPT-5.6 tier."""
|
||||
skip_unless_gpt_cells_enabled()
|
||||
run_basic_messaging_cell(
|
||||
compat_result=compat_result,
|
||||
models=AZURE_OPENAI_MODELS,
|
||||
|
|
|
|||
|
|
@ -19,14 +19,14 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
feature_id provider
|
||||
|
||||
Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes
|
||||
green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see
|
||||
`claude_code._gpt_cells`).
|
||||
green if all three pass. Mantle cells are opt-in via
|
||||
COMPAT_MANTLE_CELLS=1 (see `claude_code._gpt_cells`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from claude_code._basic_messaging import run_basic_messaging_cell
|
||||
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
|
||||
from claude_code._gpt_cells import skip_unless_mantle_cells_enabled
|
||||
|
||||
BEDROCK_MANTLE_MODELS = [
|
||||
"gpt-5-6-sol-bedrock-mantle",
|
||||
|
|
@ -38,7 +38,7 @@ BEDROCK_MANTLE_MODELS = [
|
|||
def test_basic_messaging_non_streaming_bedrock_mantle(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
non-empty reply from each GPT-5.6 tier."""
|
||||
skip_unless_gpt_cells_enabled()
|
||||
skip_unless_mantle_cells_enabled()
|
||||
run_basic_messaging_cell(
|
||||
compat_result=compat_result,
|
||||
models=BEDROCK_MANTLE_MODELS,
|
||||
|
|
|
|||
|
|
@ -17,14 +17,12 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
feature_id provider
|
||||
|
||||
Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes
|
||||
green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see
|
||||
`claude_code._gpt_cells`).
|
||||
green if all three pass.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from claude_code._basic_messaging import run_basic_messaging_cell
|
||||
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
|
||||
|
||||
OPENAI_MODELS = [
|
||||
"gpt-5-6-sol-openai",
|
||||
|
|
@ -36,7 +34,6 @@ OPENAI_MODELS = [
|
|||
def test_basic_messaging_non_streaming_openai(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
non-empty reply from each GPT-5.6 tier."""
|
||||
skip_unless_gpt_cells_enabled()
|
||||
run_basic_messaging_cell(
|
||||
compat_result=compat_result,
|
||||
models=OPENAI_MODELS,
|
||||
|
|
|
|||
|
|
@ -19,14 +19,12 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
feature_id provider
|
||||
|
||||
Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes
|
||||
green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see
|
||||
`claude_code._gpt_cells`).
|
||||
green if all three pass.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from claude_code._basic_messaging import run_basic_messaging_cell
|
||||
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
|
||||
|
||||
AZURE_OPENAI_MODELS = [
|
||||
"gpt-5-6-sol-azure-openai",
|
||||
|
|
@ -38,7 +36,6 @@ AZURE_OPENAI_MODELS = [
|
|||
def test_basic_messaging_streaming_azure_openai(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
non-empty streamed reply from each GPT-5.6 tier."""
|
||||
skip_unless_gpt_cells_enabled()
|
||||
run_basic_messaging_cell(
|
||||
compat_result=compat_result,
|
||||
models=AZURE_OPENAI_MODELS,
|
||||
|
|
|
|||
|
|
@ -19,14 +19,14 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
feature_id provider
|
||||
|
||||
Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes
|
||||
green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see
|
||||
`claude_code._gpt_cells`).
|
||||
green if all three pass. Mantle cells are opt-in via
|
||||
COMPAT_MANTLE_CELLS=1 (see `claude_code._gpt_cells`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from claude_code._basic_messaging import run_basic_messaging_cell
|
||||
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
|
||||
from claude_code._gpt_cells import skip_unless_mantle_cells_enabled
|
||||
|
||||
BEDROCK_MANTLE_MODELS = [
|
||||
"gpt-5-6-sol-bedrock-mantle",
|
||||
|
|
@ -38,7 +38,7 @@ BEDROCK_MANTLE_MODELS = [
|
|||
def test_basic_messaging_streaming_bedrock_mantle(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
non-empty streamed reply from each GPT-5.6 tier."""
|
||||
skip_unless_gpt_cells_enabled()
|
||||
skip_unless_mantle_cells_enabled()
|
||||
run_basic_messaging_cell(
|
||||
compat_result=compat_result,
|
||||
models=BEDROCK_MANTLE_MODELS,
|
||||
|
|
|
|||
|
|
@ -19,14 +19,12 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
feature_id provider
|
||||
|
||||
Every GPT cell exercises the three GPT-5.6 tiers; the cell only goes
|
||||
green if all three pass. Cells are opt-in via COMPAT_GPT_CELLS=1 (see
|
||||
`claude_code._gpt_cells`).
|
||||
green if all three pass.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from claude_code._basic_messaging import run_basic_messaging_cell
|
||||
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
|
||||
|
||||
OPENAI_MODELS = [
|
||||
"gpt-5-6-sol-openai",
|
||||
|
|
@ -38,7 +36,6 @@ OPENAI_MODELS = [
|
|||
def test_basic_messaging_streaming_openai(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
non-empty streamed reply from each GPT-5.6 tier."""
|
||||
skip_unless_gpt_cells_enabled()
|
||||
run_basic_messaging_cell(
|
||||
compat_result=compat_result,
|
||||
models=OPENAI_MODELS,
|
||||
|
|
|
|||
|
|
@ -15,9 +15,6 @@ Bash is restricted to the exact command `echo pong` plus
|
|||
`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the
|
||||
security rationale.
|
||||
|
||||
GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see
|
||||
`claude_code._gpt_cells`).
|
||||
|
||||
The (feature, provider) for this cell is inferred from the file path by
|
||||
`tests/e2e/claude_code/conftest.py`:
|
||||
|
||||
|
|
@ -28,21 +25,17 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
import pytest
|
||||
|
||||
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
|
||||
from claude_code._env import require_proxy
|
||||
from claude_code.cli_driver import (
|
||||
ClaudeCLIError,
|
||||
failure_diagnostic,
|
||||
run_claude_models_parallel,
|
||||
)
|
||||
|
||||
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
|
||||
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
|
||||
|
||||
AZURE_OPENAI_MODELS = [
|
||||
"gpt-5-6-sol-azure-openai",
|
||||
"gpt-5-6-terra-azure-openai",
|
||||
|
|
@ -77,28 +70,13 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
|
|||
def test_tool_use_azure_openai(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
tool call was emitted on the wire by each GPT-5.6 tier."""
|
||||
skip_unless_gpt_cells_enabled()
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
if not base_url or not api_key:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": (
|
||||
f"missing required env: set {PROXY_BASE_URL_ENV} and "
|
||||
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
|
||||
),
|
||||
}
|
||||
)
|
||||
pytest.fail(
|
||||
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
|
||||
)
|
||||
proxy = require_proxy(compat_result)
|
||||
|
||||
outcomes = run_claude_models_parallel(
|
||||
models=AZURE_OPENAI_MODELS,
|
||||
prompt=TOOL_USE_PROMPT,
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
base_url=proxy.base_url,
|
||||
api_key=proxy.api_key,
|
||||
extra_args=TOOL_USE_ARGS,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ Bash is restricted to the exact command `echo pong` plus
|
|||
`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the
|
||||
security rationale.
|
||||
|
||||
GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see
|
||||
Mantle cells are opt-in via COMPAT_MANTLE_CELLS=1 (see
|
||||
`claude_code._gpt_cells`).
|
||||
|
||||
The (feature, provider) for this cell is inferred from the file path by
|
||||
|
|
@ -29,21 +29,18 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
import pytest
|
||||
|
||||
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
|
||||
from claude_code._env import require_proxy
|
||||
from claude_code._gpt_cells import skip_unless_mantle_cells_enabled
|
||||
from claude_code.cli_driver import (
|
||||
ClaudeCLIError,
|
||||
failure_diagnostic,
|
||||
run_claude_models_parallel,
|
||||
)
|
||||
|
||||
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
|
||||
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
|
||||
|
||||
BEDROCK_MANTLE_MODELS = [
|
||||
"gpt-5-6-sol-bedrock-mantle",
|
||||
"gpt-5-6-terra-bedrock-mantle",
|
||||
|
|
@ -78,28 +75,14 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
|
|||
def test_tool_use_bedrock_mantle(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
tool call was emitted on the wire by each GPT-5.6 tier."""
|
||||
skip_unless_gpt_cells_enabled()
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
if not base_url or not api_key:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": (
|
||||
f"missing required env: set {PROXY_BASE_URL_ENV} and "
|
||||
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
|
||||
),
|
||||
}
|
||||
)
|
||||
pytest.fail(
|
||||
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
|
||||
)
|
||||
skip_unless_mantle_cells_enabled()
|
||||
proxy = require_proxy(compat_result)
|
||||
|
||||
outcomes = run_claude_models_parallel(
|
||||
models=BEDROCK_MANTLE_MODELS,
|
||||
prompt=TOOL_USE_PROMPT,
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
base_url=proxy.base_url,
|
||||
api_key=proxy.api_key,
|
||||
extra_args=TOOL_USE_ARGS,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -14,9 +14,6 @@ Bash is restricted to the exact command `echo pong` plus
|
|||
`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the
|
||||
security rationale.
|
||||
|
||||
GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see
|
||||
`claude_code._gpt_cells`).
|
||||
|
||||
The (feature, provider) for this cell is inferred from the file path by
|
||||
`tests/e2e/claude_code/conftest.py`:
|
||||
|
||||
|
|
@ -27,21 +24,17 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
import pytest
|
||||
|
||||
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
|
||||
from claude_code._env import require_proxy
|
||||
from claude_code.cli_driver import (
|
||||
ClaudeCLIError,
|
||||
failure_diagnostic,
|
||||
run_claude_models_parallel,
|
||||
)
|
||||
|
||||
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
|
||||
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
|
||||
|
||||
OPENAI_MODELS = [
|
||||
"gpt-5-6-sol-openai",
|
||||
"gpt-5-6-terra-openai",
|
||||
|
|
@ -76,28 +69,13 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
|
|||
def test_tool_use_openai(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
tool call was emitted on the wire by each GPT-5.6 tier."""
|
||||
skip_unless_gpt_cells_enabled()
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
if not base_url or not api_key:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": (
|
||||
f"missing required env: set {PROXY_BASE_URL_ENV} and "
|
||||
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
|
||||
),
|
||||
}
|
||||
)
|
||||
pytest.fail(
|
||||
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
|
||||
)
|
||||
proxy = require_proxy(compat_result)
|
||||
|
||||
outcomes = run_claude_models_parallel(
|
||||
models=OPENAI_MODELS,
|
||||
prompt=TOOL_USE_PROMPT,
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
base_url=proxy.base_url,
|
||||
api_key=proxy.api_key,
|
||||
extra_args=TOOL_USE_ARGS,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -17,9 +17,6 @@ Bash is restricted to the exact command `echo pong` plus
|
|||
`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the
|
||||
security rationale.
|
||||
|
||||
GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see
|
||||
`claude_code._gpt_cells`).
|
||||
|
||||
The (feature, provider) for this cell is inferred from the file path by
|
||||
`tests/e2e/claude_code/conftest.py`:
|
||||
|
||||
|
|
@ -30,21 +27,17 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
import pytest
|
||||
|
||||
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
|
||||
from claude_code._env import require_proxy
|
||||
from claude_code.cli_driver import (
|
||||
ClaudeCLIError,
|
||||
failure_diagnostic,
|
||||
run_claude_models_parallel,
|
||||
)
|
||||
|
||||
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
|
||||
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
|
||||
|
||||
AZURE_OPENAI_MODELS = [
|
||||
"gpt-5-6-sol-azure-openai",
|
||||
"gpt-5-6-terra-azure-openai",
|
||||
|
|
@ -96,28 +89,13 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
|
|||
|
||||
|
||||
def test_tool_use_streaming_azure_openai(compat_result):
|
||||
skip_unless_gpt_cells_enabled()
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
if not base_url or not api_key:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": (
|
||||
f"missing required env: set {PROXY_BASE_URL_ENV} and "
|
||||
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
|
||||
),
|
||||
}
|
||||
)
|
||||
pytest.fail(
|
||||
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
|
||||
)
|
||||
proxy = require_proxy(compat_result)
|
||||
|
||||
outcomes = run_claude_models_parallel(
|
||||
models=AZURE_OPENAI_MODELS,
|
||||
prompt=TOOL_USE_PROMPT,
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
base_url=proxy.base_url,
|
||||
api_key=proxy.api_key,
|
||||
extra_args=TOOL_USE_ARGS,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ Bash is restricted to the exact command `echo pong` plus
|
|||
`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the
|
||||
security rationale.
|
||||
|
||||
GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see
|
||||
Mantle cells are opt-in via COMPAT_MANTLE_CELLS=1 (see
|
||||
`claude_code._gpt_cells`).
|
||||
|
||||
The (feature, provider) for this cell is inferred from the file path by
|
||||
|
|
@ -30,21 +30,18 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
import pytest
|
||||
|
||||
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
|
||||
from claude_code._env import require_proxy
|
||||
from claude_code._gpt_cells import skip_unless_mantle_cells_enabled
|
||||
from claude_code.cli_driver import (
|
||||
ClaudeCLIError,
|
||||
failure_diagnostic,
|
||||
run_claude_models_parallel,
|
||||
)
|
||||
|
||||
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
|
||||
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
|
||||
|
||||
BEDROCK_MANTLE_MODELS = [
|
||||
"gpt-5-6-sol-bedrock-mantle",
|
||||
"gpt-5-6-terra-bedrock-mantle",
|
||||
|
|
@ -96,28 +93,14 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
|
|||
|
||||
|
||||
def test_tool_use_streaming_bedrock_mantle(compat_result):
|
||||
skip_unless_gpt_cells_enabled()
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
if not base_url or not api_key:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": (
|
||||
f"missing required env: set {PROXY_BASE_URL_ENV} and "
|
||||
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
|
||||
),
|
||||
}
|
||||
)
|
||||
pytest.fail(
|
||||
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
|
||||
)
|
||||
skip_unless_mantle_cells_enabled()
|
||||
proxy = require_proxy(compat_result)
|
||||
|
||||
outcomes = run_claude_models_parallel(
|
||||
models=BEDROCK_MANTLE_MODELS,
|
||||
prompt=TOOL_USE_PROMPT,
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
base_url=proxy.base_url,
|
||||
api_key=proxy.api_key,
|
||||
extra_args=TOOL_USE_ARGS,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -15,9 +15,6 @@ Bash is restricted to the exact command `echo pong` plus
|
|||
`--permission-mode dontAsk`; see `tool_use/test_anthropic.py` for the
|
||||
security rationale.
|
||||
|
||||
GPT cells are opt-in via COMPAT_GPT_CELLS=1 (see
|
||||
`claude_code._gpt_cells`).
|
||||
|
||||
The (feature, provider) for this cell is inferred from the file path by
|
||||
`tests/e2e/claude_code/conftest.py`:
|
||||
|
||||
|
|
@ -28,21 +25,17 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
import pytest
|
||||
|
||||
from claude_code._gpt_cells import skip_unless_gpt_cells_enabled
|
||||
from claude_code._env import require_proxy
|
||||
from claude_code.cli_driver import (
|
||||
ClaudeCLIError,
|
||||
failure_diagnostic,
|
||||
run_claude_models_parallel,
|
||||
)
|
||||
|
||||
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
|
||||
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
|
||||
|
||||
OPENAI_MODELS = [
|
||||
"gpt-5-6-sol-openai",
|
||||
"gpt-5-6-terra-openai",
|
||||
|
|
@ -94,28 +87,13 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
|
|||
|
||||
|
||||
def test_tool_use_streaming_openai(compat_result):
|
||||
skip_unless_gpt_cells_enabled()
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
if not base_url or not api_key:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": (
|
||||
f"missing required env: set {PROXY_BASE_URL_ENV} and "
|
||||
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
|
||||
),
|
||||
}
|
||||
)
|
||||
pytest.fail(
|
||||
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
|
||||
)
|
||||
proxy = require_proxy(compat_result)
|
||||
|
||||
outcomes = run_claude_models_parallel(
|
||||
models=OPENAI_MODELS,
|
||||
prompt=TOOL_USE_PROMPT,
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
base_url=proxy.base_url,
|
||||
api_key=proxy.api_key,
|
||||
extra_args=TOOL_USE_ARGS,
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue