scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 114

test(e2e): tag claude_code cells with Subject metadata and record CLI driver steps (#44961)

Browse source 
* test(e2e): add enum values, auto-discovering label gates and secret hiding for e2e metadata

* test(e2e): tag claude_code tests with Subject metadata and record CLI driver steps

* docs(e2e): name every markerless harness test file that carries no Subject

* test(e2e): keep the step discovery comprehensions to one for clause

* test(e2e): decorate run_claude directly so the label gate discovers its step
This commit is contained in:
ryan-crabbe-berri 2026-10-07 10:31:54 -07:00 • committed by GitHub
parent ade17902a0
commit 0ae55bdf7a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
100 changed files with 1030 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
tests/e2e/claude_code/_basic_messaging.py
View file

@ -31,6 +31,8 @@ from typing import Any, Callable, Mapping, Sequence
import pytest
from e2e_metadata import step
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -74,6 +76,7 @@ def _count_stream_event_deltas(events: Sequence[Mapping[str, Any]]) -> int:
return count
@step("Run Claude Code headless against {models} through the proxy and check every model replies")
def run_basic_messaging_cell(
*,
compat_result,

6
tests/e2e/claude_code/_passthrough.py
View file

@ -58,6 +58,8 @@ from typing import Any, Callable, Dict, Mapping, Optional, Sequence
import pytest
from e2e_metadata import step
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -118,6 +120,10 @@ def foundry_extra_env(proxy_base_url: str) -> Dict[str, str]:
}
@step(
"Run Claude Code headless against {models} through the proxy's native provider passthrough route"
" and check every model replies"
)
def run_passthrough_cell(
*,
compat_result,

10
tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py
View file

@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider).
from __future__ import annotations
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._basic_messaging import run_basic_messaging_cell
# Per the PRD: each cell is exercised against three Claude tiers via the
@ -34,6 +35,15 @@ ANTHROPIC_MODELS = [
@pytest.mark.covers("llm.messages.anthropic.basic.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.ANTHROPIC,),
models=tuple(ANTHROPIC_MODELS),
mode=Mode.NONSTREAM,
)
)
def test_basic_messaging_non_streaming_anthropic(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a reply.

10
tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py
View file

@ -26,6 +26,7 @@ the matrix builder still sees three rows for this (feature, provider).
from __future__ import annotations
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._basic_messaging import run_basic_messaging_cell
# Per-model aliases registered in the LiteLLM proxy's routing config to
@ -40,6 +41,15 @@ AZURE_MODELS = [
@pytest.mark.covers("llm.messages.azure_foundry.basic.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.AZURE_AI,),
models=tuple(AZURE_MODELS),
mode=Mode.NONSTREAM,
)
)
def test_basic_messaging_non_streaming_azure(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a reply.

10
tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py
View file

@ -25,6 +25,7 @@ green if all three pass.
from __future__ import annotations
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._basic_messaging import run_basic_messaging_cell
AZURE_OPENAI_MODELS = [
@ -34,6 +35,15 @@ AZURE_OPENAI_MODELS = [
]
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.AZURE,),
models=tuple(AZURE_OPENAI_MODELS),
mode=Mode.NONSTREAM,
)
)
def test_basic_messaging_non_streaming_azure_openai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty reply from each GPT-5.6 tier."""

10
tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py
View file

@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider).
from __future__ import annotations
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._basic_messaging import run_basic_messaging_cell
# Per-model aliases registered in the LiteLLM proxy's routing config to
@ -35,6 +36,15 @@ BEDROCK_CONVERSE_MODELS = [
@pytest.mark.covers("llm.messages.bedrock_converse.basic.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_CONVERSE_MODELS),
mode=Mode.NONSTREAM,
)
)
def test_basic_messaging_non_streaming_bedrock_converse(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a reply."""
run_basic_messaging_cell(

10
tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py
View file

@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider).
from __future__ import annotations
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._basic_messaging import run_basic_messaging_cell
# Per-model aliases registered in the LiteLLM proxy's routing config to
@ -35,6 +36,15 @@ BEDROCK_INVOKE_MODELS = [
@pytest.mark.covers("llm.messages.bedrock_invoke.basic.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_INVOKE_MODELS),
mode=Mode.NONSTREAM,
)
)
def test_basic_messaging_non_streaming_bedrock_invoke(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a reply."""
run_basic_messaging_cell(

10
tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py
View file

@ -25,6 +25,7 @@ COMPAT_MANTLE_CELLS=1 (see `claude_code._gpt_cells`).
from __future__ import annotations
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code._gpt_cells import skip_unless_mantle_cells_enabled
@ -35,6 +36,15 @@ BEDROCK_MANTLE_MODELS = [
]
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK_MANTLE,),
models=tuple(BEDROCK_MANTLE_MODELS),
mode=Mode.NONSTREAM,
)
)
def test_basic_messaging_non_streaming_bedrock_mantle(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty reply from each GPT-5.6 tier."""

10
tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py
View file

@ -22,6 +22,7 @@ green if all three pass.
from __future__ import annotations
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled
@ -32,6 +33,15 @@ OPENAI_MODELS = [
]
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.OPENAI,),
models=tuple(OPENAI_MODELS),
mode=Mode.NONSTREAM,
)
)
def test_basic_messaging_non_streaming_openai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty reply from each GPT-5.6 tier."""

10
tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py
View file

@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider).
from __future__ import annotations
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._basic_messaging import run_basic_messaging_cell
# Per-model aliases registered in the LiteLLM proxy's routing config to
@ -35,6 +36,15 @@ VERTEX_AI_MODELS = [
@pytest.mark.covers("llm.messages.vertex.basic.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.VERTEX_AI,),
models=tuple(VERTEX_AI_MODELS),
mode=Mode.NONSTREAM,
)
)
def test_basic_messaging_non_streaming_vertex_ai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a reply."""
run_basic_messaging_cell(

2
tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py
View file

@ -16,9 +16,11 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
from e2e_metadata import Domain, Subject, meta
from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON
@meta(Subject(domain=Domain.LLM_TRANSLATION))
def test_basic_messaging_non_streaming_vertex_ai_gpt(compat_result):
"""Record the static not_applicable outcome for this cell."""
compat_result.set(

10
tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py
View file

@ -26,6 +26,7 @@ sees three rows for this (feature, provider).
from __future__ import annotations
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._basic_messaging import run_basic_messaging_cell
ANTHROPIC_MODELS = [
@ -36,6 +37,15 @@ ANTHROPIC_MODELS = [
@pytest.mark.covers("llm.messages.anthropic.basic.stream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.ANTHROPIC,),
models=tuple(ANTHROPIC_MODELS),
mode=Mode.STREAM,
)
)
def test_basic_messaging_streaming_anthropic(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty streamed reply (one row per Claude tier).

10
tests/e2e/claude_code/basic_messaging_streaming/test_azure.py
View file

@ -20,6 +20,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._basic_messaging import run_basic_messaging_cell
AZURE_MODELS = [
@ -30,6 +31,15 @@ AZURE_MODELS = [
@pytest.mark.covers("llm.messages.azure_foundry.basic.stream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.AZURE_AI,),
models=tuple(AZURE_MODELS),
mode=Mode.STREAM,
)
)
def test_basic_messaging_streaming_azure(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty streamed reply (one row per Claude tier).

10
tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py
View file

@ -24,6 +24,7 @@ green if all three pass.
from __future__ import annotations
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._basic_messaging import run_basic_messaging_cell
AZURE_OPENAI_MODELS = [
@ -33,6 +34,15 @@ AZURE_OPENAI_MODELS = [
]
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.AZURE,),
models=tuple(AZURE_OPENAI_MODELS),
mode=Mode.STREAM,
)
)
def test_basic_messaging_streaming_azure_openai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty streamed reply from each GPT-5.6 tier."""

10
tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py
View file

@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._basic_messaging import run_basic_messaging_cell
BEDROCK_CONVERSE_MODELS = [
@ -26,6 +27,15 @@ BEDROCK_CONVERSE_MODELS = [
@pytest.mark.covers("llm.messages.bedrock_converse.basic.stream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_CONVERSE_MODELS),
mode=Mode.STREAM,
)
)
def test_basic_messaging_streaming_bedrock_converse(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty streamed reply (one row per Claude tier).

10
tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py
View file

@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._basic_messaging import run_basic_messaging_cell
BEDROCK_INVOKE_MODELS = [
@ -26,6 +27,15 @@ BEDROCK_INVOKE_MODELS = [
@pytest.mark.covers("llm.messages.bedrock_invoke.basic.stream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_INVOKE_MODELS),
mode=Mode.STREAM,
)
)
def test_basic_messaging_streaming_bedrock_invoke(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty streamed reply (one row per Claude tier).

10
tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py
View file

@ -25,6 +25,7 @@ COMPAT_MANTLE_CELLS=1 (see `claude_code._gpt_cells`).
from __future__ import annotations
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code._gpt_cells import skip_unless_mantle_cells_enabled
@ -35,6 +36,15 @@ BEDROCK_MANTLE_MODELS = [
]
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK_MANTLE,),
models=tuple(BEDROCK_MANTLE_MODELS),
mode=Mode.STREAM,
)
)
def test_basic_messaging_streaming_bedrock_mantle(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty streamed reply from each GPT-5.6 tier."""

10
tests/e2e/claude_code/basic_messaging_streaming/test_openai.py
View file

@ -24,6 +24,7 @@ green if all three pass.
from __future__ import annotations
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled
@ -34,6 +35,15 @@ OPENAI_MODELS = [
]
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.OPENAI,),
models=tuple(OPENAI_MODELS),
mode=Mode.STREAM,
)
)
def test_basic_messaging_streaming_openai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty streamed reply from each GPT-5.6 tier."""

10
tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py
View file

@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._basic_messaging import run_basic_messaging_cell
VERTEX_AI_MODELS = [
@ -26,6 +27,15 @@ VERTEX_AI_MODELS = [
@pytest.mark.covers("llm.messages.vertex.basic.stream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.VERTEX_AI,),
models=tuple(VERTEX_AI_MODELS),
mode=Mode.STREAM,
)
)
def test_basic_messaging_streaming_vertex_ai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty streamed reply (one row per Claude tier).

2
tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai_gpt.py
View file

@ -16,9 +16,11 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
from e2e_metadata import Domain, Subject, meta
from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON
@meta(Subject(domain=Domain.LLM_TRANSLATION))
def test_basic_messaging_streaming_vertex_ai_gpt(compat_result):
"""Record the static not_applicable outcome for this cell."""
compat_result.set(

4
tests/e2e/claude_code/cli_driver.py
View file

@ -25,6 +25,8 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, Union
from e2e_metadata import step
from claude_code.rate_limiter import (
RateLimiter,
get_default_limiter,
@ -211,6 +213,7 @@ class DriverResult:
duration_ms: Optional[int] = None
@step("Run Claude Code headless against {model} through the proxy")
def run_claude(
*,
prompt: Optional[str],
@ -395,6 +398,7 @@ def _matches_failure_shape(outcome: ModelResult, pattern: "re.Pattern[str]") ->
return bool(pattern.search(failure_diagnostic(outcome)))
@step("Run Claude Code headless against {models} in parallel through the proxy")
def run_claude_models_parallel(
*,
models: Sequence[str],

10
tests/e2e/claude_code/count_tokens/test_anthropic.py
View file

@ -39,6 +39,7 @@ from __future__ import annotations
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy_client
from claude_code.http_probe import (
assert_count_tokens_shape,
@ -54,6 +55,15 @@ ANTHROPIC_MODELS = [
@pytest.mark.covers("llm.messages.anthropic.count_tokens.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.COUNT_TOKENS,
providers=(Provider.ANTHROPIC,),
models=tuple(ANTHROPIC_MODELS),
mode=Mode.NONSTREAM,
)
)
def test_count_tokens_anthropic(compat_result):
"""Probe `/v1/messages/count_tokens` for each Anthropic tier and
assert the response shape."""

10
tests/e2e/claude_code/count_tokens/test_azure.py
View file

@ -39,6 +39,7 @@ from __future__ import annotations
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy_client
from claude_code.http_probe import (
assert_count_tokens_shape,
@ -54,6 +55,15 @@ AZURE_MODELS = [
@pytest.mark.covers("llm.messages.azure_foundry.count_tokens.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.COUNT_TOKENS,
providers=(Provider.AZURE_AI,),
models=tuple(AZURE_MODELS),
mode=Mode.NONSTREAM,
)
)
def test_count_tokens_azure(compat_result):
"""Probe `/v1/messages/count_tokens` for each Azure (Microsoft Foundry) tier and
assert the response shape."""

10
tests/e2e/claude_code/count_tokens/test_bedrock_converse.py
View file

@ -39,6 +39,7 @@ from __future__ import annotations
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy_client
from claude_code.http_probe import (
assert_count_tokens_shape,
@ -54,6 +55,15 @@ BEDROCK_CONVERSE_MODELS = [
@pytest.mark.covers("llm.messages.bedrock_converse.count_tokens.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.COUNT_TOKENS,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_CONVERSE_MODELS),
mode=Mode.NONSTREAM,
)
)
def test_count_tokens_bedrock_converse(compat_result):
"""Probe `/v1/messages/count_tokens` for each Bedrock (Converse) tier and
assert the response shape."""

10
tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py
View file

@ -39,6 +39,7 @@ from __future__ import annotations
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy_client
from claude_code.http_probe import (
assert_count_tokens_shape,
@ -54,6 +55,15 @@ BEDROCK_INVOKE_MODELS = [
@pytest.mark.covers("llm.messages.bedrock_invoke.count_tokens.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.COUNT_TOKENS,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_INVOKE_MODELS),
mode=Mode.NONSTREAM,
)
)
def test_count_tokens_bedrock_invoke(compat_result):
"""Probe `/v1/messages/count_tokens` for each Bedrock (Invoke) tier and
assert the response shape."""

10
tests/e2e/claude_code/count_tokens/test_vertex_ai.py
View file

@ -39,6 +39,7 @@ from __future__ import annotations
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy_client
from claude_code.http_probe import (
assert_count_tokens_shape,
@ -55,6 +56,15 @@ VERTEX_AI_MODELS = [
@pytest.mark.skip(reason="stage red: Vertex returns not supported for token counting for Claude aliases")
@pytest.mark.covers("llm.messages.vertex.count_tokens.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.COUNT_TOKENS,
providers=(Provider.VERTEX_AI,),
models=tuple(VERTEX_AI_MODELS),
mode=Mode.NONSTREAM,
)
)
def test_count_tokens_vertex_ai(compat_result):
"""Probe `/v1/messages/count_tokens` for each Vertex AI tier and
assert the response shape."""

7
tests/e2e/claude_code/http_probe.py
View file

@ -42,6 +42,7 @@ from e2e_http import (
UnknownApiError,
ValidationError,
)
from e2e_metadata import step
from models import (
AnthropicAssistantTurn,
AnthropicCustomTool,
@ -109,6 +110,7 @@ def _acquire(model: str, rate_limiter: RateLimiter | None) -> None:
limiter.acquire(infer_provider(model))
@step('Count tokens with /v1/messages/count_tokens for {model} on the message "{message}"')
def probe_count_tokens(
*,
client: ProxyClient,
@ -132,6 +134,7 @@ def probe_count_tokens(
)
@step("Send a /v1/messages request to {model} with the tool_search tool declared")
def probe_tool_search(
*,
client: ProxyClient,
@ -228,6 +231,10 @@ def _replay_history(answer: AnthropicMessagesResponse) -> tuple[AnthropicMessage
)
@step(
"Send a /v1/messages request to {model} with the tool_search tool declared,"
" then send its answer back as history in a second request"
)
def probe_tool_search_multiturn(
*,
client: ProxyClient,

10
tests/e2e/claude_code/long_context_1m/test_anthropic.py
View file

@ -58,6 +58,7 @@ from typing import Sequence
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -155,6 +156,15 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str:
@pytest.mark.skip(reason="stage red: 1M long_context not green on stage Anthropic path yet (200k sonnet / model alias)")
@pytest.mark.covers("llm.messages.anthropic.long_context_1m.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.ANTHROPIC,),
models=tuple(ANTHROPIC_MODELS),
mode=Mode.NONSTREAM,
)
)
def test_long_context_1m_anthropic(compat_result):
"""Drive the `claude` CLI with a ~210k-token prompt and the
`context-1m-2025-08-07` beta header; assert no 400 / 413 and a

10
tests/e2e/claude_code/long_context_1m/test_azure.py
View file

@ -58,6 +58,7 @@ from typing import Sequence
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -155,6 +156,15 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str:
@pytest.mark.skip(reason="stage red: 1M long_context not green on stage Azure Foundry deployments yet")
@pytest.mark.covers("llm.messages.azure_foundry.long_context_1m.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.AZURE_AI,),
models=tuple(AZURE_MODELS),
mode=Mode.NONSTREAM,
)
)
def test_long_context_1m_azure(compat_result):
"""Drive the `claude` CLI (Azure (Microsoft Foundry)) with a ~210k-token prompt and the
`context-1m-2025-08-07` beta header; assert no 400 / 413 and a

10
tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py
View file

@ -58,6 +58,7 @@ from typing import Sequence
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -155,6 +156,15 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str:
@pytest.mark.skip(reason="stage red: 1M long_context not green on stage Bedrock Converse deployments yet")
@pytest.mark.covers("llm.messages.bedrock_converse.long_context_1m.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_CONVERSE_MODELS),
mode=Mode.NONSTREAM,
)
)
def test_long_context_1m_bedrock_converse(compat_result):
"""Drive the `claude` CLI (Bedrock (Converse)) with a ~210k-token prompt and the
`context-1m-2025-08-07` beta header; assert no 400 / 413 and a

10
tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py
View file

@ -58,6 +58,7 @@ from typing import Sequence
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -155,6 +156,15 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str:
@pytest.mark.skip(reason="stage red: 1M long_context not green on stage Bedrock Invoke deployments yet")
@pytest.mark.covers("llm.messages.bedrock_invoke.long_context_1m.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_INVOKE_MODELS),
mode=Mode.NONSTREAM,
)
)
def test_long_context_1m_bedrock_invoke(compat_result):
"""Drive the `claude` CLI (Bedrock (Invoke)) with a ~210k-token prompt and the
`context-1m-2025-08-07` beta header; assert no 400 / 413 and a

10
tests/e2e/claude_code/long_context_1m/test_vertex_ai.py
View file

@ -58,6 +58,7 @@ from typing import Sequence
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -155,6 +156,15 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str:
@pytest.mark.skip(reason="stage red: 1M long_context not green on stage Vertex deployments yet")
@pytest.mark.covers("llm.messages.vertex.long_context_1m.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.VERTEX_AI,),
models=tuple(VERTEX_AI_MODELS),
mode=Mode.NONSTREAM,
)
)
def test_long_context_1m_vertex_ai(compat_result):
"""Drive the `claude` CLI (Vertex AI) with a ~210k-token prompt and the
`context-1m-2025-08-07` beta header; assert no 400 / 413 and a

10
tests/e2e/claude_code/passthrough/test_anthropic.py
View file

@ -22,6 +22,7 @@ no per-provider transformation is involved.
from __future__ import annotations
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._passthrough import (
ANTHROPIC_PASSTHROUGH_BASE_PATH,
run_passthrough_cell,
@ -34,6 +35,15 @@ ANTHROPIC_MODELS = [
]
@meta(
Subject(
domain=Domain.PASSTHROUGH,
route=Route.PASSTHROUGH,
providers=(Provider.ANTHROPIC,),
models=tuple(ANTHROPIC_MODELS),
mode=Mode.STREAM,
)
)
def test_passthrough_anthropic(compat_result):
"""Drive the `claude` CLI through `{proxy}/anthropic` and assert a reply."""
run_passthrough_cell(

10
tests/e2e/claude_code/passthrough/test_azure.py
View file

@ -42,6 +42,7 @@ from __future__ import annotations
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._passthrough import foundry_extra_env, run_passthrough_cell
AZURE_MODELS = [
@ -52,6 +53,15 @@ AZURE_MODELS = [
@pytest.mark.skip(reason="stage red: /azure passthrough drops client headers (e.g. anthropic-version); product gap")
@meta(
Subject(
domain=Domain.PASSTHROUGH,
route=Route.PASSTHROUGH,
providers=(Provider.AZURE,),
models=tuple(AZURE_MODELS),
mode=Mode.STREAM,
)
)
def test_passthrough_azure(compat_result):
"""Drive the `claude` CLI through `{proxy}/azure` and assert a reply."""
run_passthrough_cell(

3
tests/e2e/claude_code/passthrough/test_bedrock_converse.py
View file

@ -18,7 +18,10 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
from e2e_metadata import Domain, Subject, meta
@meta(Subject(domain=Domain.PASSTHROUGH))
def test_passthrough_bedrock_converse(compat_result):
"""Report not_applicable: Claude Code has no Converse-wire mode."""
compat_result.set(

10
tests/e2e/claude_code/passthrough/test_bedrock_invoke.py
View file

@ -23,6 +23,7 @@ cell.
from __future__ import annotations
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._passthrough import bedrock_extra_env, run_passthrough_cell
BEDROCK_INVOKE_MODELS = [
@ -32,6 +33,15 @@ BEDROCK_INVOKE_MODELS = [
]
@meta(
Subject(
domain=Domain.PASSTHROUGH,
route=Route.PASSTHROUGH,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_INVOKE_MODELS),
mode=Mode.STREAM,
)
)
def test_passthrough_bedrock_invoke(compat_result):
"""Drive the `claude` CLI through `{proxy}/bedrock` and assert a reply."""
run_passthrough_cell(

10
tests/e2e/claude_code/passthrough/test_vertex_ai.py
View file

@ -26,6 +26,7 @@ Google and every tier fails with a 401.
from __future__ import annotations
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from claude_code._passthrough import run_passthrough_cell, vertex_extra_env
VERTEX_MODELS = [
@ -35,6 +36,15 @@ VERTEX_MODELS = [
]
@meta(
Subject(
domain=Domain.PASSTHROUGH,
route=Route.PASSTHROUGH,
providers=(Provider.VERTEX_AI,),
models=tuple(VERTEX_MODELS),
mode=Mode.STREAM,
)
)
def test_passthrough_vertex_ai(compat_result):
"""Drive the `claude` CLI through `{proxy}/vertex_ai` and assert a reply."""
run_passthrough_cell(

11
tests/e2e/claude_code/pdf_input/test_anthropic.py
View file

@ -24,6 +24,7 @@ from __future__ import annotations
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -106,6 +107,16 @@ def _build_minimal_pdf(marker: str) -> bytes:
@pytest.mark.covers("llm.messages.anthropic.pdf_input.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.ANTHROPIC,),
models=tuple(ANTHROPIC_MODELS),
capabilities=(Capability.PDF_INPUT,),
mode=Mode.NONSTREAM,
)
)
def test_pdf_input_anthropic(compat_result, tmp_path):
"""Drive the `claude` CLI against the LiteLLM proxy with a PDF
attached via the Read tool and assert the reply references it."""

11
tests/e2e/claude_code/pdf_input/test_azure.py
View file

@ -17,6 +17,7 @@ from __future__ import annotations
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -83,6 +84,16 @@ def _build_minimal_pdf(marker: str) -> bytes:
@pytest.mark.covers("llm.messages.azure_foundry.pdf_input.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.AZURE_AI,),
models=tuple(AZURE_MODELS),
capabilities=(Capability.PDF_INPUT,),
mode=Mode.NONSTREAM,
)
)
def test_pdf_input_azure(compat_result, tmp_path):
base_url, api_key = require_proxy(compat_result)

11
tests/e2e/claude_code/pdf_input/test_bedrock_converse.py
View file

@ -23,6 +23,7 @@ from __future__ import annotations
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -89,6 +90,16 @@ def _build_minimal_pdf(marker: str) -> bytes:
@pytest.mark.covers("llm.messages.bedrock_converse.pdf_input.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_CONVERSE_MODELS),
capabilities=(Capability.PDF_INPUT,),
mode=Mode.NONSTREAM,
)
)
def test_pdf_input_bedrock_converse(compat_result, tmp_path):
base_url, api_key = require_proxy(compat_result)

11
tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py
View file

@ -22,6 +22,7 @@ from __future__ import annotations
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -88,6 +89,16 @@ def _build_minimal_pdf(marker: str) -> bytes:
@pytest.mark.covers("llm.messages.bedrock_invoke.pdf_input.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_INVOKE_MODELS),
capabilities=(Capability.PDF_INPUT,),
mode=Mode.NONSTREAM,
)
)
def test_pdf_input_bedrock_invoke(compat_result, tmp_path):
base_url, api_key = require_proxy(compat_result)

11
tests/e2e/claude_code/pdf_input/test_vertex_ai.py
View file

@ -17,6 +17,7 @@ from __future__ import annotations
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -83,6 +84,16 @@ def _build_minimal_pdf(marker: str) -> bytes:
@pytest.mark.covers("llm.messages.vertex.pdf_input.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.VERTEX_AI,),
models=tuple(VERTEX_AI_MODELS),
capabilities=(Capability.PDF_INPUT,),
mode=Mode.NONSTREAM,
)
)
def test_pdf_input_vertex_ai(compat_result, tmp_path):
base_url, api_key = require_proxy(compat_result)

11
tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py
View file

@ -28,6 +28,7 @@ from typing import Any, Mapping, Optional
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -64,6 +65,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
@pytest.mark.covers("llm.messages.anthropic.prompt_cache_1h.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.ANTHROPIC,),
models=tuple(ANTHROPIC_MODELS),
capabilities=(Capability.PROMPT_CACHING,),
mode=Mode.NONSTREAM,
)
)
def test_prompt_caching_1h_anthropic(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy with the 1h
TTL opt-in env var set, and assert the upstream usage block

11
tests/e2e/claude_code/prompt_caching_1h/test_azure.py
View file

@ -19,6 +19,7 @@ from typing import Any, Mapping, Optional
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -48,6 +49,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
@pytest.mark.covers("llm.messages.azure_foundry.prompt_cache_1h.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.AZURE_AI,),
models=tuple(AZURE_MODELS),
capabilities=(Capability.PROMPT_CACHING,),
mode=Mode.NONSTREAM,
)
)
def test_prompt_caching_1h_azure(compat_result):
base_url, api_key = require_proxy(compat_result)

11
tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py
View file

@ -24,6 +24,7 @@ from typing import Any, Mapping, Optional
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -56,6 +57,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
@pytest.mark.covers("llm.messages.bedrock_converse.prompt_cache_1h.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_CONVERSE_MODELS),
capabilities=(Capability.PROMPT_CACHING,),
mode=Mode.NONSTREAM,
)
)
def test_prompt_caching_1h_bedrock_converse(compat_result):
base_url, api_key = require_proxy(compat_result)

11
tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py
View file

@ -26,6 +26,7 @@ from typing import Any, Mapping, Optional
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -60,6 +61,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
@pytest.mark.covers("llm.messages.bedrock_invoke.prompt_cache_1h.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_INVOKE_MODELS),
capabilities=(Capability.PROMPT_CACHING,),
mode=Mode.NONSTREAM,
)
)
def test_prompt_caching_1h_bedrock_invoke(compat_result):
base_url, api_key = require_proxy(compat_result)

11
tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py
View file

@ -19,6 +19,7 @@ from typing import Any, Mapping, Optional
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -48,6 +49,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
@pytest.mark.covers("llm.messages.vertex.prompt_cache_1h.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.VERTEX_AI,),
models=tuple(VERTEX_AI_MODELS),
capabilities=(Capability.PROMPT_CACHING,),
mode=Mode.NONSTREAM,
)
)
def test_prompt_caching_1h_vertex_ai(compat_result):
base_url, api_key = require_proxy(compat_result)

11
tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py
View file

@ -26,6 +26,7 @@ from typing import Any, Mapping, Optional
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -55,6 +56,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
@pytest.mark.covers("llm.messages.anthropic.prompt_cache_5m.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.ANTHROPIC,),
models=tuple(ANTHROPIC_MODELS),
capabilities=(Capability.PROMPT_CACHING,),
mode=Mode.NONSTREAM,
)
)
def test_prompt_caching_5m_anthropic(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
upstream usage block surfaces a non-zero cache token count."""

11
tests/e2e/claude_code/prompt_caching_5m/test_azure.py
View file

@ -26,6 +26,7 @@ from typing import Any, Mapping, Optional
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -53,6 +54,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
@pytest.mark.covers("llm.messages.azure_foundry.prompt_cache_5m.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.AZURE_AI,),
models=tuple(AZURE_MODELS),
capabilities=(Capability.PROMPT_CACHING,),
mode=Mode.NONSTREAM,
)
)
def test_prompt_caching_5m_azure(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
upstream usage block surfaces a non-zero cache token count."""

11
tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py
View file

@ -19,6 +19,7 @@ from typing import Any, Mapping, Optional
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -46,6 +47,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
@pytest.mark.covers("llm.messages.bedrock_converse.prompt_cache_5m.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_CONVERSE_MODELS),
capabilities=(Capability.PROMPT_CACHING,),
mode=Mode.NONSTREAM,
)
)
def test_prompt_caching_5m_bedrock_converse(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
upstream usage block surfaces a non-zero cache token count."""

11
tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py
View file

@ -19,6 +19,7 @@ from typing import Any, Mapping, Optional
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -46,6 +47,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
@pytest.mark.covers("llm.messages.bedrock_invoke.prompt_cache_5m.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_INVOKE_MODELS),
capabilities=(Capability.PROMPT_CACHING,),
mode=Mode.NONSTREAM,
)
)
def test_prompt_caching_5m_bedrock_invoke(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
upstream usage block surfaces a non-zero cache token count."""

11
tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py
View file

@ -19,6 +19,7 @@ from typing import Any, Mapping, Optional
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -46,6 +47,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
@pytest.mark.covers("llm.messages.vertex.prompt_cache_5m.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.VERTEX_AI,),
models=tuple(VERTEX_AI_MODELS),
capabilities=(Capability.PROMPT_CACHING,),
mode=Mode.NONSTREAM,
)
)
def test_prompt_caching_5m_vertex_ai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
upstream usage block surfaces a non-zero cache token count."""

11
tests/e2e/claude_code/structured_outputs/test_anthropic.py
View file

@ -53,6 +53,7 @@ from typing import Any, Mapping, Optional, Sequence, Tuple
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -151,6 +152,16 @@ def _validate_against_schema(
@pytest.mark.covers("llm.messages.anthropic.structured_output.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.ANTHROPIC,),
models=tuple(ANTHROPIC_MODELS),
capabilities=(Capability.RESPONSE_SCHEMA,),
mode=Mode.NONSTREAM,
)
)
def test_structured_outputs_anthropic(compat_result):
"""Drive `claude --json-schema ...` against the LiteLLM proxy and
assert the trailing `result` event contains a schema-conforming

11
tests/e2e/claude_code/structured_outputs/test_azure.py
View file

@ -53,6 +53,7 @@ from typing import Any, Mapping, Optional, Sequence, Tuple
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -151,6 +152,16 @@ def _validate_against_schema(
@pytest.mark.covers("llm.messages.azure_foundry.structured_output.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.AZURE_AI,),
models=tuple(AZURE_MODELS),
capabilities=(Capability.RESPONSE_SCHEMA,),
mode=Mode.NONSTREAM,
)
)
def test_structured_outputs_azure(compat_result):
"""Drive `claude --json-schema ...` against the LiteLLM proxy and
assert the trailing `result` event contains a schema-conforming

12
tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py
View file

@ -53,6 +53,8 @@ from typing import Any, Mapping, Optional, Sequence, Tuple
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -151,6 +153,16 @@ def _validate_against_schema(
@pytest.mark.covers("llm.messages.bedrock_converse.structured_output.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_CONVERSE_MODELS),
capabilities=(Capability.RESPONSE_SCHEMA,),
mode=Mode.NONSTREAM,
)
)
def test_structured_outputs_bedrock_converse(compat_result):
"""Drive `claude --json-schema ...` against the LiteLLM proxy and
assert the trailing `result` event contains a schema-conforming

12
tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py
View file

@ -53,6 +53,8 @@ from typing import Any, Mapping, Optional, Sequence, Tuple
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -151,6 +153,16 @@ def _validate_against_schema(
@pytest.mark.covers("llm.messages.bedrock_invoke.structured_output.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_INVOKE_MODELS),
capabilities=(Capability.RESPONSE_SCHEMA,),
mode=Mode.NONSTREAM,
)
)
def test_structured_outputs_bedrock_invoke(compat_result):
"""Drive `claude --json-schema ...` against the LiteLLM proxy and
assert the trailing `result` event contains a schema-conforming

12
tests/e2e/claude_code/structured_outputs/test_vertex_ai.py
View file

@ -53,6 +53,8 @@ from typing import Any, Mapping, Optional, Sequence, Tuple
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -151,6 +153,16 @@ def _validate_against_schema(
@pytest.mark.covers("llm.messages.vertex.structured_output.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.VERTEX_AI,),
models=tuple(VERTEX_AI_MODELS),
capabilities=(Capability.RESPONSE_SCHEMA,),
mode=Mode.NONSTREAM,
)
)
def test_structured_outputs_vertex_ai(compat_result):
"""Drive `claude --json-schema ...` against the LiteLLM proxy and
assert the trailing `result` event contains a schema-conforming

12
tests/e2e/claude_code/thinking/test_anthropic.py
View file

@ -24,6 +24,8 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -75,6 +77,16 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
@pytest.mark.covers("llm.messages.anthropic.thinking.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.ANTHROPIC,),
models=tuple(ANTHROPIC_MODELS),
capabilities=(Capability.REASONING,),
mode=Mode.NONSTREAM,
)
)
def test_thinking_anthropic(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
enabled and assert a `thinking` content block was emitted."""

12
tests/e2e/claude_code/thinking/test_azure.py
View file

@ -27,6 +27,8 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -63,6 +65,16 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
@pytest.mark.covers("llm.messages.azure_foundry.thinking.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.AZURE_AI,),
models=tuple(AZURE_MODELS),
capabilities=(Capability.REASONING,),
mode=Mode.NONSTREAM,
)
)
def test_thinking_azure(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
enabled and assert a `thinking` content block was emitted."""

12
tests/e2e/claude_code/thinking/test_bedrock_converse.py
View file

@ -19,6 +19,8 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -55,6 +57,16 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
@pytest.mark.covers("llm.messages.bedrock_converse.thinking.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_CONVERSE_MODELS),
capabilities=(Capability.REASONING,),
mode=Mode.NONSTREAM,
)
)
def test_thinking_bedrock_converse(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
enabled and assert a `thinking` content block was emitted."""

12
tests/e2e/claude_code/thinking/test_bedrock_invoke.py
View file

@ -19,6 +19,8 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -55,6 +57,16 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
@pytest.mark.covers("llm.messages.bedrock_invoke.thinking.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_INVOKE_MODELS),
capabilities=(Capability.REASONING,),
mode=Mode.NONSTREAM,
)
)
def test_thinking_bedrock_invoke(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
enabled and assert a `thinking` content block was emitted."""

12
tests/e2e/claude_code/thinking/test_vertex_ai.py
View file

@ -19,6 +19,8 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -55,6 +57,16 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
@pytest.mark.covers("llm.messages.vertex.thinking.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.VERTEX_AI,),
models=tuple(VERTEX_AI_MODELS),
capabilities=(Capability.REASONING,),
mode=Mode.NONSTREAM,
)
)
def test_thinking_vertex_ai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
enabled and assert a `thinking` content block was emitted."""

12
tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py
View file

@ -28,6 +28,8 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -89,6 +91,16 @@ def _has_block_type(
@pytest.mark.covers("llm.messages.anthropic.thinking_with_tool_use.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.ANTHROPIC,),
models=tuple(ANTHROPIC_MODELS),
capabilities=(Capability.REASONING, Capability.FUNCTION_CALLING,),
mode=Mode.NONSTREAM,
)
)
def test_thinking_with_tool_use_anthropic(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
enabled and tool use, and assert both `thinking` and `tool_use`

12
tests/e2e/claude_code/thinking_with_tool_use/test_azure.py
View file

@ -22,6 +22,8 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -70,6 +72,16 @@ def _has_block_type(
@pytest.mark.covers("llm.messages.azure_foundry.thinking_with_tool_use.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.AZURE_AI,),
models=tuple(AZURE_MODELS),
capabilities=(Capability.REASONING, Capability.FUNCTION_CALLING,),
mode=Mode.NONSTREAM,
)
)
def test_thinking_with_tool_use_azure(compat_result):
base_url, api_key = require_proxy(compat_result)

12
tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py
View file

@ -27,6 +27,8 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -75,6 +77,16 @@ def _has_block_type(
@pytest.mark.covers("llm.messages.bedrock_converse.thinking_with_tool_use.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_CONVERSE_MODELS),
capabilities=(Capability.REASONING, Capability.FUNCTION_CALLING,),
mode=Mode.NONSTREAM,
)
)
def test_thinking_with_tool_use_bedrock_converse(compat_result):
base_url, api_key = require_proxy(compat_result)

12
tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py
View file

@ -29,6 +29,8 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -77,6 +79,16 @@ def _has_block_type(
@pytest.mark.covers("llm.messages.bedrock_invoke.thinking_with_tool_use.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_INVOKE_MODELS),
capabilities=(Capability.REASONING, Capability.FUNCTION_CALLING,),
mode=Mode.NONSTREAM,
)
)
def test_thinking_with_tool_use_bedrock_invoke(compat_result):
base_url, api_key = require_proxy(compat_result)

12
tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py
View file

@ -27,6 +27,8 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -75,6 +77,16 @@ def _has_block_type(
@pytest.mark.covers("llm.messages.vertex.thinking_with_tool_use.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.VERTEX_AI,),
models=tuple(VERTEX_AI_MODELS),
capabilities=(Capability.REASONING, Capability.FUNCTION_CALLING,),
mode=Mode.NONSTREAM,
)
)
def test_thinking_with_tool_use_vertex_ai(compat_result):
base_url, api_key = require_proxy(compat_result)

12
tests/e2e/claude_code/tool_search/test_anthropic.py
View file

@ -45,6 +45,8 @@ from __future__ import annotations
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy_client
from claude_code.http_probe import (
assert_tool_search_shape,
@ -60,6 +62,16 @@ ANTHROPIC_MODELS = [
@pytest.mark.covers("llm.messages.anthropic.tool_search.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.ANTHROPIC,),
models=tuple(ANTHROPIC_MODELS),
capabilities=(Capability.TOOL_SEARCH,),
mode=Mode.NONSTREAM,
)
)
def test_tool_search_anthropic(compat_result):
"""Probe `/v1/messages` with a `tool_search_tool_regex_20251119`
tool and assert the proxy + upstream accept it for every Anthropic

12
tests/e2e/claude_code/tool_search/test_azure.py
View file

@ -45,6 +45,8 @@ from __future__ import annotations
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy_client
from claude_code.http_probe import (
assert_tool_search_shape,
@ -61,6 +63,16 @@ AZURE_MODELS = [
@pytest.mark.skip(reason="stage red: Azure Foundry tool_search_server not supported in workspace for probed models")
@pytest.mark.covers("llm.messages.azure_foundry.tool_search.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.AZURE_AI,),
models=tuple(AZURE_MODELS),
capabilities=(Capability.TOOL_SEARCH,),
mode=Mode.NONSTREAM,
)
)
def test_tool_search_azure(compat_result):
"""Probe `/v1/messages` with a `tool_search_tool_regex_20251119`
tool and assert the proxy + upstream accept it for every Azure (Microsoft Foundry)

12
tests/e2e/claude_code/tool_search/test_bedrock_converse.py
View file

@ -45,6 +45,8 @@ from __future__ import annotations
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy_client
from claude_code.http_probe import (
assert_tool_search_shape,
@ -60,6 +62,16 @@ BEDROCK_CONVERSE_MODELS = [
@pytest.mark.covers("llm.messages.bedrock_converse.tool_search.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_CONVERSE_MODELS),
capabilities=(Capability.TOOL_SEARCH,),
mode=Mode.NONSTREAM,
)
)
def test_tool_search_bedrock_converse(compat_result):
"""Probe `/v1/messages` with a `tool_search_tool_regex_20251119`
tool and assert the proxy + upstream accept it for every Bedrock (Converse)

22
tests/e2e/claude_code/tool_search/test_bedrock_invoke.py
View file

@ -50,6 +50,8 @@ from __future__ import annotations
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy_client
from claude_code.http_probe import (
assert_tool_search_replay_shape,
@ -67,6 +69,16 @@ BEDROCK_INVOKE_MODELS = [
@pytest.mark.covers("llm.messages.bedrock_invoke.tool_search.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_INVOKE_MODELS),
capabilities=(Capability.TOOL_SEARCH,),
mode=Mode.NONSTREAM,
)
)
def test_tool_search_bedrock_invoke(compat_result):
"""Probe `/v1/messages` with a `tool_search_tool_regex_20251119`
tool and assert the proxy + upstream accept it for every Bedrock (Invoke)
@ -90,6 +102,16 @@ def test_tool_search_bedrock_invoke(compat_result):
@pytest.mark.covers("llm.messages.bedrock_invoke.tool_search_history.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_INVOKE_MODELS),
capabilities=(Capability.TOOL_SEARCH,),
mode=Mode.NONSTREAM,
)
)
def test_tool_search_history_bedrock_invoke(compat_result):
"""Send the tool-search request, take the real assistant turn back, and
replay it as history with the tools still declared.

12
tests/e2e/claude_code/tool_search/test_vertex_ai.py
View file

@ -45,6 +45,8 @@ from __future__ import annotations
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy_client
from claude_code.http_probe import (
assert_tool_search_shape,
@ -61,6 +63,16 @@ VERTEX_AI_MODELS = [
@pytest.mark.skip(reason="stage red: Vertex rejects tool_search when deployment extra_headers inject context-1m beta; product/config")
@pytest.mark.covers("llm.messages.vertex.tool_search.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.VERTEX_AI,),
models=tuple(VERTEX_AI_MODELS),
capabilities=(Capability.TOOL_SEARCH,),
mode=Mode.NONSTREAM,
)
)
def test_tool_search_vertex_ai(compat_result):
"""Probe `/v1/messages` with a `tool_search_tool_regex_20251119`
tool and assert the proxy + upstream accept it for every Vertex AI

11
tests/e2e/claude_code/tool_use/test_anthropic.py
View file

@ -19,6 +19,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -72,6 +73,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
@pytest.mark.covers("llm.messages.anthropic.tool_use.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.ANTHROPIC,),
models=tuple(ANTHROPIC_MODELS),
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.NONSTREAM,
)
)
def test_tool_use_anthropic(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
tool call was emitted on the wire."""

11
tests/e2e/claude_code/tool_use/test_azure.py
View file

@ -23,6 +23,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -66,6 +67,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
@pytest.mark.covers("llm.messages.azure_foundry.tool_use.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.AZURE_AI,),
models=tuple(AZURE_MODELS),
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.NONSTREAM,
)
)
def test_tool_use_azure(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
tool call was emitted on the wire."""

11
tests/e2e/claude_code/tool_use/test_azure_openai.py
View file

@ -29,6 +29,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -67,6 +68,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
return False
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.AZURE,),
models=tuple(AZURE_OPENAI_MODELS),
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.NONSTREAM,
)
)
def test_tool_use_azure_openai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
tool call was emitted on the wire by each GPT-5.6 tier."""

11
tests/e2e/claude_code/tool_use/test_bedrock_converse.py
View file

@ -19,6 +19,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -62,6 +63,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
@pytest.mark.covers("llm.messages.bedrock_converse.tool_use.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_CONVERSE_MODELS),
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.NONSTREAM,
)
)
def test_tool_use_bedrock_converse(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
tool call was emitted on the wire."""

11
tests/e2e/claude_code/tool_use/test_bedrock_invoke.py
View file

@ -19,6 +19,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -62,6 +63,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
@pytest.mark.covers("llm.messages.bedrock_invoke.tool_use.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_INVOKE_MODELS),
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.NONSTREAM,
)
)
def test_tool_use_bedrock_invoke(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
tool call was emitted on the wire."""

11
tests/e2e/claude_code/tool_use/test_bedrock_mantle.py
View file

@ -33,6 +33,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code._gpt_cells import skip_unless_mantle_cells_enabled
from claude_code.cli_driver import (
@ -72,6 +73,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
return False
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK_MANTLE,),
models=tuple(BEDROCK_MANTLE_MODELS),
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.NONSTREAM,
)
)
def test_tool_use_bedrock_mantle(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
tool call was emitted on the wire by each GPT-5.6 tier."""

11
tests/e2e/claude_code/tool_use/test_openai.py
View file

@ -28,6 +28,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled
from claude_code.cli_driver import (
@ -67,6 +68,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
return False
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.OPENAI,),
models=tuple(OPENAI_MODELS),
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.NONSTREAM,
)
)
def test_tool_use_openai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
tool call was emitted on the wire by each GPT-5.6 tier."""

11
tests/e2e/claude_code/tool_use/test_vertex_ai.py
View file

@ -19,6 +19,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -62,6 +63,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
@pytest.mark.covers("llm.messages.vertex.tool_use.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.VERTEX_AI,),
models=tuple(VERTEX_AI_MODELS),
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.NONSTREAM,
)
)
def test_tool_use_vertex_ai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
tool call was emitted on the wire."""

2
tests/e2e/claude_code/tool_use/test_vertex_ai_gpt.py
View file

@ -20,9 +20,11 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
from e2e_metadata import Domain, Subject, meta
from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON
@meta(Subject(domain=Domain.LLM_TRANSLATION))
def test_tool_use_vertex_ai_gpt(compat_result):
"""Record the static not_applicable outcome for this cell."""
compat_result.set(

11
tests/e2e/claude_code/tool_use_streaming/test_anthropic.py
View file

@ -29,6 +29,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -98,6 +99,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
@pytest.mark.covers("llm.messages.anthropic.tool_use.stream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.ANTHROPIC,),
models=tuple(ANTHROPIC_MODELS),
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.STREAM,
)
)
def test_tool_use_streaming_anthropic(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
proxy preserves fine-grained tool streaming end-to-end."""

11
tests/e2e/claude_code/tool_use_streaming/test_azure.py
View file

@ -21,6 +21,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -83,6 +84,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
@pytest.mark.covers("llm.messages.azure_foundry.tool_use.stream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.AZURE_AI,),
models=tuple(AZURE_MODELS),
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.STREAM,
)
)
def test_tool_use_streaming_azure(compat_result):
base_url, api_key = require_proxy(compat_result)

11
tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py
View file

@ -31,6 +31,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -88,6 +89,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
)
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.AZURE,),
models=tuple(AZURE_OPENAI_MODELS),
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.STREAM,
)
)
def test_tool_use_streaming_azure_openai(compat_result):
proxy = require_proxy(compat_result)

11
tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py
View file

@ -27,6 +27,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -89,6 +90,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
@pytest.mark.covers("llm.messages.bedrock_converse.tool_use.stream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_CONVERSE_MODELS),
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.STREAM,
)
)
def test_tool_use_streaming_bedrock_converse(compat_result):
base_url, api_key = require_proxy(compat_result)

11
tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py
View file

@ -25,6 +25,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -87,6 +88,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
@pytest.mark.covers("llm.messages.bedrock_invoke.tool_use.stream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_INVOKE_MODELS),
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.STREAM,
)
)
def test_tool_use_streaming_bedrock_invoke(compat_result):
base_url, api_key = require_proxy(compat_result)

11
tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py
View file

@ -34,6 +34,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code._gpt_cells import skip_unless_mantle_cells_enabled
from claude_code.cli_driver import (
@ -92,6 +93,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
)
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK_MANTLE,),
models=tuple(BEDROCK_MANTLE_MODELS),
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.STREAM,
)
)
def test_tool_use_streaming_bedrock_mantle(compat_result):
skip_unless_mantle_cells_enabled()
proxy = require_proxy(compat_result)

11
tests/e2e/claude_code/tool_use_streaming/test_openai.py
View file

@ -29,6 +29,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled
from claude_code.cli_driver import (
@ -87,6 +88,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
)
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.OPENAI,),
models=tuple(OPENAI_MODELS),
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.STREAM,
)
)
def test_tool_use_streaming_openai(compat_result):
skip_unless_openai_gpt_cells_enabled()
proxy = require_proxy(compat_result)

11
tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py
View file

@ -24,6 +24,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -86,6 +87,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
@pytest.mark.covers("llm.messages.vertex.tool_use.stream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.VERTEX_AI,),
models=tuple(VERTEX_AI_MODELS),
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.STREAM,
)
)
def test_tool_use_streaming_vertex_ai(compat_result):
base_url, api_key = require_proxy(compat_result)

2
tests/e2e/claude_code/tool_use_streaming/test_vertex_ai_gpt.py
View file

@ -20,9 +20,11 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
from e2e_metadata import Domain, Subject, meta
from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON
@meta(Subject(domain=Domain.LLM_TRANSLATION))
def test_tool_use_streaming_vertex_ai_gpt(compat_result):
"""Record the static not_applicable outcome for this cell."""
compat_result.set(

11
tests/e2e/claude_code/vision/test_anthropic.py
View file

@ -27,6 +27,7 @@ from __future__ import annotations
import json
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -83,6 +84,16 @@ def _build_stdin_input() -> str:
@pytest.mark.covers("llm.messages.anthropic.vision.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.ANTHROPIC,),
models=tuple(ANTHROPIC_MODELS),
capabilities=(Capability.VISION,),
mode=Mode.NONSTREAM,
)
)
def test_vision_anthropic(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy with an image
attached via stream-json input and assert a non-empty reply."""

11
tests/e2e/claude_code/vision/test_azure.py
View file

@ -27,6 +27,7 @@ from __future__ import annotations
import json
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -83,6 +84,16 @@ def _build_stdin_input() -> str:
@pytest.mark.covers("llm.messages.azure_foundry.vision.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.AZURE_AI,),
models=tuple(AZURE_MODELS),
capabilities=(Capability.VISION,),
mode=Mode.NONSTREAM,
)
)
def test_vision_azure(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy with an image
attached via stream-json input and assert a non-empty reply."""

11
tests/e2e/claude_code/vision/test_bedrock_converse.py
View file

@ -27,6 +27,7 @@ from __future__ import annotations
import json
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -83,6 +84,16 @@ def _build_stdin_input() -> str:
@pytest.mark.covers("llm.messages.bedrock_converse.vision.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_CONVERSE_MODELS),
capabilities=(Capability.VISION,),
mode=Mode.NONSTREAM,
)
)
def test_vision_bedrock_converse(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy with an image
attached via stream-json input and assert a non-empty reply."""

11
tests/e2e/claude_code/vision/test_bedrock_invoke.py
View file

@ -27,6 +27,7 @@ from __future__ import annotations
import json
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -83,6 +84,16 @@ def _build_stdin_input() -> str:
@pytest.mark.covers("llm.messages.bedrock_invoke.vision.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_INVOKE_MODELS),
capabilities=(Capability.VISION,),
mode=Mode.NONSTREAM,
)
)
def test_vision_bedrock_invoke(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy with an image
attached via stream-json input and assert a non-empty reply."""

11
tests/e2e/claude_code/vision/test_vertex_ai.py
View file

@ -27,6 +27,7 @@ from __future__ import annotations
import json
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -83,6 +84,16 @@ def _build_stdin_input() -> str:
@pytest.mark.covers("llm.messages.vertex.vision.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.VERTEX_AI,),
models=tuple(VERTEX_AI_MODELS),
capabilities=(Capability.VISION,),
mode=Mode.NONSTREAM,
)
)
def test_vision_vertex_ai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy with an image
attached via stream-json input and assert a non-empty reply."""

11
tests/e2e/claude_code/web_search/test_anthropic.py
View file

@ -31,6 +31,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -85,6 +86,16 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool:
@pytest.mark.covers("llm.messages.anthropic.web_search.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.ANTHROPIC,),
models=tuple(ANTHROPIC_MODELS),
capabilities=(Capability.WEB_SEARCH,),
mode=Mode.NONSTREAM,
)
)
def test_web_search_anthropic(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
upstream emitted a `tool_use` block calling `WebSearch`, proving

11
tests/e2e/claude_code/web_search/test_azure.py
View file

@ -31,6 +31,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -85,6 +86,16 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool:
@pytest.mark.covers("llm.messages.azure_foundry.web_search.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.AZURE_AI,),
models=tuple(AZURE_MODELS),
capabilities=(Capability.WEB_SEARCH,),
mode=Mode.NONSTREAM,
)
)
def test_web_search_azure(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
upstream emitted a `tool_use` block calling `WebSearch`, proving

11
tests/e2e/claude_code/web_search/test_bedrock_converse.py
View file

@ -31,6 +31,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -85,6 +86,16 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool:
@pytest.mark.covers("llm.messages.bedrock_converse.web_search.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_CONVERSE_MODELS),
capabilities=(Capability.WEB_SEARCH,),
mode=Mode.NONSTREAM,
)
)
def test_web_search_bedrock_converse(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
upstream emitted a `tool_use` block calling `WebSearch`, proving

11
tests/e2e/claude_code/web_search/test_bedrock_invoke.py
View file

@ -31,6 +31,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -85,6 +86,16 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool:
@pytest.mark.covers("llm.messages.bedrock_invoke.web_search.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=tuple(BEDROCK_INVOKE_MODELS),
capabilities=(Capability.WEB_SEARCH,),
mode=Mode.NONSTREAM,
)
)
def test_web_search_bedrock_invoke(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
upstream emitted a `tool_use` block calling `WebSearch`, proving

11
tests/e2e/claude_code/web_search/test_vertex_ai.py
View file

@ -31,6 +31,7 @@ from typing import Any, Mapping, Sequence
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from claude_code._env import require_proxy
from claude_code.cli_driver import (
ClaudeCLIError,
@ -85,6 +86,16 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool:
@pytest.mark.covers("llm.messages.vertex.web_search.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.VERTEX_AI,),
models=tuple(VERTEX_AI_MODELS),
capabilities=(Capability.WEB_SEARCH,),
mode=Mode.NONSTREAM,
)
)
def test_web_search_vertex_ai(compat_result):
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
upstream emitted a `tool_use` block calling `WebSearch`, proving