From 0ae55bdf7ad0bc81a73399f5e8e820ad2a109917 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 7 Oct 2026 10:31:54 -0700 Subject: [PATCH] test(e2e): tag claude_code cells with Subject metadata and record CLI driver steps (#44961) * test(e2e): add enum values, auto-discovering label gates and secret hiding for e2e metadata * test(e2e): tag claude_code tests with Subject metadata and record CLI driver steps * docs(e2e): name every markerless harness test file that carries no Subject * test(e2e): keep the step discovery comprehensions to one for clause * test(e2e): decorate run_claude directly so the label gate discovers its step --- tests/e2e/claude_code/_basic_messaging.py | 3 +++ tests/e2e/claude_code/_passthrough.py | 6 +++++ .../test_anthropic.py | 10 +++++++++ .../test_azure.py | 10 +++++++++ .../test_azure_openai.py | 10 +++++++++ .../test_bedrock_converse.py | 10 +++++++++ .../test_bedrock_invoke.py | 10 +++++++++ .../test_bedrock_mantle.py | 10 +++++++++ .../test_openai.py | 10 +++++++++ .../test_vertex_ai.py | 10 +++++++++ .../test_vertex_ai_gpt.py | 2 ++ .../test_anthropic.py | 10 +++++++++ .../basic_messaging_streaming/test_azure.py | 10 +++++++++ .../test_azure_openai.py | 10 +++++++++ .../test_bedrock_converse.py | 10 +++++++++ .../test_bedrock_invoke.py | 10 +++++++++ .../test_bedrock_mantle.py | 10 +++++++++ .../basic_messaging_streaming/test_openai.py | 10 +++++++++ .../test_vertex_ai.py | 10 +++++++++ .../test_vertex_ai_gpt.py | 2 ++ tests/e2e/claude_code/cli_driver.py | 4 ++++ .../count_tokens/test_anthropic.py | 10 +++++++++ .../claude_code/count_tokens/test_azure.py | 10 +++++++++ .../count_tokens/test_bedrock_converse.py | 10 +++++++++ .../count_tokens/test_bedrock_invoke.py | 10 +++++++++ .../count_tokens/test_vertex_ai.py | 10 +++++++++ tests/e2e/claude_code/http_probe.py | 7 ++++++ .../long_context_1m/test_anthropic.py | 10 +++++++++ .../claude_code/long_context_1m/test_azure.py | 10 +++++++++ .../long_context_1m/test_bedrock_converse.py | 10 +++++++++ .../long_context_1m/test_bedrock_invoke.py | 10 +++++++++ .../long_context_1m/test_vertex_ai.py | 10 +++++++++ .../claude_code/passthrough/test_anthropic.py | 10 +++++++++ .../e2e/claude_code/passthrough/test_azure.py | 10 +++++++++ .../passthrough/test_bedrock_converse.py | 3 +++ .../passthrough/test_bedrock_invoke.py | 10 +++++++++ .../claude_code/passthrough/test_vertex_ai.py | 10 +++++++++ .../claude_code/pdf_input/test_anthropic.py | 11 ++++++++++ tests/e2e/claude_code/pdf_input/test_azure.py | 11 ++++++++++ .../pdf_input/test_bedrock_converse.py | 11 ++++++++++ .../pdf_input/test_bedrock_invoke.py | 11 ++++++++++ .../claude_code/pdf_input/test_vertex_ai.py | 11 ++++++++++ .../prompt_caching_1h/test_anthropic.py | 11 ++++++++++ .../prompt_caching_1h/test_azure.py | 11 ++++++++++ .../test_bedrock_converse.py | 11 ++++++++++ .../prompt_caching_1h/test_bedrock_invoke.py | 11 ++++++++++ .../prompt_caching_1h/test_vertex_ai.py | 11 ++++++++++ .../prompt_caching_5m/test_anthropic.py | 11 ++++++++++ .../prompt_caching_5m/test_azure.py | 11 ++++++++++ .../test_bedrock_converse.py | 11 ++++++++++ .../prompt_caching_5m/test_bedrock_invoke.py | 11 ++++++++++ .../prompt_caching_5m/test_vertex_ai.py | 11 ++++++++++ .../structured_outputs/test_anthropic.py | 11 ++++++++++ .../structured_outputs/test_azure.py | 11 ++++++++++ .../test_bedrock_converse.py | 12 ++++++++++ .../structured_outputs/test_bedrock_invoke.py | 12 ++++++++++ .../structured_outputs/test_vertex_ai.py | 12 ++++++++++ .../claude_code/thinking/test_anthropic.py | 12 ++++++++++ tests/e2e/claude_code/thinking/test_azure.py | 12 ++++++++++ .../thinking/test_bedrock_converse.py | 12 ++++++++++ .../thinking/test_bedrock_invoke.py | 12 ++++++++++ .../claude_code/thinking/test_vertex_ai.py | 12 ++++++++++ .../thinking_with_tool_use/test_anthropic.py | 12 ++++++++++ .../thinking_with_tool_use/test_azure.py | 12 ++++++++++ .../test_bedrock_converse.py | 12 ++++++++++ .../test_bedrock_invoke.py | 12 ++++++++++ .../thinking_with_tool_use/test_vertex_ai.py | 12 ++++++++++ .../claude_code/tool_search/test_anthropic.py | 12 ++++++++++ .../e2e/claude_code/tool_search/test_azure.py | 12 ++++++++++ .../tool_search/test_bedrock_converse.py | 12 ++++++++++ .../tool_search/test_bedrock_invoke.py | 22 +++++++++++++++++++ .../claude_code/tool_search/test_vertex_ai.py | 12 ++++++++++ .../claude_code/tool_use/test_anthropic.py | 11 ++++++++++ tests/e2e/claude_code/tool_use/test_azure.py | 11 ++++++++++ .../claude_code/tool_use/test_azure_openai.py | 11 ++++++++++ .../tool_use/test_bedrock_converse.py | 11 ++++++++++ .../tool_use/test_bedrock_invoke.py | 11 ++++++++++ .../tool_use/test_bedrock_mantle.py | 11 ++++++++++ tests/e2e/claude_code/tool_use/test_openai.py | 11 ++++++++++ .../claude_code/tool_use/test_vertex_ai.py | 11 ++++++++++ .../tool_use/test_vertex_ai_gpt.py | 2 ++ .../tool_use_streaming/test_anthropic.py | 11 ++++++++++ .../tool_use_streaming/test_azure.py | 11 ++++++++++ .../tool_use_streaming/test_azure_openai.py | 11 ++++++++++ .../test_bedrock_converse.py | 11 ++++++++++ .../tool_use_streaming/test_bedrock_invoke.py | 11 ++++++++++ .../tool_use_streaming/test_bedrock_mantle.py | 11 ++++++++++ .../tool_use_streaming/test_openai.py | 11 ++++++++++ .../tool_use_streaming/test_vertex_ai.py | 11 ++++++++++ .../tool_use_streaming/test_vertex_ai_gpt.py | 2 ++ .../e2e/claude_code/vision/test_anthropic.py | 11 ++++++++++ tests/e2e/claude_code/vision/test_azure.py | 11 ++++++++++ .../vision/test_bedrock_converse.py | 11 ++++++++++ .../claude_code/vision/test_bedrock_invoke.py | 11 ++++++++++ .../e2e/claude_code/vision/test_vertex_ai.py | 11 ++++++++++ .../claude_code/web_search/test_anthropic.py | 11 ++++++++++ .../e2e/claude_code/web_search/test_azure.py | 11 ++++++++++ .../web_search/test_bedrock_converse.py | 11 ++++++++++ .../web_search/test_bedrock_invoke.py | 11 ++++++++++ .../claude_code/web_search/test_vertex_ai.py | 11 ++++++++++ 100 files changed, 1030 insertions(+) diff --git a/tests/e2e/claude_code/_basic_messaging.py b/tests/e2e/claude_code/_basic_messaging.py index 7c581cc5e38..b207bb6808f 100644 --- a/tests/e2e/claude_code/_basic_messaging.py +++ b/tests/e2e/claude_code/_basic_messaging.py @@ -31,6 +31,8 @@ from typing import Any, Callable, Mapping, Sequence import pytest +from e2e_metadata import step + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -74,6 +76,7 @@ def _count_stream_event_deltas(events: Sequence[Mapping[str, Any]]) -> int: return count +@step("Run Claude Code headless against {models} through the proxy and check every model replies") def run_basic_messaging_cell( *, compat_result, diff --git a/tests/e2e/claude_code/_passthrough.py b/tests/e2e/claude_code/_passthrough.py index be7a475dff7..3693ce25a9c 100644 --- a/tests/e2e/claude_code/_passthrough.py +++ b/tests/e2e/claude_code/_passthrough.py @@ -58,6 +58,8 @@ from typing import Any, Callable, Dict, Mapping, Optional, Sequence import pytest +from e2e_metadata import step + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -118,6 +120,10 @@ def foundry_extra_env(proxy_base_url: str) -> Dict[str, str]: } +@step( + "Run Claude Code headless against {models} through the proxy's native provider passthrough route" + " and check every model replies" +) def run_passthrough_cell( *, compat_result, diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py index 21383b85da5..1a7e26d37c7 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py @@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider). from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell # Per the PRD: each cell is exercised against three Claude tiers via the @@ -34,6 +35,15 @@ ANTHROPIC_MODELS = [ @pytest.mark.covers("llm.messages.anthropic.basic.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_basic_messaging_non_streaming_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a reply. diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py index 19e88dbe3cb..1fb1cc6fac7 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py @@ -26,6 +26,7 @@ the matrix builder still sees three rows for this (feature, provider). from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell # Per-model aliases registered in the LiteLLM proxy's routing config to @@ -40,6 +41,15 @@ AZURE_MODELS = [ @pytest.mark.covers("llm.messages.azure_foundry.basic.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_basic_messaging_non_streaming_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a reply. diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py index 77876c8f7ee..2b90b4e9f02 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py @@ -25,6 +25,7 @@ green if all three pass. from __future__ import annotations +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell AZURE_OPENAI_MODELS = [ @@ -34,6 +35,15 @@ AZURE_OPENAI_MODELS = [ ] +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE,), + models=tuple(AZURE_OPENAI_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_basic_messaging_non_streaming_azure_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty reply from each GPT-5.6 tier.""" diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py index 2b0f49bc205..4b02f10e6a9 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py @@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider). from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell # Per-model aliases registered in the LiteLLM proxy's routing config to @@ -35,6 +36,15 @@ BEDROCK_CONVERSE_MODELS = [ @pytest.mark.covers("llm.messages.bedrock_converse.basic.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_basic_messaging_non_streaming_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.""" run_basic_messaging_cell( diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py index 937ea5ee27e..874dcebdc35 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py @@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider). from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell # Per-model aliases registered in the LiteLLM proxy's routing config to @@ -35,6 +36,15 @@ BEDROCK_INVOKE_MODELS = [ @pytest.mark.covers("llm.messages.bedrock_invoke.basic.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_basic_messaging_non_streaming_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.""" run_basic_messaging_cell( diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py index 8a64547a732..28e723bb450 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py @@ -25,6 +25,7 @@ COMPAT_MANTLE_CELLS=1 (see `claude_code._gpt_cells`). from __future__ import annotations +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell from claude_code._gpt_cells import skip_unless_mantle_cells_enabled @@ -35,6 +36,15 @@ BEDROCK_MANTLE_MODELS = [ ] +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK_MANTLE,), + models=tuple(BEDROCK_MANTLE_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_basic_messaging_non_streaming_bedrock_mantle(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty reply from each GPT-5.6 tier.""" diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py index 323c2f11173..27ab244c52a 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py @@ -22,6 +22,7 @@ green if all three pass. from __future__ import annotations +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled @@ -32,6 +33,15 @@ OPENAI_MODELS = [ ] +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.OPENAI,), + models=tuple(OPENAI_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_basic_messaging_non_streaming_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty reply from each GPT-5.6 tier.""" diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py index c46e5a8f762..b0b41dc3f87 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py @@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider). from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell # Per-model aliases registered in the LiteLLM proxy's routing config to @@ -35,6 +36,15 @@ VERTEX_AI_MODELS = [ @pytest.mark.covers("llm.messages.vertex.basic.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_basic_messaging_non_streaming_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.""" run_basic_messaging_cell( diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py index 3b155b6ac9d..7e8358a7975 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py @@ -16,9 +16,11 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations +from e2e_metadata import Domain, Subject, meta from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON +@meta(Subject(domain=Domain.LLM_TRANSLATION)) def test_basic_messaging_non_streaming_vertex_ai_gpt(compat_result): """Record the static not_applicable outcome for this cell.""" compat_result.set( diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py b/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py index ce453f3e523..69ca0ee475a 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py @@ -26,6 +26,7 @@ sees three rows for this (feature, provider). from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell ANTHROPIC_MODELS = [ @@ -36,6 +37,15 @@ ANTHROPIC_MODELS = [ @pytest.mark.covers("llm.messages.anthropic.basic.stream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + mode=Mode.STREAM, + ) +) def test_basic_messaging_streaming_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (one row per Claude tier). diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py b/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py index 3307194e862..f9fc39c8f65 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py @@ -20,6 +20,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell AZURE_MODELS = [ @@ -30,6 +31,15 @@ AZURE_MODELS = [ @pytest.mark.covers("llm.messages.azure_foundry.basic.stream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + mode=Mode.STREAM, + ) +) def test_basic_messaging_streaming_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (one row per Claude tier). diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py b/tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py index 357596590c7..d1ff8578f09 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py @@ -24,6 +24,7 @@ green if all three pass. from __future__ import annotations +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell AZURE_OPENAI_MODELS = [ @@ -33,6 +34,15 @@ AZURE_OPENAI_MODELS = [ ] +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE,), + models=tuple(AZURE_OPENAI_MODELS), + mode=Mode.STREAM, + ) +) def test_basic_messaging_streaming_azure_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply from each GPT-5.6 tier.""" diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py index a8bc0b77a5d..3ad0b930df4 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py @@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell BEDROCK_CONVERSE_MODELS = [ @@ -26,6 +27,15 @@ BEDROCK_CONVERSE_MODELS = [ @pytest.mark.covers("llm.messages.bedrock_converse.basic.stream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + mode=Mode.STREAM, + ) +) def test_basic_messaging_streaming_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (one row per Claude tier). diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py index c0ece0e0721..1de3236d2aa 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py @@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell BEDROCK_INVOKE_MODELS = [ @@ -26,6 +27,15 @@ BEDROCK_INVOKE_MODELS = [ @pytest.mark.covers("llm.messages.bedrock_invoke.basic.stream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + mode=Mode.STREAM, + ) +) def test_basic_messaging_streaming_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (one row per Claude tier). diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py index 38297e6a3e5..679731003bb 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py @@ -25,6 +25,7 @@ COMPAT_MANTLE_CELLS=1 (see `claude_code._gpt_cells`). from __future__ import annotations +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell from claude_code._gpt_cells import skip_unless_mantle_cells_enabled @@ -35,6 +36,15 @@ BEDROCK_MANTLE_MODELS = [ ] +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK_MANTLE,), + models=tuple(BEDROCK_MANTLE_MODELS), + mode=Mode.STREAM, + ) +) def test_basic_messaging_streaming_bedrock_mantle(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply from each GPT-5.6 tier.""" diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py b/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py index a7945fb92c0..c1617253f39 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py @@ -24,6 +24,7 @@ green if all three pass. from __future__ import annotations +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled @@ -34,6 +35,15 @@ OPENAI_MODELS = [ ] +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.OPENAI,), + models=tuple(OPENAI_MODELS), + mode=Mode.STREAM, + ) +) def test_basic_messaging_streaming_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply from each GPT-5.6 tier.""" diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py b/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py index 13f1a0abf40..1201c71dce6 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py @@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell VERTEX_AI_MODELS = [ @@ -26,6 +27,15 @@ VERTEX_AI_MODELS = [ @pytest.mark.covers("llm.messages.vertex.basic.stream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + mode=Mode.STREAM, + ) +) def test_basic_messaging_streaming_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (one row per Claude tier). diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai_gpt.py b/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai_gpt.py index f6aa01de521..6a6cec8411a 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai_gpt.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai_gpt.py @@ -16,9 +16,11 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations +from e2e_metadata import Domain, Subject, meta from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON +@meta(Subject(domain=Domain.LLM_TRANSLATION)) def test_basic_messaging_streaming_vertex_ai_gpt(compat_result): """Record the static not_applicable outcome for this cell.""" compat_result.set( diff --git a/tests/e2e/claude_code/cli_driver.py b/tests/e2e/claude_code/cli_driver.py index 6996849de80..6f7a7b39a39 100644 --- a/tests/e2e/claude_code/cli_driver.py +++ b/tests/e2e/claude_code/cli_driver.py @@ -25,6 +25,8 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, field from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, Union +from e2e_metadata import step + from claude_code.rate_limiter import ( RateLimiter, get_default_limiter, @@ -211,6 +213,7 @@ class DriverResult: duration_ms: Optional[int] = None +@step("Run Claude Code headless against {model} through the proxy") def run_claude( *, prompt: Optional[str], @@ -395,6 +398,7 @@ def _matches_failure_shape(outcome: ModelResult, pattern: "re.Pattern[str]") -> return bool(pattern.search(failure_diagnostic(outcome))) +@step("Run Claude Code headless against {models} in parallel through the proxy") def run_claude_models_parallel( *, models: Sequence[str], diff --git a/tests/e2e/claude_code/count_tokens/test_anthropic.py b/tests/e2e/claude_code/count_tokens/test_anthropic.py index 05110d24e86..19e7bcfbc81 100644 --- a/tests/e2e/claude_code/count_tokens/test_anthropic.py +++ b/tests/e2e/claude_code/count_tokens/test_anthropic.py @@ -39,6 +39,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy_client from claude_code.http_probe import ( assert_count_tokens_shape, @@ -54,6 +55,15 @@ ANTHROPIC_MODELS = [ @pytest.mark.covers("llm.messages.anthropic.count_tokens.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.COUNT_TOKENS, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_count_tokens_anthropic(compat_result): """Probe `/v1/messages/count_tokens` for each Anthropic tier and assert the response shape.""" diff --git a/tests/e2e/claude_code/count_tokens/test_azure.py b/tests/e2e/claude_code/count_tokens/test_azure.py index c60c623ae89..256babf3cd7 100644 --- a/tests/e2e/claude_code/count_tokens/test_azure.py +++ b/tests/e2e/claude_code/count_tokens/test_azure.py @@ -39,6 +39,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy_client from claude_code.http_probe import ( assert_count_tokens_shape, @@ -54,6 +55,15 @@ AZURE_MODELS = [ @pytest.mark.covers("llm.messages.azure_foundry.count_tokens.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.COUNT_TOKENS, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_count_tokens_azure(compat_result): """Probe `/v1/messages/count_tokens` for each Azure (Microsoft Foundry) tier and assert the response shape.""" diff --git a/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py b/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py index 0cb4766bb31..de5f4eae3ee 100644 --- a/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py +++ b/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py @@ -39,6 +39,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy_client from claude_code.http_probe import ( assert_count_tokens_shape, @@ -54,6 +55,15 @@ BEDROCK_CONVERSE_MODELS = [ @pytest.mark.covers("llm.messages.bedrock_converse.count_tokens.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.COUNT_TOKENS, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_count_tokens_bedrock_converse(compat_result): """Probe `/v1/messages/count_tokens` for each Bedrock (Converse) tier and assert the response shape.""" diff --git a/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py b/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py index f1389574527..56dea8a5ad6 100644 --- a/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py @@ -39,6 +39,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy_client from claude_code.http_probe import ( assert_count_tokens_shape, @@ -54,6 +55,15 @@ BEDROCK_INVOKE_MODELS = [ @pytest.mark.covers("llm.messages.bedrock_invoke.count_tokens.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.COUNT_TOKENS, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_count_tokens_bedrock_invoke(compat_result): """Probe `/v1/messages/count_tokens` for each Bedrock (Invoke) tier and assert the response shape.""" diff --git a/tests/e2e/claude_code/count_tokens/test_vertex_ai.py b/tests/e2e/claude_code/count_tokens/test_vertex_ai.py index 0894214d4f0..63d12fc5dae 100644 --- a/tests/e2e/claude_code/count_tokens/test_vertex_ai.py +++ b/tests/e2e/claude_code/count_tokens/test_vertex_ai.py @@ -39,6 +39,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy_client from claude_code.http_probe import ( assert_count_tokens_shape, @@ -55,6 +56,15 @@ VERTEX_AI_MODELS = [ @pytest.mark.skip(reason="stage red: Vertex returns not supported for token counting for Claude aliases") @pytest.mark.covers("llm.messages.vertex.count_tokens.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.COUNT_TOKENS, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_count_tokens_vertex_ai(compat_result): """Probe `/v1/messages/count_tokens` for each Vertex AI tier and assert the response shape.""" diff --git a/tests/e2e/claude_code/http_probe.py b/tests/e2e/claude_code/http_probe.py index 8aba54576c4..a6c587f98cf 100644 --- a/tests/e2e/claude_code/http_probe.py +++ b/tests/e2e/claude_code/http_probe.py @@ -42,6 +42,7 @@ from e2e_http import ( UnknownApiError, ValidationError, ) +from e2e_metadata import step from models import ( AnthropicAssistantTurn, AnthropicCustomTool, @@ -109,6 +110,7 @@ def _acquire(model: str, rate_limiter: RateLimiter | None) -> None: limiter.acquire(infer_provider(model)) +@step('Count tokens with /v1/messages/count_tokens for {model} on the message "{message}"') def probe_count_tokens( *, client: ProxyClient, @@ -132,6 +134,7 @@ def probe_count_tokens( ) +@step("Send a /v1/messages request to {model} with the tool_search tool declared") def probe_tool_search( *, client: ProxyClient, @@ -228,6 +231,10 @@ def _replay_history(answer: AnthropicMessagesResponse) -> tuple[AnthropicMessage ) +@step( + "Send a /v1/messages request to {model} with the tool_search tool declared," + " then send its answer back as history in a second request" +) def probe_tool_search_multiturn( *, client: ProxyClient, diff --git a/tests/e2e/claude_code/long_context_1m/test_anthropic.py b/tests/e2e/claude_code/long_context_1m/test_anthropic.py index 0f53e512ace..1de515c11f0 100644 --- a/tests/e2e/claude_code/long_context_1m/test_anthropic.py +++ b/tests/e2e/claude_code/long_context_1m/test_anthropic.py @@ -58,6 +58,7 @@ from typing import Sequence import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -155,6 +156,15 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: @pytest.mark.skip(reason="stage red: 1M long_context not green on stage Anthropic path yet (200k sonnet / model alias)") @pytest.mark.covers("llm.messages.anthropic.long_context_1m.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_long_context_1m_anthropic(compat_result): """Drive the `claude` CLI with a ~210k-token prompt and the `context-1m-2025-08-07` beta header; assert no 400 / 413 and a diff --git a/tests/e2e/claude_code/long_context_1m/test_azure.py b/tests/e2e/claude_code/long_context_1m/test_azure.py index cdaa7f08178..eaad5e3de9f 100644 --- a/tests/e2e/claude_code/long_context_1m/test_azure.py +++ b/tests/e2e/claude_code/long_context_1m/test_azure.py @@ -58,6 +58,7 @@ from typing import Sequence import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -155,6 +156,15 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: @pytest.mark.skip(reason="stage red: 1M long_context not green on stage Azure Foundry deployments yet") @pytest.mark.covers("llm.messages.azure_foundry.long_context_1m.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_long_context_1m_azure(compat_result): """Drive the `claude` CLI (Azure (Microsoft Foundry)) with a ~210k-token prompt and the `context-1m-2025-08-07` beta header; assert no 400 / 413 and a diff --git a/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py b/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py index 38aeef2ae63..eb72a36bf14 100644 --- a/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py +++ b/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py @@ -58,6 +58,7 @@ from typing import Sequence import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -155,6 +156,15 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: @pytest.mark.skip(reason="stage red: 1M long_context not green on stage Bedrock Converse deployments yet") @pytest.mark.covers("llm.messages.bedrock_converse.long_context_1m.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_long_context_1m_bedrock_converse(compat_result): """Drive the `claude` CLI (Bedrock (Converse)) with a ~210k-token prompt and the `context-1m-2025-08-07` beta header; assert no 400 / 413 and a diff --git a/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py b/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py index f652af4aa22..dd960c15428 100644 --- a/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py @@ -58,6 +58,7 @@ from typing import Sequence import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -155,6 +156,15 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: @pytest.mark.skip(reason="stage red: 1M long_context not green on stage Bedrock Invoke deployments yet") @pytest.mark.covers("llm.messages.bedrock_invoke.long_context_1m.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_long_context_1m_bedrock_invoke(compat_result): """Drive the `claude` CLI (Bedrock (Invoke)) with a ~210k-token prompt and the `context-1m-2025-08-07` beta header; assert no 400 / 413 and a diff --git a/tests/e2e/claude_code/long_context_1m/test_vertex_ai.py b/tests/e2e/claude_code/long_context_1m/test_vertex_ai.py index 0ad68aac138..1f344d80ba8 100644 --- a/tests/e2e/claude_code/long_context_1m/test_vertex_ai.py +++ b/tests/e2e/claude_code/long_context_1m/test_vertex_ai.py @@ -58,6 +58,7 @@ from typing import Sequence import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -155,6 +156,15 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: @pytest.mark.skip(reason="stage red: 1M long_context not green on stage Vertex deployments yet") @pytest.mark.covers("llm.messages.vertex.long_context_1m.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_long_context_1m_vertex_ai(compat_result): """Drive the `claude` CLI (Vertex AI) with a ~210k-token prompt and the `context-1m-2025-08-07` beta header; assert no 400 / 413 and a diff --git a/tests/e2e/claude_code/passthrough/test_anthropic.py b/tests/e2e/claude_code/passthrough/test_anthropic.py index 8382342ae12..bf6ab0524d0 100644 --- a/tests/e2e/claude_code/passthrough/test_anthropic.py +++ b/tests/e2e/claude_code/passthrough/test_anthropic.py @@ -22,6 +22,7 @@ no per-provider transformation is involved. from __future__ import annotations +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._passthrough import ( ANTHROPIC_PASSTHROUGH_BASE_PATH, run_passthrough_cell, @@ -34,6 +35,15 @@ ANTHROPIC_MODELS = [ ] +@meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + mode=Mode.STREAM, + ) +) def test_passthrough_anthropic(compat_result): """Drive the `claude` CLI through `{proxy}/anthropic` and assert a reply.""" run_passthrough_cell( diff --git a/tests/e2e/claude_code/passthrough/test_azure.py b/tests/e2e/claude_code/passthrough/test_azure.py index 7365b4f50da..d2690025f08 100644 --- a/tests/e2e/claude_code/passthrough/test_azure.py +++ b/tests/e2e/claude_code/passthrough/test_azure.py @@ -42,6 +42,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._passthrough import foundry_extra_env, run_passthrough_cell AZURE_MODELS = [ @@ -52,6 +53,15 @@ AZURE_MODELS = [ @pytest.mark.skip(reason="stage red: /azure passthrough drops client headers (e.g. anthropic-version); product gap") +@meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.AZURE,), + models=tuple(AZURE_MODELS), + mode=Mode.STREAM, + ) +) def test_passthrough_azure(compat_result): """Drive the `claude` CLI through `{proxy}/azure` and assert a reply.""" run_passthrough_cell( diff --git a/tests/e2e/claude_code/passthrough/test_bedrock_converse.py b/tests/e2e/claude_code/passthrough/test_bedrock_converse.py index d1093a7a958..e1605be9e49 100644 --- a/tests/e2e/claude_code/passthrough/test_bedrock_converse.py +++ b/tests/e2e/claude_code/passthrough/test_bedrock_converse.py @@ -18,7 +18,10 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations +from e2e_metadata import Domain, Subject, meta + +@meta(Subject(domain=Domain.PASSTHROUGH)) def test_passthrough_bedrock_converse(compat_result): """Report not_applicable: Claude Code has no Converse-wire mode.""" compat_result.set( diff --git a/tests/e2e/claude_code/passthrough/test_bedrock_invoke.py b/tests/e2e/claude_code/passthrough/test_bedrock_invoke.py index f1f28ab5b4c..d95118991fa 100644 --- a/tests/e2e/claude_code/passthrough/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/passthrough/test_bedrock_invoke.py @@ -23,6 +23,7 @@ cell. from __future__ import annotations +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._passthrough import bedrock_extra_env, run_passthrough_cell BEDROCK_INVOKE_MODELS = [ @@ -32,6 +33,15 @@ BEDROCK_INVOKE_MODELS = [ ] +@meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + mode=Mode.STREAM, + ) +) def test_passthrough_bedrock_invoke(compat_result): """Drive the `claude` CLI through `{proxy}/bedrock` and assert a reply.""" run_passthrough_cell( diff --git a/tests/e2e/claude_code/passthrough/test_vertex_ai.py b/tests/e2e/claude_code/passthrough/test_vertex_ai.py index 790f8b60c8f..3f84cdf02fa 100644 --- a/tests/e2e/claude_code/passthrough/test_vertex_ai.py +++ b/tests/e2e/claude_code/passthrough/test_vertex_ai.py @@ -26,6 +26,7 @@ Google and every tier fails with a 401. from __future__ import annotations +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._passthrough import run_passthrough_cell, vertex_extra_env VERTEX_MODELS = [ @@ -35,6 +36,15 @@ VERTEX_MODELS = [ ] +@meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_MODELS), + mode=Mode.STREAM, + ) +) def test_passthrough_vertex_ai(compat_result): """Drive the `claude` CLI through `{proxy}/vertex_ai` and assert a reply.""" run_passthrough_cell( diff --git a/tests/e2e/claude_code/pdf_input/test_anthropic.py b/tests/e2e/claude_code/pdf_input/test_anthropic.py index 21c8028ef1c..59a91036f26 100644 --- a/tests/e2e/claude_code/pdf_input/test_anthropic.py +++ b/tests/e2e/claude_code/pdf_input/test_anthropic.py @@ -24,6 +24,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -106,6 +107,16 @@ def _build_minimal_pdf(marker: str) -> bytes: @pytest.mark.covers("llm.messages.anthropic.pdf_input.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.PDF_INPUT,), + mode=Mode.NONSTREAM, + ) +) def test_pdf_input_anthropic(compat_result, tmp_path): """Drive the `claude` CLI against the LiteLLM proxy with a PDF attached via the Read tool and assert the reply references it.""" diff --git a/tests/e2e/claude_code/pdf_input/test_azure.py b/tests/e2e/claude_code/pdf_input/test_azure.py index 34ae3732b99..3f7642321cf 100644 --- a/tests/e2e/claude_code/pdf_input/test_azure.py +++ b/tests/e2e/claude_code/pdf_input/test_azure.py @@ -17,6 +17,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -83,6 +84,16 @@ def _build_minimal_pdf(marker: str) -> bytes: @pytest.mark.covers("llm.messages.azure_foundry.pdf_input.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.PDF_INPUT,), + mode=Mode.NONSTREAM, + ) +) def test_pdf_input_azure(compat_result, tmp_path): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py b/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py index 76aa84f0f47..14020799c21 100644 --- a/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py +++ b/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py @@ -23,6 +23,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -89,6 +90,16 @@ def _build_minimal_pdf(marker: str) -> bytes: @pytest.mark.covers("llm.messages.bedrock_converse.pdf_input.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.PDF_INPUT,), + mode=Mode.NONSTREAM, + ) +) def test_pdf_input_bedrock_converse(compat_result, tmp_path): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py b/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py index 4450266bb6b..5b3ba208421 100644 --- a/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py @@ -22,6 +22,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -88,6 +89,16 @@ def _build_minimal_pdf(marker: str) -> bytes: @pytest.mark.covers("llm.messages.bedrock_invoke.pdf_input.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.PDF_INPUT,), + mode=Mode.NONSTREAM, + ) +) def test_pdf_input_bedrock_invoke(compat_result, tmp_path): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/pdf_input/test_vertex_ai.py b/tests/e2e/claude_code/pdf_input/test_vertex_ai.py index b78f58cfda1..5f42c00efcb 100644 --- a/tests/e2e/claude_code/pdf_input/test_vertex_ai.py +++ b/tests/e2e/claude_code/pdf_input/test_vertex_ai.py @@ -17,6 +17,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -83,6 +84,16 @@ def _build_minimal_pdf(marker: str) -> bytes: @pytest.mark.covers("llm.messages.vertex.pdf_input.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.PDF_INPUT,), + mode=Mode.NONSTREAM, + ) +) def test_pdf_input_vertex_ai(compat_result, tmp_path): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py b/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py index 637be1c551d..60045eff8b9 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py @@ -28,6 +28,7 @@ from typing import Any, Mapping, Optional import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -64,6 +65,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.anthropic.prompt_cache_1h.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) def test_prompt_caching_1h_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with the 1h TTL opt-in env var set, and assert the upstream usage block diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_azure.py b/tests/e2e/claude_code/prompt_caching_1h/test_azure.py index f34557b3c5f..86dc4c8e50a 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_azure.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_azure.py @@ -19,6 +19,7 @@ from typing import Any, Mapping, Optional import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -48,6 +49,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.azure_foundry.prompt_cache_1h.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) def test_prompt_caching_1h_azure(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py index bf62a49444c..d1106ca4350 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py @@ -24,6 +24,7 @@ from typing import Any, Mapping, Optional import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -56,6 +57,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.bedrock_converse.prompt_cache_1h.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) def test_prompt_caching_1h_bedrock_converse(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py index dc3468702d4..9c1166eb3b6 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py @@ -26,6 +26,7 @@ from typing import Any, Mapping, Optional import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -60,6 +61,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.bedrock_invoke.prompt_cache_1h.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) def test_prompt_caching_1h_bedrock_invoke(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py b/tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py index 66cf961fcfc..89b6d7e1d4e 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py @@ -19,6 +19,7 @@ from typing import Any, Mapping, Optional import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -48,6 +49,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.vertex.prompt_cache_1h.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) def test_prompt_caching_1h_vertex_ai(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py b/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py index ef551beb45c..c97df9afe2c 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py @@ -26,6 +26,7 @@ from typing import Any, Mapping, Optional import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -55,6 +56,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.anthropic.prompt_cache_5m.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) def test_prompt_caching_5m_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_azure.py b/tests/e2e/claude_code/prompt_caching_5m/test_azure.py index 9d4137e0726..7705af62ab4 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_azure.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_azure.py @@ -26,6 +26,7 @@ from typing import Any, Mapping, Optional import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -53,6 +54,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.azure_foundry.prompt_cache_5m.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) def test_prompt_caching_5m_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py index c9b34c010b0..4a5a7dfd660 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py @@ -19,6 +19,7 @@ from typing import Any, Mapping, Optional import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -46,6 +47,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.bedrock_converse.prompt_cache_5m.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) def test_prompt_caching_5m_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py index b95c509ba3c..bbe483242bd 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py @@ -19,6 +19,7 @@ from typing import Any, Mapping, Optional import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -46,6 +47,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.bedrock_invoke.prompt_cache_5m.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) def test_prompt_caching_5m_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py b/tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py index f79377b7372..3f325efcc93 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py @@ -19,6 +19,7 @@ from typing import Any, Mapping, Optional import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -46,6 +47,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.vertex.prompt_cache_5m.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) def test_prompt_caching_5m_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" diff --git a/tests/e2e/claude_code/structured_outputs/test_anthropic.py b/tests/e2e/claude_code/structured_outputs/test_anthropic.py index 3dc4c7ab8f2..c8cb7e7386c 100644 --- a/tests/e2e/claude_code/structured_outputs/test_anthropic.py +++ b/tests/e2e/claude_code/structured_outputs/test_anthropic.py @@ -53,6 +53,7 @@ from typing import Any, Mapping, Optional, Sequence, Tuple import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -151,6 +152,16 @@ def _validate_against_schema( @pytest.mark.covers("llm.messages.anthropic.structured_output.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.NONSTREAM, + ) +) def test_structured_outputs_anthropic(compat_result): """Drive `claude --json-schema ...` against the LiteLLM proxy and assert the trailing `result` event contains a schema-conforming diff --git a/tests/e2e/claude_code/structured_outputs/test_azure.py b/tests/e2e/claude_code/structured_outputs/test_azure.py index 7a776ed55ad..0708920f5e7 100644 --- a/tests/e2e/claude_code/structured_outputs/test_azure.py +++ b/tests/e2e/claude_code/structured_outputs/test_azure.py @@ -53,6 +53,7 @@ from typing import Any, Mapping, Optional, Sequence, Tuple import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -151,6 +152,16 @@ def _validate_against_schema( @pytest.mark.covers("llm.messages.azure_foundry.structured_output.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.NONSTREAM, + ) +) def test_structured_outputs_azure(compat_result): """Drive `claude --json-schema ...` against the LiteLLM proxy and assert the trailing `result` event contains a schema-conforming diff --git a/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py b/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py index 345d7c327cf..7228c194f85 100644 --- a/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py +++ b/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py @@ -53,6 +53,8 @@ from typing import Any, Mapping, Optional, Sequence, Tuple import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -151,6 +153,16 @@ def _validate_against_schema( @pytest.mark.covers("llm.messages.bedrock_converse.structured_output.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.NONSTREAM, + ) +) def test_structured_outputs_bedrock_converse(compat_result): """Drive `claude --json-schema ...` against the LiteLLM proxy and assert the trailing `result` event contains a schema-conforming diff --git a/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py b/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py index 0cf48c72d4f..c415de9e396 100644 --- a/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py @@ -53,6 +53,8 @@ from typing import Any, Mapping, Optional, Sequence, Tuple import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -151,6 +153,16 @@ def _validate_against_schema( @pytest.mark.covers("llm.messages.bedrock_invoke.structured_output.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.NONSTREAM, + ) +) def test_structured_outputs_bedrock_invoke(compat_result): """Drive `claude --json-schema ...` against the LiteLLM proxy and assert the trailing `result` event contains a schema-conforming diff --git a/tests/e2e/claude_code/structured_outputs/test_vertex_ai.py b/tests/e2e/claude_code/structured_outputs/test_vertex_ai.py index 24f5a0c35d4..a4cf669c19f 100644 --- a/tests/e2e/claude_code/structured_outputs/test_vertex_ai.py +++ b/tests/e2e/claude_code/structured_outputs/test_vertex_ai.py @@ -53,6 +53,8 @@ from typing import Any, Mapping, Optional, Sequence, Tuple import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -151,6 +153,16 @@ def _validate_against_schema( @pytest.mark.covers("llm.messages.vertex.structured_output.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.NONSTREAM, + ) +) def test_structured_outputs_vertex_ai(compat_result): """Drive `claude --json-schema ...` against the LiteLLM proxy and assert the trailing `result` event contains a schema-conforming diff --git a/tests/e2e/claude_code/thinking/test_anthropic.py b/tests/e2e/claude_code/thinking/test_anthropic.py index ebb2445fb6d..2c3a8f0a8a9 100644 --- a/tests/e2e/claude_code/thinking/test_anthropic.py +++ b/tests/e2e/claude_code/thinking/test_anthropic.py @@ -24,6 +24,8 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -75,6 +77,16 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.anthropic.thinking.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) +) def test_thinking_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" diff --git a/tests/e2e/claude_code/thinking/test_azure.py b/tests/e2e/claude_code/thinking/test_azure.py index ffd5ca92df0..0828dd033f4 100644 --- a/tests/e2e/claude_code/thinking/test_azure.py +++ b/tests/e2e/claude_code/thinking/test_azure.py @@ -27,6 +27,8 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -63,6 +65,16 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.azure_foundry.thinking.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) +) def test_thinking_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" diff --git a/tests/e2e/claude_code/thinking/test_bedrock_converse.py b/tests/e2e/claude_code/thinking/test_bedrock_converse.py index 0b409f18ea7..fd075298234 100644 --- a/tests/e2e/claude_code/thinking/test_bedrock_converse.py +++ b/tests/e2e/claude_code/thinking/test_bedrock_converse.py @@ -19,6 +19,8 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -55,6 +57,16 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.bedrock_converse.thinking.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) +) def test_thinking_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" diff --git a/tests/e2e/claude_code/thinking/test_bedrock_invoke.py b/tests/e2e/claude_code/thinking/test_bedrock_invoke.py index a2c97eae321..c115ba9e408 100644 --- a/tests/e2e/claude_code/thinking/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/thinking/test_bedrock_invoke.py @@ -19,6 +19,8 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -55,6 +57,16 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.bedrock_invoke.thinking.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) +) def test_thinking_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" diff --git a/tests/e2e/claude_code/thinking/test_vertex_ai.py b/tests/e2e/claude_code/thinking/test_vertex_ai.py index f1a1c5b6cee..b947f97e7ac 100644 --- a/tests/e2e/claude_code/thinking/test_vertex_ai.py +++ b/tests/e2e/claude_code/thinking/test_vertex_ai.py @@ -19,6 +19,8 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -55,6 +57,16 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.vertex.thinking.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) +) def test_thinking_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py b/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py index 7e39ea26d42..8ddbdba04f9 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py @@ -28,6 +28,8 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -89,6 +91,16 @@ def _has_block_type( @pytest.mark.covers("llm.messages.anthropic.thinking_with_tool_use.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.REASONING, Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_thinking_with_tool_use_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and tool use, and assert both `thinking` and `tool_use` diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py b/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py index 0371a10f8a6..6dbd6b27fde 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py @@ -22,6 +22,8 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -70,6 +72,16 @@ def _has_block_type( @pytest.mark.covers("llm.messages.azure_foundry.thinking_with_tool_use.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.REASONING, Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_thinking_with_tool_use_azure(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py index 026d2a3707f..b3869f157b5 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py @@ -27,6 +27,8 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -75,6 +77,16 @@ def _has_block_type( @pytest.mark.covers("llm.messages.bedrock_converse.thinking_with_tool_use.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.REASONING, Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_thinking_with_tool_use_bedrock_converse(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py index 1dd4cf0a73c..384ce0bb3c6 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py @@ -29,6 +29,8 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -77,6 +79,16 @@ def _has_block_type( @pytest.mark.covers("llm.messages.bedrock_invoke.thinking_with_tool_use.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.REASONING, Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_thinking_with_tool_use_bedrock_invoke(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py b/tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py index b25228edb55..c3a3dd58533 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py @@ -27,6 +27,8 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -75,6 +77,16 @@ def _has_block_type( @pytest.mark.covers("llm.messages.vertex.thinking_with_tool_use.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.REASONING, Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_thinking_with_tool_use_vertex_ai(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/tool_search/test_anthropic.py b/tests/e2e/claude_code/tool_search/test_anthropic.py index a23d1b3d2bb..f08c30b507e 100644 --- a/tests/e2e/claude_code/tool_search/test_anthropic.py +++ b/tests/e2e/claude_code/tool_search/test_anthropic.py @@ -45,6 +45,8 @@ from __future__ import annotations import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy_client from claude_code.http_probe import ( assert_tool_search_shape, @@ -60,6 +62,16 @@ ANTHROPIC_MODELS = [ @pytest.mark.covers("llm.messages.anthropic.tool_search.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.TOOL_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_tool_search_anthropic(compat_result): """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` tool and assert the proxy + upstream accept it for every Anthropic diff --git a/tests/e2e/claude_code/tool_search/test_azure.py b/tests/e2e/claude_code/tool_search/test_azure.py index b094a35ea63..87628bc02d5 100644 --- a/tests/e2e/claude_code/tool_search/test_azure.py +++ b/tests/e2e/claude_code/tool_search/test_azure.py @@ -45,6 +45,8 @@ from __future__ import annotations import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy_client from claude_code.http_probe import ( assert_tool_search_shape, @@ -61,6 +63,16 @@ AZURE_MODELS = [ @pytest.mark.skip(reason="stage red: Azure Foundry tool_search_server not supported in workspace for probed models") @pytest.mark.covers("llm.messages.azure_foundry.tool_search.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.TOOL_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_tool_search_azure(compat_result): """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` tool and assert the proxy + upstream accept it for every Azure (Microsoft Foundry) diff --git a/tests/e2e/claude_code/tool_search/test_bedrock_converse.py b/tests/e2e/claude_code/tool_search/test_bedrock_converse.py index f395122a5ab..84fd47bcb39 100644 --- a/tests/e2e/claude_code/tool_search/test_bedrock_converse.py +++ b/tests/e2e/claude_code/tool_search/test_bedrock_converse.py @@ -45,6 +45,8 @@ from __future__ import annotations import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy_client from claude_code.http_probe import ( assert_tool_search_shape, @@ -60,6 +62,16 @@ BEDROCK_CONVERSE_MODELS = [ @pytest.mark.covers("llm.messages.bedrock_converse.tool_search.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.TOOL_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_tool_search_bedrock_converse(compat_result): """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` tool and assert the proxy + upstream accept it for every Bedrock (Converse) diff --git a/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py b/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py index 5b4c50e9dc5..b4a7f721a92 100644 --- a/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py @@ -50,6 +50,8 @@ from __future__ import annotations import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy_client from claude_code.http_probe import ( assert_tool_search_replay_shape, @@ -67,6 +69,16 @@ BEDROCK_INVOKE_MODELS = [ @pytest.mark.covers("llm.messages.bedrock_invoke.tool_search.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.TOOL_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_tool_search_bedrock_invoke(compat_result): """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` tool and assert the proxy + upstream accept it for every Bedrock (Invoke) @@ -90,6 +102,16 @@ def test_tool_search_bedrock_invoke(compat_result): @pytest.mark.covers("llm.messages.bedrock_invoke.tool_search_history.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.TOOL_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_tool_search_history_bedrock_invoke(compat_result): """Send the tool-search request, take the real assistant turn back, and replay it as history with the tools still declared. diff --git a/tests/e2e/claude_code/tool_search/test_vertex_ai.py b/tests/e2e/claude_code/tool_search/test_vertex_ai.py index 7d0d35b1c1d..af629a7692e 100644 --- a/tests/e2e/claude_code/tool_search/test_vertex_ai.py +++ b/tests/e2e/claude_code/tool_search/test_vertex_ai.py @@ -45,6 +45,8 @@ from __future__ import annotations import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy_client from claude_code.http_probe import ( assert_tool_search_shape, @@ -61,6 +63,16 @@ VERTEX_AI_MODELS = [ @pytest.mark.skip(reason="stage red: Vertex rejects tool_search when deployment extra_headers inject context-1m beta; product/config") @pytest.mark.covers("llm.messages.vertex.tool_search.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.TOOL_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_tool_search_vertex_ai(compat_result): """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` tool and assert the proxy + upstream accept it for every Vertex AI diff --git a/tests/e2e/claude_code/tool_use/test_anthropic.py b/tests/e2e/claude_code/tool_use/test_anthropic.py index 9ff4c58907f..429557322f6 100644 --- a/tests/e2e/claude_code/tool_use/test_anthropic.py +++ b/tests/e2e/claude_code/tool_use/test_anthropic.py @@ -19,6 +19,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -72,6 +73,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.anthropic.tool_use.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_tool_use_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" diff --git a/tests/e2e/claude_code/tool_use/test_azure.py b/tests/e2e/claude_code/tool_use/test_azure.py index 9e7398267c4..96946093d9d 100644 --- a/tests/e2e/claude_code/tool_use/test_azure.py +++ b/tests/e2e/claude_code/tool_use/test_azure.py @@ -23,6 +23,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -66,6 +67,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.azure_foundry.tool_use.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_tool_use_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" diff --git a/tests/e2e/claude_code/tool_use/test_azure_openai.py b/tests/e2e/claude_code/tool_use/test_azure_openai.py index 7e1eecdbc03..3cb12b4023e 100644 --- a/tests/e2e/claude_code/tool_use/test_azure_openai.py +++ b/tests/e2e/claude_code/tool_use/test_azure_openai.py @@ -29,6 +29,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -67,6 +68,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE,), + models=tuple(AZURE_OPENAI_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_tool_use_azure_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire by each GPT-5.6 tier.""" diff --git a/tests/e2e/claude_code/tool_use/test_bedrock_converse.py b/tests/e2e/claude_code/tool_use/test_bedrock_converse.py index 33d4d3820d2..f361757042b 100644 --- a/tests/e2e/claude_code/tool_use/test_bedrock_converse.py +++ b/tests/e2e/claude_code/tool_use/test_bedrock_converse.py @@ -19,6 +19,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -62,6 +63,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.bedrock_converse.tool_use.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_tool_use_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" diff --git a/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py b/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py index 47ae3aef1da..16150c5eb9d 100644 --- a/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py @@ -19,6 +19,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -62,6 +63,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.bedrock_invoke.tool_use.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_tool_use_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" diff --git a/tests/e2e/claude_code/tool_use/test_bedrock_mantle.py b/tests/e2e/claude_code/tool_use/test_bedrock_mantle.py index e9cb70e74e9..b04b19e5542 100644 --- a/tests/e2e/claude_code/tool_use/test_bedrock_mantle.py +++ b/tests/e2e/claude_code/tool_use/test_bedrock_mantle.py @@ -33,6 +33,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code._gpt_cells import skip_unless_mantle_cells_enabled from claude_code.cli_driver import ( @@ -72,6 +73,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK_MANTLE,), + models=tuple(BEDROCK_MANTLE_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_tool_use_bedrock_mantle(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire by each GPT-5.6 tier.""" diff --git a/tests/e2e/claude_code/tool_use/test_openai.py b/tests/e2e/claude_code/tool_use/test_openai.py index ffb7e795c2b..d8a9ba83480 100644 --- a/tests/e2e/claude_code/tool_use/test_openai.py +++ b/tests/e2e/claude_code/tool_use/test_openai.py @@ -28,6 +28,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled from claude_code.cli_driver import ( @@ -67,6 +68,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.OPENAI,), + models=tuple(OPENAI_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_tool_use_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire by each GPT-5.6 tier.""" diff --git a/tests/e2e/claude_code/tool_use/test_vertex_ai.py b/tests/e2e/claude_code/tool_use/test_vertex_ai.py index 79a3016345c..a082a61920e 100644 --- a/tests/e2e/claude_code/tool_use/test_vertex_ai.py +++ b/tests/e2e/claude_code/tool_use/test_vertex_ai.py @@ -19,6 +19,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -62,6 +63,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.vertex.tool_use.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_tool_use_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" diff --git a/tests/e2e/claude_code/tool_use/test_vertex_ai_gpt.py b/tests/e2e/claude_code/tool_use/test_vertex_ai_gpt.py index d1ebbced9dc..c15ba00d325 100644 --- a/tests/e2e/claude_code/tool_use/test_vertex_ai_gpt.py +++ b/tests/e2e/claude_code/tool_use/test_vertex_ai_gpt.py @@ -20,9 +20,11 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations +from e2e_metadata import Domain, Subject, meta from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON +@meta(Subject(domain=Domain.LLM_TRANSLATION)) def test_tool_use_vertex_ai_gpt(compat_result): """Record the static not_applicable outcome for this cell.""" compat_result.set( diff --git a/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py b/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py index 152652dcf3c..91002e7ebf9 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py @@ -29,6 +29,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -98,6 +99,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.anthropic.tool_use.stream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) def test_tool_use_streaming_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the proxy preserves fine-grained tool streaming end-to-end.""" diff --git a/tests/e2e/claude_code/tool_use_streaming/test_azure.py b/tests/e2e/claude_code/tool_use_streaming/test_azure.py index 8a1cc1852dd..4ec93f5a667 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_azure.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_azure.py @@ -21,6 +21,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -83,6 +84,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.azure_foundry.tool_use.stream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) def test_tool_use_streaming_azure(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py b/tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py index ad5d4e0f613..020f4e79eb4 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py @@ -31,6 +31,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -88,6 +89,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: ) +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE,), + models=tuple(AZURE_OPENAI_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) def test_tool_use_streaming_azure_openai(compat_result): proxy = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py index 3b04ed5962f..2d4b5764eb9 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py @@ -27,6 +27,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -89,6 +90,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.bedrock_converse.tool_use.stream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) def test_tool_use_streaming_bedrock_converse(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py index c7b61129782..2cd7a68d57a 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py @@ -25,6 +25,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -87,6 +88,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.bedrock_invoke.tool_use.stream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) def test_tool_use_streaming_bedrock_invoke(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py index 20fae5d48db..190eb51adc0 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py @@ -34,6 +34,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code._gpt_cells import skip_unless_mantle_cells_enabled from claude_code.cli_driver import ( @@ -92,6 +93,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: ) +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK_MANTLE,), + models=tuple(BEDROCK_MANTLE_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) def test_tool_use_streaming_bedrock_mantle(compat_result): skip_unless_mantle_cells_enabled() proxy = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_openai.py b/tests/e2e/claude_code/tool_use_streaming/test_openai.py index a5ce31b1fd6..3555741b197 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_openai.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_openai.py @@ -29,6 +29,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled from claude_code.cli_driver import ( @@ -87,6 +88,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: ) +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.OPENAI,), + models=tuple(OPENAI_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) def test_tool_use_streaming_openai(compat_result): skip_unless_openai_gpt_cells_enabled() proxy = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py b/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py index 2912e3aae3d..7bb29742cfd 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py @@ -24,6 +24,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -86,6 +87,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.vertex.tool_use.stream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) def test_tool_use_streaming_vertex_ai(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai_gpt.py b/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai_gpt.py index 7037e91fee0..5e6c203cb57 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai_gpt.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai_gpt.py @@ -20,9 +20,11 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations +from e2e_metadata import Domain, Subject, meta from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON +@meta(Subject(domain=Domain.LLM_TRANSLATION)) def test_tool_use_streaming_vertex_ai_gpt(compat_result): """Record the static not_applicable outcome for this cell.""" compat_result.set( diff --git a/tests/e2e/claude_code/vision/test_anthropic.py b/tests/e2e/claude_code/vision/test_anthropic.py index f681b2be5ae..b37bbb4ebf5 100644 --- a/tests/e2e/claude_code/vision/test_anthropic.py +++ b/tests/e2e/claude_code/vision/test_anthropic.py @@ -27,6 +27,7 @@ from __future__ import annotations import json import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -83,6 +84,16 @@ def _build_stdin_input() -> str: @pytest.mark.covers("llm.messages.anthropic.vision.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.VISION,), + mode=Mode.NONSTREAM, + ) +) def test_vision_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with an image attached via stream-json input and assert a non-empty reply.""" diff --git a/tests/e2e/claude_code/vision/test_azure.py b/tests/e2e/claude_code/vision/test_azure.py index f0eaaad84a2..739c8aba74d 100644 --- a/tests/e2e/claude_code/vision/test_azure.py +++ b/tests/e2e/claude_code/vision/test_azure.py @@ -27,6 +27,7 @@ from __future__ import annotations import json import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -83,6 +84,16 @@ def _build_stdin_input() -> str: @pytest.mark.covers("llm.messages.azure_foundry.vision.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.VISION,), + mode=Mode.NONSTREAM, + ) +) def test_vision_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with an image attached via stream-json input and assert a non-empty reply.""" diff --git a/tests/e2e/claude_code/vision/test_bedrock_converse.py b/tests/e2e/claude_code/vision/test_bedrock_converse.py index 2a5aba5a393..07f21b7d657 100644 --- a/tests/e2e/claude_code/vision/test_bedrock_converse.py +++ b/tests/e2e/claude_code/vision/test_bedrock_converse.py @@ -27,6 +27,7 @@ from __future__ import annotations import json import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -83,6 +84,16 @@ def _build_stdin_input() -> str: @pytest.mark.covers("llm.messages.bedrock_converse.vision.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.VISION,), + mode=Mode.NONSTREAM, + ) +) def test_vision_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with an image attached via stream-json input and assert a non-empty reply.""" diff --git a/tests/e2e/claude_code/vision/test_bedrock_invoke.py b/tests/e2e/claude_code/vision/test_bedrock_invoke.py index 5c995cd479e..847f2c80484 100644 --- a/tests/e2e/claude_code/vision/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/vision/test_bedrock_invoke.py @@ -27,6 +27,7 @@ from __future__ import annotations import json import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -83,6 +84,16 @@ def _build_stdin_input() -> str: @pytest.mark.covers("llm.messages.bedrock_invoke.vision.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.VISION,), + mode=Mode.NONSTREAM, + ) +) def test_vision_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with an image attached via stream-json input and assert a non-empty reply.""" diff --git a/tests/e2e/claude_code/vision/test_vertex_ai.py b/tests/e2e/claude_code/vision/test_vertex_ai.py index 8d385e295d0..256b762f06a 100644 --- a/tests/e2e/claude_code/vision/test_vertex_ai.py +++ b/tests/e2e/claude_code/vision/test_vertex_ai.py @@ -27,6 +27,7 @@ from __future__ import annotations import json import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -83,6 +84,16 @@ def _build_stdin_input() -> str: @pytest.mark.covers("llm.messages.vertex.vision.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.VISION,), + mode=Mode.NONSTREAM, + ) +) def test_vision_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with an image attached via stream-json input and assert a non-empty reply.""" diff --git a/tests/e2e/claude_code/web_search/test_anthropic.py b/tests/e2e/claude_code/web_search/test_anthropic.py index a20a2133dc9..8b7f4259a68 100644 --- a/tests/e2e/claude_code/web_search/test_anthropic.py +++ b/tests/e2e/claude_code/web_search/test_anthropic.py @@ -31,6 +31,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -85,6 +86,16 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.anthropic.web_search.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.WEB_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_web_search_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream emitted a `tool_use` block calling `WebSearch`, proving diff --git a/tests/e2e/claude_code/web_search/test_azure.py b/tests/e2e/claude_code/web_search/test_azure.py index 8f9f638fbee..da3d1ebeb0b 100644 --- a/tests/e2e/claude_code/web_search/test_azure.py +++ b/tests/e2e/claude_code/web_search/test_azure.py @@ -31,6 +31,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -85,6 +86,16 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.azure_foundry.web_search.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.WEB_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_web_search_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream emitted a `tool_use` block calling `WebSearch`, proving diff --git a/tests/e2e/claude_code/web_search/test_bedrock_converse.py b/tests/e2e/claude_code/web_search/test_bedrock_converse.py index 32f37b2be79..36c163dd2d6 100644 --- a/tests/e2e/claude_code/web_search/test_bedrock_converse.py +++ b/tests/e2e/claude_code/web_search/test_bedrock_converse.py @@ -31,6 +31,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -85,6 +86,16 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.bedrock_converse.web_search.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.WEB_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_web_search_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream emitted a `tool_use` block calling `WebSearch`, proving diff --git a/tests/e2e/claude_code/web_search/test_bedrock_invoke.py b/tests/e2e/claude_code/web_search/test_bedrock_invoke.py index 68d1b30e83f..1328d361404 100644 --- a/tests/e2e/claude_code/web_search/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/web_search/test_bedrock_invoke.py @@ -31,6 +31,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -85,6 +86,16 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.bedrock_invoke.web_search.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.WEB_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_web_search_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream emitted a `tool_use` block calling `WebSearch`, proving diff --git a/tests/e2e/claude_code/web_search/test_vertex_ai.py b/tests/e2e/claude_code/web_search/test_vertex_ai.py index 540a8396c98..7ab352b1953 100644 --- a/tests/e2e/claude_code/web_search/test_vertex_ai.py +++ b/tests/e2e/claude_code/web_search/test_vertex_ai.py @@ -31,6 +31,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -85,6 +86,16 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.vertex.web_search.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.WEB_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_web_search_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream emitted a `tool_use` block calling `WebSearch`, proving