mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
test(e2e): wire claude_code compat matrix cells into the coverage registry
This commit is contained in:
mateo-berri
parent
ff4a40f017
commit
c24abe0fd8
78 changed files with 173 additions and 2 deletions
|
|
@ -81,8 +81,9 @@ llm.<endpoint>.<route>.<capability>.<streaming>.<assertion>
|
|||
route : openai | azure_openai | anthropic | bedrock_converse | bedrock_invoke | vertex
|
||||
| azure_foundry | cohere | together_ai
|
||||
(vocab varies per endpoint; messages is anthropic-format only)
|
||||
capability : basic | tool_use | prompt_cache_5m | vision | thinking | structured_output
|
||||
| service_tier | mid_conversation_system
|
||||
capability : basic | tool_use | prompt_cache_5m | prompt_cache_1h | vision | thinking
|
||||
| thinking_with_tool_use | structured_output | pdf | web_search | tool_search
|
||||
| count_tokens | long_context | service_tier | mid_conversation_system
|
||||
streaming : stream | nonstream (omit where n/a)
|
||||
assertion : works | cost_logged | cache_hit
|
||||
label (not in id): model = haiku-4.5 | sonnet-4.6 | opus-4.7 | gpt-*
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ the matrix builder still sees three rows for this (feature, provider).
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from claude_code._basic_messaging import run_basic_messaging_cell
|
||||
|
||||
# Per the PRD: each cell is exercised against three Claude tiers via the
|
||||
|
|
@ -32,6 +34,7 @@ ANTHROPIC_MODELS = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.anthropic.basic.nonstream.works")
|
||||
def test_basic_messaging_non_streaming_anthropic(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a reply.
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ the matrix builder still sees three rows for this (feature, provider).
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from claude_code._basic_messaging import run_basic_messaging_cell
|
||||
|
||||
# Per-model aliases registered in the LiteLLM proxy's routing config to
|
||||
|
|
@ -38,6 +40,7 @@ AZURE_MODELS = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.azure_foundry.basic.nonstream.works")
|
||||
def test_basic_messaging_non_streaming_azure(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a reply.
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ the matrix builder still sees three rows for this (feature, provider).
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from claude_code._basic_messaging import run_basic_messaging_cell
|
||||
|
||||
# Per-model aliases registered in the LiteLLM proxy's routing config to
|
||||
|
|
@ -33,6 +35,7 @@ BEDROCK_CONVERSE_MODELS = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_converse.basic.nonstream.works")
|
||||
def test_basic_messaging_non_streaming_bedrock_converse(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a reply."""
|
||||
run_basic_messaging_cell(
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ the matrix builder still sees three rows for this (feature, provider).
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from claude_code._basic_messaging import run_basic_messaging_cell
|
||||
|
||||
# Per-model aliases registered in the LiteLLM proxy's routing config to
|
||||
|
|
@ -33,6 +35,7 @@ BEDROCK_INVOKE_MODELS = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_invoke.basic.nonstream.works")
|
||||
def test_basic_messaging_non_streaming_bedrock_invoke(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a reply."""
|
||||
run_basic_messaging_cell(
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ the matrix builder still sees three rows for this (feature, provider).
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from claude_code._basic_messaging import run_basic_messaging_cell
|
||||
|
||||
# Per-model aliases registered in the LiteLLM proxy's routing config to
|
||||
|
|
@ -33,6 +35,7 @@ VERTEX_AI_MODELS = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.vertex.basic.nonstream.works")
|
||||
def test_basic_messaging_non_streaming_vertex_ai(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a reply."""
|
||||
run_basic_messaging_cell(
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ sees three rows for this (feature, provider).
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from claude_code._basic_messaging import run_basic_messaging_cell
|
||||
|
||||
ANTHROPIC_MODELS = [
|
||||
|
|
@ -34,6 +36,7 @@ ANTHROPIC_MODELS = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.anthropic.basic.stream.works")
|
||||
def test_basic_messaging_streaming_anthropic(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
non-empty streamed reply (one row per Claude tier).
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from claude_code._basic_messaging import run_basic_messaging_cell
|
||||
|
||||
AZURE_MODELS = [
|
||||
|
|
@ -28,6 +30,7 @@ AZURE_MODELS = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.azure_foundry.basic.stream.works")
|
||||
def test_basic_messaging_streaming_azure(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
non-empty streamed reply (one row per Claude tier).
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from claude_code._basic_messaging import run_basic_messaging_cell
|
||||
|
||||
BEDROCK_CONVERSE_MODELS = [
|
||||
|
|
@ -24,6 +26,7 @@ BEDROCK_CONVERSE_MODELS = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_converse.basic.stream.works")
|
||||
def test_basic_messaging_streaming_bedrock_converse(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
non-empty streamed reply (one row per Claude tier).
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from claude_code._basic_messaging import run_basic_messaging_cell
|
||||
|
||||
BEDROCK_INVOKE_MODELS = [
|
||||
|
|
@ -24,6 +26,7 @@ BEDROCK_INVOKE_MODELS = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_invoke.basic.stream.works")
|
||||
def test_basic_messaging_streaming_bedrock_invoke(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
non-empty streamed reply (one row per Claude tier).
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from claude_code._basic_messaging import run_basic_messaging_cell
|
||||
|
||||
VERTEX_AI_MODELS = [
|
||||
|
|
@ -24,6 +26,7 @@ VERTEX_AI_MODELS = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.vertex.basic.stream.works")
|
||||
def test_basic_messaging_streaming_vertex_ai(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
non-empty streamed reply (one row per Claude tier).
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ ANTHROPIC_MODELS = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.anthropic.count_tokens.nonstream.works")
|
||||
def test_count_tokens_anthropic(compat_result):
|
||||
"""Probe `/v1/messages/count_tokens` for each Anthropic tier and
|
||||
assert the response shape."""
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ AZURE_MODELS = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.azure_foundry.count_tokens.nonstream.works")
|
||||
def test_count_tokens_azure(compat_result):
|
||||
"""Probe `/v1/messages/count_tokens` for each Azure (Microsoft Foundry) tier and
|
||||
assert the response shape."""
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ BEDROCK_CONVERSE_MODELS = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_converse.count_tokens.nonstream.works")
|
||||
def test_count_tokens_bedrock_converse(compat_result):
|
||||
"""Probe `/v1/messages/count_tokens` for each Bedrock (Converse) tier and
|
||||
assert the response shape."""
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ BEDROCK_INVOKE_MODELS = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_invoke.count_tokens.nonstream.works")
|
||||
def test_count_tokens_bedrock_invoke(compat_result):
|
||||
"""Probe `/v1/messages/count_tokens` for each Bedrock (Invoke) tier and
|
||||
assert the response shape."""
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ VERTEX_AI_MODELS = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.vertex.count_tokens.nonstream.works")
|
||||
def test_count_tokens_vertex_ai(compat_result):
|
||||
"""Probe `/v1/messages/count_tokens` for each Vertex AI tier and
|
||||
assert the response shape."""
|
||||
|
|
|
|||
|
|
@ -155,6 +155,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str:
|
|||
return preamble + "".join(pad_lines) + closing
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.anthropic.long_context.nonstream.works")
|
||||
def test_long_context_1m_anthropic(compat_result):
|
||||
"""Drive the `claude` CLI with a ~210k-token prompt and the
|
||||
`context-1m-2025-08-07` beta header; assert no 400 / 413 and a
|
||||
|
|
|
|||
|
|
@ -155,6 +155,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str:
|
|||
return preamble + "".join(pad_lines) + closing
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.azure_foundry.long_context.nonstream.works")
|
||||
def test_long_context_1m_azure(compat_result):
|
||||
"""Drive the `claude` CLI (Azure (Microsoft Foundry)) with a ~210k-token prompt and the
|
||||
`context-1m-2025-08-07` beta header; assert no 400 / 413 and a
|
||||
|
|
|
|||
|
|
@ -155,6 +155,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str:
|
|||
return preamble + "".join(pad_lines) + closing
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_converse.long_context.nonstream.works")
|
||||
def test_long_context_1m_bedrock_converse(compat_result):
|
||||
"""Drive the `claude` CLI (Bedrock (Converse)) with a ~210k-token prompt and the
|
||||
`context-1m-2025-08-07` beta header; assert no 400 / 413 and a
|
||||
|
|
|
|||
|
|
@ -155,6 +155,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str:
|
|||
return preamble + "".join(pad_lines) + closing
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_invoke.long_context.nonstream.works")
|
||||
def test_long_context_1m_bedrock_invoke(compat_result):
|
||||
"""Drive the `claude` CLI (Bedrock (Invoke)) with a ~210k-token prompt and the
|
||||
`context-1m-2025-08-07` beta header; assert no 400 / 413 and a
|
||||
|
|
|
|||
|
|
@ -155,6 +155,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str:
|
|||
return preamble + "".join(pad_lines) + closing
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.vertex.long_context.nonstream.works")
|
||||
def test_long_context_1m_vertex_ai(compat_result):
|
||||
"""Drive the `claude` CLI (Vertex AI) with a ~210k-token prompt and the
|
||||
`context-1m-2025-08-07` beta header; assert no 400 / 413 and a
|
||||
|
|
|
|||
|
|
@ -108,6 +108,7 @@ def _build_minimal_pdf(marker: str) -> bytes:
|
|||
return bytes(out)
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.anthropic.pdf.nonstream.works")
|
||||
def test_pdf_input_anthropic(compat_result, tmp_path):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy with a PDF
|
||||
attached via the Read tool and assert the reply references it."""
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ def _build_minimal_pdf(marker: str) -> bytes:
|
|||
return bytes(out)
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.azure_foundry.pdf.nonstream.works")
|
||||
def test_pdf_input_azure(compat_result, tmp_path):
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
|
|
|
|||
|
|
@ -91,6 +91,7 @@ def _build_minimal_pdf(marker: str) -> bytes:
|
|||
return bytes(out)
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_converse.pdf.nonstream.works")
|
||||
def test_pdf_input_bedrock_converse(compat_result, tmp_path):
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
|
|
|
|||
|
|
@ -90,6 +90,7 @@ def _build_minimal_pdf(marker: str) -> bytes:
|
|||
return bytes(out)
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_invoke.pdf.nonstream.works")
|
||||
def test_pdf_input_bedrock_invoke(compat_result, tmp_path):
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ def _build_minimal_pdf(marker: str) -> bytes:
|
|||
return bytes(out)
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.vertex.pdf.nonstream.works")
|
||||
def test_pdf_input_vertex_ai(compat_result, tmp_path):
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
|
|
|
|||
|
|
@ -65,6 +65,7 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.anthropic.prompt_cache_1h.nonstream.works")
|
||||
def test_prompt_caching_1h_anthropic(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy with the 1h
|
||||
TTL opt-in env var set, and assert the upstream usage block
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.azure_foundry.prompt_cache_1h.nonstream.works")
|
||||
def test_prompt_caching_1h_azure(compat_result):
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_converse.prompt_cache_1h.nonstream.works")
|
||||
def test_prompt_caching_1h_bedrock_converse(compat_result):
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_invoke.prompt_cache_1h.nonstream.works")
|
||||
def test_prompt_caching_1h_bedrock_invoke(compat_result):
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.vertex.prompt_cache_1h.nonstream.works")
|
||||
def test_prompt_caching_1h_vertex_ai(compat_result):
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.anthropic.prompt_cache_5m.nonstream.works")
|
||||
def test_prompt_caching_5m_anthropic(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
|
||||
upstream usage block surfaces a non-zero cache token count."""
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.azure_foundry.prompt_cache_5m.nonstream.works")
|
||||
def test_prompt_caching_5m_azure(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
|
||||
upstream usage block surfaces a non-zero cache token count."""
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_converse.prompt_cache_5m.nonstream.works")
|
||||
def test_prompt_caching_5m_bedrock_converse(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
|
||||
upstream usage block surfaces a non-zero cache token count."""
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_invoke.prompt_cache_5m.nonstream.works")
|
||||
def test_prompt_caching_5m_bedrock_invoke(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
|
||||
upstream usage block surfaces a non-zero cache token count."""
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.vertex.prompt_cache_5m.nonstream.works")
|
||||
def test_prompt_caching_5m_vertex_ai(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
|
||||
upstream usage block surfaces a non-zero cache token count."""
|
||||
|
|
|
|||
|
|
@ -152,6 +152,7 @@ def _validate_against_schema(
|
|||
return None
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.anthropic.structured_output.nonstream.works")
|
||||
def test_structured_outputs_anthropic(compat_result):
|
||||
"""Drive `claude --json-schema ...` against the LiteLLM proxy and
|
||||
assert the trailing `result` event contains a schema-conforming
|
||||
|
|
|
|||
|
|
@ -152,6 +152,7 @@ def _validate_against_schema(
|
|||
return None
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.azure_foundry.structured_output.nonstream.works")
|
||||
def test_structured_outputs_azure(compat_result):
|
||||
"""Drive `claude --json-schema ...` against the LiteLLM proxy and
|
||||
assert the trailing `result` event contains a schema-conforming
|
||||
|
|
|
|||
|
|
@ -152,6 +152,7 @@ def _validate_against_schema(
|
|||
return None
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_converse.structured_output.nonstream.works")
|
||||
def test_structured_outputs_bedrock_converse(compat_result):
|
||||
"""Drive `claude --json-schema ...` against the LiteLLM proxy and
|
||||
assert the trailing `result` event contains a schema-conforming
|
||||
|
|
|
|||
|
|
@ -152,6 +152,7 @@ def _validate_against_schema(
|
|||
return None
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_invoke.structured_output.nonstream.works")
|
||||
def test_structured_outputs_bedrock_invoke(compat_result):
|
||||
"""Drive `claude --json-schema ...` against the LiteLLM proxy and
|
||||
assert the trailing `result` event contains a schema-conforming
|
||||
|
|
|
|||
|
|
@ -152,6 +152,7 @@ def _validate_against_schema(
|
|||
return None
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.vertex.structured_output.nonstream.works")
|
||||
def test_structured_outputs_vertex_ai(compat_result):
|
||||
"""Drive `claude --json-schema ...` against the LiteLLM proxy and
|
||||
assert the trailing `result` event contains a schema-conforming
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.anthropic.thinking.nonstream.works")
|
||||
def test_thinking_anthropic(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
|
||||
enabled and assert a `thinking` content block was emitted."""
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.azure_foundry.thinking.nonstream.works")
|
||||
def test_thinking_azure(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
|
||||
enabled and assert a `thinking` content block was emitted."""
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_converse.thinking.nonstream.works")
|
||||
def test_thinking_bedrock_converse(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
|
||||
enabled and assert a `thinking` content block was emitted."""
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_invoke.thinking.nonstream.works")
|
||||
def test_thinking_bedrock_invoke(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
|
||||
enabled and assert a `thinking` content block was emitted."""
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.vertex.thinking.nonstream.works")
|
||||
def test_thinking_vertex_ai(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
|
||||
enabled and assert a `thinking` content block was emitted."""
|
||||
|
|
|
|||
|
|
@ -90,6 +90,7 @@ def _has_block_type(
|
|||
return False
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.anthropic.thinking_with_tool_use.nonstream.works")
|
||||
def test_thinking_with_tool_use_anthropic(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
|
||||
enabled and tool use, and assert both `thinking` and `tool_use`
|
||||
|
|
|
|||
|
|
@ -71,6 +71,7 @@ def _has_block_type(
|
|||
return False
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.azure_foundry.thinking_with_tool_use.nonstream.works")
|
||||
def test_thinking_with_tool_use_azure(compat_result):
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ def _has_block_type(
|
|||
return False
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_converse.thinking_with_tool_use.nonstream.works")
|
||||
def test_thinking_with_tool_use_bedrock_converse(compat_result):
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ def _has_block_type(
|
|||
return False
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_invoke.thinking_with_tool_use.nonstream.works")
|
||||
def test_thinking_with_tool_use_bedrock_invoke(compat_result):
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ def _has_block_type(
|
|||
return False
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.vertex.thinking_with_tool_use.nonstream.works")
|
||||
def test_thinking_with_tool_use_vertex_ai(compat_result):
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ ANTHROPIC_MODELS = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.anthropic.tool_search.nonstream.works")
|
||||
def test_tool_search_anthropic(compat_result):
|
||||
"""Probe `/v1/messages` with a `tool_search_tool_regex_20251119`
|
||||
tool and assert the proxy + upstream accept it for every Anthropic
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ AZURE_MODELS = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.azure_foundry.tool_search.nonstream.works")
|
||||
def test_tool_search_azure(compat_result):
|
||||
"""Probe `/v1/messages` with a `tool_search_tool_regex_20251119`
|
||||
tool and assert the proxy + upstream accept it for every Azure (Microsoft Foundry)
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ BEDROCK_CONVERSE_MODELS = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_converse.tool_search.nonstream.works")
|
||||
def test_tool_search_bedrock_converse(compat_result):
|
||||
"""Probe `/v1/messages` with a `tool_search_tool_regex_20251119`
|
||||
tool and assert the proxy + upstream accept it for every Bedrock (Converse)
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ BEDROCK_INVOKE_MODELS = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_invoke.tool_search.nonstream.works")
|
||||
def test_tool_search_bedrock_invoke(compat_result):
|
||||
"""Probe `/v1/messages` with a `tool_search_tool_regex_20251119`
|
||||
tool and assert the proxy + upstream accept it for every Bedrock (Invoke)
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ VERTEX_AI_MODELS = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.vertex.tool_search.nonstream.works")
|
||||
def test_tool_search_vertex_ai(compat_result):
|
||||
"""Probe `/v1/messages` with a `tool_search_tool_regex_20251119`
|
||||
tool and assert the proxy + upstream accept it for every Vertex AI
|
||||
|
|
|
|||
|
|
@ -73,6 +73,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.anthropic.tool_use.nonstream.works")
|
||||
def test_tool_use_anthropic(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
tool call was emitted on the wire."""
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.azure_foundry.tool_use.nonstream.works")
|
||||
def test_tool_use_azure(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
tool call was emitted on the wire."""
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_converse.tool_use.nonstream.works")
|
||||
def test_tool_use_bedrock_converse(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
tool call was emitted on the wire."""
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_invoke.tool_use.nonstream.works")
|
||||
def test_tool_use_bedrock_invoke(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
tool call was emitted on the wire."""
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.vertex.tool_use.nonstream.works")
|
||||
def test_tool_use_vertex_ai(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
tool call was emitted on the wire."""
|
||||
|
|
|
|||
|
|
@ -99,6 +99,7 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.anthropic.tool_use.stream.works")
|
||||
def test_tool_use_streaming_anthropic(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
|
||||
proxy preserves fine-grained tool streaming end-to-end."""
|
||||
|
|
|
|||
|
|
@ -84,6 +84,7 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.azure_foundry.tool_use.stream.works")
|
||||
def test_tool_use_streaming_azure(compat_result):
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
|
|
|
|||
|
|
@ -90,6 +90,7 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_converse.tool_use.stream.works")
|
||||
def test_tool_use_streaming_bedrock_converse(compat_result):
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
|
|
|
|||
|
|
@ -88,6 +88,7 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_invoke.tool_use.stream.works")
|
||||
def test_tool_use_streaming_bedrock_invoke(compat_result):
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.vertex.tool_use.stream.works")
|
||||
def test_tool_use_streaming_vertex_ai(compat_result):
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ def _build_stdin_input() -> str:
|
|||
return json.dumps(user_event) + "\n"
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.anthropic.vision.nonstream.works")
|
||||
def test_vision_anthropic(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy with an image
|
||||
attached via stream-json input and assert a non-empty reply."""
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ def _build_stdin_input() -> str:
|
|||
return json.dumps(user_event) + "\n"
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.azure_foundry.vision.nonstream.works")
|
||||
def test_vision_azure(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy with an image
|
||||
attached via stream-json input and assert a non-empty reply."""
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ def _build_stdin_input() -> str:
|
|||
return json.dumps(user_event) + "\n"
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_converse.vision.nonstream.works")
|
||||
def test_vision_bedrock_converse(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy with an image
|
||||
attached via stream-json input and assert a non-empty reply."""
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ def _build_stdin_input() -> str:
|
|||
return json.dumps(user_event) + "\n"
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_invoke.vision.nonstream.works")
|
||||
def test_vision_bedrock_invoke(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy with an image
|
||||
attached via stream-json input and assert a non-empty reply."""
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ def _build_stdin_input() -> str:
|
|||
return json.dumps(user_event) + "\n"
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.vertex.vision.nonstream.works")
|
||||
def test_vision_vertex_ai(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy with an image
|
||||
attached via stream-json input and assert a non-empty reply."""
|
||||
|
|
|
|||
|
|
@ -86,6 +86,7 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.anthropic.web_search.nonstream.works")
|
||||
def test_web_search_anthropic(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
|
||||
upstream emitted a `tool_use` block calling `WebSearch`, proving
|
||||
|
|
|
|||
|
|
@ -86,6 +86,7 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.azure_foundry.web_search.nonstream.works")
|
||||
def test_web_search_azure(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
|
||||
upstream emitted a `tool_use` block calling `WebSearch`, proving
|
||||
|
|
|
|||
|
|
@ -86,6 +86,7 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_converse.web_search.nonstream.works")
|
||||
def test_web_search_bedrock_converse(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
|
||||
upstream emitted a `tool_use` block calling `WebSearch`, proving
|
||||
|
|
|
|||
|
|
@ -86,6 +86,7 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.bedrock_invoke.web_search.nonstream.works")
|
||||
def test_web_search_bedrock_invoke(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
|
||||
upstream emitted a `tool_use` block calling `WebSearch`, proving
|
||||
|
|
|
|||
|
|
@ -86,6 +86,7 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
@pytest.mark.covers("llm.messages.vertex.web_search.nonstream.works")
|
||||
def test_web_search_vertex_ai(compat_result):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
|
||||
upstream emitted a `tool_use` block calling `WebSearch`, proving
|
||||
|
|
|
|||
|
|
@ -39,8 +39,76 @@
|
|||
- {id: llm.messages.anthropic.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Vision via Messages API"}
|
||||
- {id: llm.messages.anthropic.prompt_cache_5m.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Prompt caching via Messages API"}
|
||||
- {id: llm.messages.anthropic.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Extended thinking via Messages API"}
|
||||
- {id: llm.messages.anthropic.thinking_with_tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: thinking_with_tool_use, streaming: nonstream, assertions: [works], source: "claude_code/thinking_with_tool_use/test_anthropic.py", rationale: "Claude Code compat matrix: thinking interleaved with tool use"}
|
||||
- {id: llm.messages.anthropic.pdf.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: pdf, streaming: nonstream, assertions: [works], source: "claude_code/pdf_input/test_anthropic.py", rationale: "Claude Code compat matrix: PDF document input"}
|
||||
- {id: llm.messages.anthropic.prompt_cache_1h.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: prompt_cache_1h, streaming: nonstream, assertions: [works], source: "claude_code/prompt_caching_1h/test_anthropic.py", rationale: "Claude Code compat matrix: 1h-TTL cache_control survives translation"}
|
||||
- {id: llm.messages.anthropic.web_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: web_search, streaming: nonstream, assertions: [works], source: "claude_code/web_search/test_anthropic.py", rationale: "Claude Code compat matrix: server-side web_search tool"}
|
||||
- {id: llm.messages.anthropic.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: structured_output, streaming: nonstream, assertions: [works], source: "claude_code/structured_outputs/test_anthropic.py", rationale: "Claude Code compat matrix: --json-schema StructuredOutput tool round-trip"}
|
||||
- {id: llm.messages.anthropic.count_tokens.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: count_tokens, streaming: nonstream, assertions: [works], source: "claude_code/count_tokens/test_anthropic.py", rationale: "Claude Code compat matrix: count_tokens endpoint probe"}
|
||||
- {id: llm.messages.anthropic.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: tool_search, streaming: nonstream, assertions: [works], source: "claude_code/tool_search/test_anthropic.py", rationale: "Claude Code compat matrix: tool_search discovery tool + beta header translation"}
|
||||
- {id: llm.messages.anthropic.long_context.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: long_context, streaming: nonstream, assertions: [works], source: "claude_code/long_context_1m/test_anthropic.py", rationale: "Claude Code compat matrix: context-1m beta header pass-through"}
|
||||
- {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Flagged Claude 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (#32578/#32831/#32882)", fail_before_fix: proven}
|
||||
- {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Claude <= 4.7 rejects role system inside messages; unflagged models must hoist reminders into top-level system or every Claude Code session 400s (#32831)", fail_before_fix: proven}
|
||||
- {id: llm.messages.bedrock_invoke.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: basic, streaming: nonstream, assertions: [works], source: "claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py", rationale: "Claude Code compat matrix: basic messaging"}
|
||||
- {id: llm.messages.bedrock_invoke.basic.stream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: basic, streaming: stream, assertions: [works], source: "claude_code/basic_messaging_streaming/test_bedrock_invoke.py", rationale: "Claude Code compat matrix: streamed messaging stays streamed through the proxy"}
|
||||
- {id: llm.messages.bedrock_invoke.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: tool_use, streaming: nonstream, assertions: [works], source: "claude_code/tool_use/test_bedrock_invoke.py", rationale: "Claude Code compat matrix: tool use round-trip"}
|
||||
- {id: llm.messages.bedrock_invoke.tool_use.stream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: tool_use, streaming: stream, assertions: [works], source: "claude_code/tool_use_streaming/test_bedrock_invoke.py", rationale: "Claude Code compat matrix: fine-grained streaming tool use"}
|
||||
- {id: llm.messages.bedrock_invoke.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "claude_code/prompt_caching_5m/test_bedrock_invoke.py", rationale: "Claude Code compat matrix: 5m-TTL cache_control survives translation"}
|
||||
- {id: llm.messages.bedrock_invoke.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: vision, streaming: nonstream, assertions: [works], source: "claude_code/vision/test_bedrock_invoke.py", rationale: "Claude Code compat matrix: image input"}
|
||||
- {id: llm.messages.bedrock_invoke.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: thinking, streaming: nonstream, assertions: [works], source: "claude_code/thinking/test_bedrock_invoke.py", rationale: "Claude Code compat matrix: manual + adaptive thinking"}
|
||||
- {id: llm.messages.bedrock_invoke.thinking_with_tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: thinking_with_tool_use, streaming: nonstream, assertions: [works], source: "claude_code/thinking_with_tool_use/test_bedrock_invoke.py", rationale: "Claude Code compat matrix: thinking interleaved with tool use"}
|
||||
- {id: llm.messages.bedrock_invoke.pdf.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: pdf, streaming: nonstream, assertions: [works], source: "claude_code/pdf_input/test_bedrock_invoke.py", rationale: "Claude Code compat matrix: PDF document input"}
|
||||
- {id: llm.messages.bedrock_invoke.prompt_cache_1h.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: prompt_cache_1h, streaming: nonstream, assertions: [works], source: "claude_code/prompt_caching_1h/test_bedrock_invoke.py", rationale: "Claude Code compat matrix: 1h-TTL cache_control survives translation"}
|
||||
- {id: llm.messages.bedrock_invoke.web_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: web_search, streaming: nonstream, assertions: [works], source: "claude_code/web_search/test_bedrock_invoke.py", rationale: "Claude Code compat matrix: server-side web_search tool"}
|
||||
- {id: llm.messages.bedrock_invoke.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: structured_output, streaming: nonstream, assertions: [works], source: "claude_code/structured_outputs/test_bedrock_invoke.py", rationale: "Claude Code compat matrix: --json-schema StructuredOutput tool round-trip"}
|
||||
- {id: llm.messages.bedrock_invoke.count_tokens.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: count_tokens, streaming: nonstream, assertions: [works], source: "claude_code/count_tokens/test_bedrock_invoke.py", rationale: "Claude Code compat matrix: count_tokens endpoint probe"}
|
||||
- {id: llm.messages.bedrock_invoke.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: tool_search, streaming: nonstream, assertions: [works], source: "claude_code/tool_search/test_bedrock_invoke.py", rationale: "Claude Code compat matrix: tool_search discovery tool + beta header translation"}
|
||||
- {id: llm.messages.bedrock_invoke.long_context.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: long_context, streaming: nonstream, assertions: [works], source: "claude_code/long_context_1m/test_bedrock_invoke.py", rationale: "Claude Code compat matrix: context-1m beta header pass-through"}
|
||||
- {id: llm.messages.bedrock_converse.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "claude_code/basic_messaging_non_streaming/test_bedrock_converse.py", rationale: "Claude Code compat matrix: basic messaging"}
|
||||
- {id: llm.messages.bedrock_converse.basic.stream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_converse, capability: basic, streaming: stream, assertions: [works], source: "claude_code/basic_messaging_streaming/test_bedrock_converse.py", rationale: "Claude Code compat matrix: streamed messaging stays streamed through the proxy"}
|
||||
- {id: llm.messages.bedrock_converse.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_converse, capability: tool_use, streaming: nonstream, assertions: [works], source: "claude_code/tool_use/test_bedrock_converse.py", rationale: "Claude Code compat matrix: tool use round-trip"}
|
||||
- {id: llm.messages.bedrock_converse.tool_use.stream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_converse, capability: tool_use, streaming: stream, assertions: [works], source: "claude_code/tool_use_streaming/test_bedrock_converse.py", rationale: "Claude Code compat matrix: fine-grained streaming tool use"}
|
||||
- {id: llm.messages.bedrock_converse.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "claude_code/prompt_caching_5m/test_bedrock_converse.py", rationale: "Claude Code compat matrix: 5m-TTL cache_control survives translation"}
|
||||
- {id: llm.messages.bedrock_converse.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_converse, capability: vision, streaming: nonstream, assertions: [works], source: "claude_code/vision/test_bedrock_converse.py", rationale: "Claude Code compat matrix: image input"}
|
||||
- {id: llm.messages.bedrock_converse.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: thinking, streaming: nonstream, assertions: [works], source: "claude_code/thinking/test_bedrock_converse.py", rationale: "Claude Code compat matrix: manual + adaptive thinking"}
|
||||
- {id: llm.messages.bedrock_converse.thinking_with_tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: thinking_with_tool_use, streaming: nonstream, assertions: [works], source: "claude_code/thinking_with_tool_use/test_bedrock_converse.py", rationale: "Claude Code compat matrix: thinking interleaved with tool use"}
|
||||
- {id: llm.messages.bedrock_converse.pdf.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: pdf, streaming: nonstream, assertions: [works], source: "claude_code/pdf_input/test_bedrock_converse.py", rationale: "Claude Code compat matrix: PDF document input"}
|
||||
- {id: llm.messages.bedrock_converse.prompt_cache_1h.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: prompt_cache_1h, streaming: nonstream, assertions: [works], source: "claude_code/prompt_caching_1h/test_bedrock_converse.py", rationale: "Claude Code compat matrix: 1h-TTL cache_control survives translation"}
|
||||
- {id: llm.messages.bedrock_converse.web_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: web_search, streaming: nonstream, assertions: [works], source: "claude_code/web_search/test_bedrock_converse.py", rationale: "Claude Code compat matrix: server-side web_search tool"}
|
||||
- {id: llm.messages.bedrock_converse.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: structured_output, streaming: nonstream, assertions: [works], source: "claude_code/structured_outputs/test_bedrock_converse.py", rationale: "Claude Code compat matrix: --json-schema StructuredOutput tool round-trip"}
|
||||
- {id: llm.messages.bedrock_converse.count_tokens.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: count_tokens, streaming: nonstream, assertions: [works], source: "claude_code/count_tokens/test_bedrock_converse.py", rationale: "Claude Code compat matrix: count_tokens endpoint probe"}
|
||||
- {id: llm.messages.bedrock_converse.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: tool_search, streaming: nonstream, assertions: [works], source: "claude_code/tool_search/test_bedrock_converse.py", rationale: "Claude Code compat matrix: tool_search discovery tool + beta header translation"}
|
||||
- {id: llm.messages.bedrock_converse.long_context.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: long_context, streaming: nonstream, assertions: [works], source: "claude_code/long_context_1m/test_bedrock_converse.py", rationale: "Claude Code compat matrix: context-1m beta header pass-through"}
|
||||
- {id: llm.messages.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "claude_code/basic_messaging_non_streaming/test_vertex_ai.py", rationale: "Claude Code compat matrix: basic messaging"}
|
||||
- {id: llm.messages.vertex.basic.stream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: basic, streaming: stream, assertions: [works], source: "claude_code/basic_messaging_streaming/test_vertex_ai.py", rationale: "Claude Code compat matrix: streamed messaging stays streamed through the proxy"}
|
||||
- {id: llm.messages.vertex.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: tool_use, streaming: nonstream, assertions: [works], source: "claude_code/tool_use/test_vertex_ai.py", rationale: "Claude Code compat matrix: tool use round-trip"}
|
||||
- {id: llm.messages.vertex.tool_use.stream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: tool_use, streaming: stream, assertions: [works], source: "claude_code/tool_use_streaming/test_vertex_ai.py", rationale: "Claude Code compat matrix: fine-grained streaming tool use"}
|
||||
- {id: llm.messages.vertex.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "claude_code/prompt_caching_5m/test_vertex_ai.py", rationale: "Claude Code compat matrix: 5m-TTL cache_control survives translation"}
|
||||
- {id: llm.messages.vertex.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: vision, streaming: nonstream, assertions: [works], source: "claude_code/vision/test_vertex_ai.py", rationale: "Claude Code compat matrix: image input"}
|
||||
- {id: llm.messages.vertex.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: thinking, streaming: nonstream, assertions: [works], source: "claude_code/thinking/test_vertex_ai.py", rationale: "Claude Code compat matrix: manual + adaptive thinking"}
|
||||
- {id: llm.messages.vertex.thinking_with_tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: thinking_with_tool_use, streaming: nonstream, assertions: [works], source: "claude_code/thinking_with_tool_use/test_vertex_ai.py", rationale: "Claude Code compat matrix: thinking interleaved with tool use"}
|
||||
- {id: llm.messages.vertex.pdf.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: pdf, streaming: nonstream, assertions: [works], source: "claude_code/pdf_input/test_vertex_ai.py", rationale: "Claude Code compat matrix: PDF document input"}
|
||||
- {id: llm.messages.vertex.prompt_cache_1h.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: prompt_cache_1h, streaming: nonstream, assertions: [works], source: "claude_code/prompt_caching_1h/test_vertex_ai.py", rationale: "Claude Code compat matrix: 1h-TTL cache_control survives translation"}
|
||||
- {id: llm.messages.vertex.web_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: web_search, streaming: nonstream, assertions: [works], source: "claude_code/web_search/test_vertex_ai.py", rationale: "Claude Code compat matrix: server-side web_search tool"}
|
||||
- {id: llm.messages.vertex.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: structured_output, streaming: nonstream, assertions: [works], source: "claude_code/structured_outputs/test_vertex_ai.py", rationale: "Claude Code compat matrix: --json-schema StructuredOutput tool round-trip"}
|
||||
- {id: llm.messages.vertex.count_tokens.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: count_tokens, streaming: nonstream, assertions: [works], source: "claude_code/count_tokens/test_vertex_ai.py", rationale: "Claude Code compat matrix: count_tokens endpoint probe"}
|
||||
- {id: llm.messages.vertex.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: tool_search, streaming: nonstream, assertions: [works], source: "claude_code/tool_search/test_vertex_ai.py", rationale: "Claude Code compat matrix: tool_search discovery tool + beta header translation"}
|
||||
- {id: llm.messages.vertex.long_context.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: long_context, streaming: nonstream, assertions: [works], source: "claude_code/long_context_1m/test_vertex_ai.py", rationale: "Claude Code compat matrix: context-1m beta header pass-through"}
|
||||
- {id: llm.messages.azure_foundry.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: basic, streaming: nonstream, assertions: [works], source: "claude_code/basic_messaging_non_streaming/test_azure.py", rationale: "Claude Code compat matrix: basic messaging"}
|
||||
- {id: llm.messages.azure_foundry.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: basic, streaming: stream, assertions: [works], source: "claude_code/basic_messaging_streaming/test_azure.py", rationale: "Claude Code compat matrix: streamed messaging stays streamed through the proxy"}
|
||||
- {id: llm.messages.azure_foundry.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: tool_use, streaming: nonstream, assertions: [works], source: "claude_code/tool_use/test_azure.py", rationale: "Claude Code compat matrix: tool use round-trip"}
|
||||
- {id: llm.messages.azure_foundry.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: tool_use, streaming: stream, assertions: [works], source: "claude_code/tool_use_streaming/test_azure.py", rationale: "Claude Code compat matrix: fine-grained streaming tool use"}
|
||||
- {id: llm.messages.azure_foundry.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "claude_code/prompt_caching_5m/test_azure.py", rationale: "Claude Code compat matrix: 5m-TTL cache_control survives translation"}
|
||||
- {id: llm.messages.azure_foundry.vision.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: vision, streaming: nonstream, assertions: [works], source: "claude_code/vision/test_azure.py", rationale: "Claude Code compat matrix: image input"}
|
||||
- {id: llm.messages.azure_foundry.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: thinking, streaming: nonstream, assertions: [works], source: "claude_code/thinking/test_azure.py", rationale: "Claude Code compat matrix: manual + adaptive thinking"}
|
||||
- {id: llm.messages.azure_foundry.thinking_with_tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: thinking_with_tool_use, streaming: nonstream, assertions: [works], source: "claude_code/thinking_with_tool_use/test_azure.py", rationale: "Claude Code compat matrix: thinking interleaved with tool use"}
|
||||
- {id: llm.messages.azure_foundry.pdf.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: pdf, streaming: nonstream, assertions: [works], source: "claude_code/pdf_input/test_azure.py", rationale: "Claude Code compat matrix: PDF document input"}
|
||||
- {id: llm.messages.azure_foundry.prompt_cache_1h.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: prompt_cache_1h, streaming: nonstream, assertions: [works], source: "claude_code/prompt_caching_1h/test_azure.py", rationale: "Claude Code compat matrix: 1h-TTL cache_control survives translation"}
|
||||
- {id: llm.messages.azure_foundry.web_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: web_search, streaming: nonstream, assertions: [works], source: "claude_code/web_search/test_azure.py", rationale: "Claude Code compat matrix: server-side web_search tool"}
|
||||
- {id: llm.messages.azure_foundry.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: structured_output, streaming: nonstream, assertions: [works], source: "claude_code/structured_outputs/test_azure.py", rationale: "Claude Code compat matrix: --json-schema StructuredOutput tool round-trip"}
|
||||
- {id: llm.messages.azure_foundry.count_tokens.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: count_tokens, streaming: nonstream, assertions: [works], source: "claude_code/count_tokens/test_azure.py", rationale: "Claude Code compat matrix: count_tokens endpoint probe"}
|
||||
- {id: llm.messages.azure_foundry.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: tool_search, streaming: nonstream, assertions: [works], source: "claude_code/tool_search/test_azure.py", rationale: "Claude Code compat matrix: tool_search discovery tool + beta header translation"}
|
||||
- {id: llm.messages.azure_foundry.long_context.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: long_context, streaming: nonstream, assertions: [works], source: "claude_code/long_context_1m/test_azure.py", rationale: "Claude Code compat matrix: context-1m beta header pass-through"}
|
||||
- {id: llm.responses.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Core endpoint; OpenAI Responses native"}
|
||||
- {id: llm.responses.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Streaming via /v1/responses"}
|
||||
- {id: llm.responses.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "response_api_endpoints/endpoints.py:26", rationale: "Cost logged on responses"}
|
||||
|
|
|
|||
|
|
@ -54,13 +54,20 @@ LlmRoute = Literal[
|
|||
|
||||
LlmCapability = Literal[
|
||||
"basic",
|
||||
"count_tokens",
|
||||
"long_context",
|
||||
"mid_conversation_system",
|
||||
"pdf",
|
||||
"prompt_cache_1h",
|
||||
"prompt_cache_5m",
|
||||
"service_tier",
|
||||
"structured_output",
|
||||
"thinking",
|
||||
"thinking_with_tool_use",
|
||||
"tool_search",
|
||||
"tool_use",
|
||||
"vision",
|
||||
"web_search",
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue