litellm/tests/e2e/claude_code/manifest.yaml

# Claude Code Compatibility Matrix — feature manifest.
#
# Defines the row order of the matrix and maps each feature_id to its
# human-readable display name. Adding a new feature to the matrix is a
# three-step change:
#   1. Append an entry to `features:` below.
#   2. Create a directory `tests/e2e/claude_code/<feature_id>/`.
#   3. Add per-provider test files inside that directory.
#
# `feature_id` MUST match the directory name on disk; the test harness
# infers (feature, provider) for each test from its file path.

schema_version: "1"

# Provider column order in the rendered matrix. The first five are
# the v0 Claude columns; the GPT-5.6 (Sol / Terra / Luna) columns
# added 2026-07 follow them. `vertex_ai_gpt` is a static
# not_applicable column: GCP does not offer the closed-weight GPT-5.6
# family (Model Garden carries only the open-weight gpt-oss MaaS
# models), and the column documents that gap explicitly. GPT columns
# currently back the two basic_messaging rows plus tool_use and
# tool_use_streaming; other rows render not_tested for them.
providers:
  - anthropic
  - bedrock_invoke
  - bedrock_converse
  - vertex_ai
  - azure
  - openai
  - azure_openai
  - bedrock_mantle
  - vertex_ai_gpt

# Feature row order.
features:
  - id: basic_messaging_non_streaming
    name: Basic messaging (non-streaming)
  - id: basic_messaging_streaming
    name: Basic messaging (streaming)
  - id: tool_use
    name: Tool use
  - id: prompt_caching_5m
    name: Prompt caching (5m TTL)
  - id: vision
    name: Vision
  - id: thinking
    name: Thinking
    # The single row covers both API shapes Anthropic exposes — manual
    # `thinking: {type: "enabled", budget_tokens: N}` (Haiku 4.5) and
    # `thinking: {type: "adaptive"}` (Opus 4.7); Sonnet 4.6 supports
    # either and Claude Code picks per model. A break in either
    # transformer surfaces as a red cell because all three tiers must
    # pass for the cell to go green. The row was named
    # `extended_thinking` historically; Anthropic's docs now reserve
    # that name for the deprecated manual mode only, so the row was
    # renamed to the feature-level "Thinking".
  - id: tool_use_streaming
    name: Tool use (streaming / fine-grained)
  - id: thinking_with_tool_use
    name: Extended thinking + tool use
  - id: pdf_input
    name: PDF document input
  - id: prompt_caching_1h
    name: Prompt caching (1h TTL)
  - id: web_search
    name: Web search (server tool)
  - id: structured_outputs
    name: Structured outputs
    # Drives `claude --json-schema '<schema>'`. Implementation note:
    # Claude Code translates `--json-schema` to a synthetic
    # `StructuredOutput` tool whose `input_schema` is the user's
    # schema, then surfaces the tool_use input as
    # `structured_output: {...}` on the trailing `result` event.
    # This row tests that proxy-side handling of that tool round-
    # trips end-to-end. It does NOT test Anthropic's server-side
    # `output_config.schema` parameter (a separate feature used
    # internally by Claude Code for session-title generation) --
    # `output_config` regressions surface in the HTTP-probe rows.
  - id: count_tokens
    name: count_tokens endpoint
    # HTTP-probe row. Sends a direct POST to
    # `{proxy}/v1/messages/count_tokens` for each Claude tier and
    # asserts the response is shaped `{"input_tokens": <positive
    # int>}`. The CLI uses this endpoint internally but never
    # surfaces its result in stream-json, so the only way to test
    # the proxy's handling of it is to hit it directly. LiteLLM has
    # shipped fixes here (e.g. Claude Code release-notes 2.1.121
    # "Vertex AI count_tokens returning 400 errors for proxy
    # gateways"), which is exactly the regression class this row
    # is meant to catch.
  - id: tool_search
    name: Tool search (MCP discovery)
    # HTTP-probe row. Sends a request whose `tools` array includes
    # a `tool_search_tool_regex_20251119` discovery tool and asserts
    # the proxy + upstream accept it. This verifies LiteLLM's
    # per-provider beta-header translation
    # (`advanced-tool-use-2025-11-20` for Anthropic/Azure,
    # `tool-search-tool-2025-10-19` for Vertex/Bedrock) is wired up.
    # We deliberately don't try to trigger Claude Code's MCP-fan-out
    # heuristic via `--mcp-config` -- that would couple the row to
    # an internal behavior threshold that changes between Claude
    # Code releases. The HTTP probe hits the bug surface LiteLLM
    # has actually shipped fixes for (2.1.117, 2.1.72, 2.1.70 per
    # the Claude Code release notes).
  - id: passthrough
    name: Native API passthrough
    # Drives the CLI in each cloud's native mode against LiteLLM's
    # passthrough routes instead of the /v1/messages translation
    # layer -- the "LLM gateway" setup from
    # https://code.claude.com/docs/en/gateway. anthropic uses
    # ANTHROPIC_BASE_URL={proxy}/anthropic; bedrock_invoke uses
    # CLAUDE_CODE_USE_BEDROCK=1 against {proxy}/bedrock (InvokeModel
    # wire, alias resolved from the URL by the router); vertex_ai
    # uses CLAUDE_CODE_USE_VERTEX=1 against {proxy}/vertex_ai/v1
    # (rawPredict wire, alias + project + location resolved from the
    # deployment, which therefore needs `use_in_pass_through: true`);
    # azure uses CLAUDE_CODE_USE_FOUNDRY=1 against {proxy}/azure and
    # needs AZURE_API_BASE/AZURE_API_KEY on the proxy (see
    # passthrough/test_azure.py and the cron env example).
    # bedrock_converse is structurally not_applicable: Claude Code
    # has no Converse-wire client.
  - id: long_context_1m
    name: Long context (1M)
    # Sends a ~210k-token padded prompt with the
    # `context-1m-2025-08-07` beta header. Just-above the standard
    # 200k context window so the request can only succeed when the
    # beta header makes it all the way through the proxy to the
    # upstream. Haiku 4.5 is intentionally omitted from this row's
    # model list (its window is 200k); Sonnet 4.6 and Opus 4.7 are
    # the only tiers exercised. Costs roughly $4/cell/run --
    # tighten the prompt-token target if pricing changes meaningfully.