mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-10 22:41:41 +00:00
0439bcbfed
* refactor(e2e): fold claude_code HTTP probes onto shared Gateway methods Migrate tests/e2e/claude_code/http_probe.py off its own httpx client onto the shared transport, and promote count_tokens and native anthropic messages to first-class Gateway methods (Gateway.count_tokens / Gateway.messages) with typed request/response models in the shared models.py so other suites reuse them. The probes now take an injected Gateway and issue their request through the shared count_tokens/messages methods, reusing the split control/data-plane routing, timeout, and typed Result handling the rest of tests/e2e uses. The wire shape is preserved: the pydantic bodies serialize byte-for-byte to what the old httpx probes sent, and the anthropic-version header is carried by a small AnthropicHeaders model. httpx is gone from the module. * test(e2e): drop unit-level probe harness test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
77 lines
2.7 KiB
Python
77 lines
2.7 KiB
Python
"""count_tokens x Anthropic.
|
|
|
|
HTTP-probe row. Unlike the CLI-driven rows, this test never invokes
|
|
the `claude` CLI: it `POST`s directly to
|
|
`{proxy}/v1/messages/count_tokens` for each Claude tier and asserts
|
|
the response is shaped `{"input_tokens": <positive int>}`.
|
|
|
|
The (feature, provider) for this cell is inferred from the file path by
|
|
`tests/e2e/claude_code/conftest.py`:
|
|
|
|
tests/e2e/claude_code/count_tokens/test_anthropic.py
|
|
^^^^^^^^^^^^ ^^^^^^^^^
|
|
feature_id provider
|
|
|
|
Why HTTP probe instead of CLI:
|
|
|
|
Claude Code calls `count_tokens` internally to compute budget /
|
|
context-window usage display, but the result is consumed by the CLI
|
|
in-process and never appears in stream-json events. There is no CLI
|
|
flag that emits the count to stdout in a way our existing
|
|
stream-json parser can pick up, so we can't test the endpoint round
|
|
trip through the CLI surface.
|
|
|
|
The proxy *is* expected to expose `/v1/messages/count_tokens` for
|
|
every Claude-style provider it routes to -- LiteLLM has historically
|
|
had provider-specific bugs in this endpoint (Vertex AI `count_tokens`
|
|
returned 400 to proxy gateways; see Claude Code release notes 2.1.121).
|
|
Treating it as a matrix row keeps regressions in the cron's daily
|
|
diff.
|
|
|
|
The cell goes red if *any* tier's probe fails the minimal shape
|
|
check; the matrix's per-cell aggregator handles that automatically.
|
|
Three tiers run sequentially because count_tokens is cheap (<100ms
|
|
per request typical) and the parallelization that matters for the
|
|
CLI rows isn't useful here.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from claude_code._env import require_proxy_client
|
|
from claude_code.http_probe import (
|
|
assert_count_tokens_shape,
|
|
probe_count_tokens,
|
|
)
|
|
|
|
|
|
ANTHROPIC_MODELS = [
|
|
"claude-haiku-4-5",
|
|
"claude-sonnet-4-5",
|
|
"claude-opus-4-7",
|
|
]
|
|
|
|
|
|
@pytest.mark.covers("llm.messages.anthropic.count_tokens.nonstream.works")
|
|
def test_count_tokens_anthropic(compat_result):
|
|
"""Probe `/v1/messages/count_tokens` for each Anthropic tier and
|
|
assert the response shape."""
|
|
client, api_key = require_proxy_client(compat_result)
|
|
|
|
failures = []
|
|
for model in ANTHROPIC_MODELS:
|
|
result = probe_count_tokens(
|
|
client=client, api_key=api_key, model=model
|
|
)
|
|
shape_error = assert_count_tokens_shape(result)
|
|
if shape_error is not None:
|
|
error = f"[{model}] count_tokens probe failed: {shape_error}"
|
|
compat_result.add({"status": "fail", "error": error})
|
|
failures.append(error)
|
|
continue
|
|
|
|
compat_result.add({"status": "pass"})
|
|
|
|
if failures:
|
|
pytest.fail("; ".join(failures), pytrace=False)
|