mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-22 00:31:44 +00:00
fix(claude_code): rate-limit HTTP probe rows alongside CLI rows
probe_count_tokens and probe_tool_search were sending requests directly via httpx with no call to the cross-process RateLimiter that cli_driver.run_claude uses. During a full matrix run those unthrottled probes would silently violate the limiter's aggregate per-provider budget and could push adjacent CLI cells over the 429 threshold. Acquire one token from the same process-wide limiter (keyed by infer_provider(model)) before each probe, with an injectable rate_limiter seam matching run_claude's API for unit tests. Co-authored-by: Yassin Kortam <yassin@berri.ai>
This commit is contained in:
parent
6d6689258e
commit
6b91a7acd8
1 changed files with 27 additions and 0 deletions
|
|
@ -32,6 +32,12 @@ from typing import Any, Mapping, Optional
|
|||
|
||||
import httpx
|
||||
|
||||
from tests.claude_code.rate_limiter import (
|
||||
RateLimiter,
|
||||
get_default_limiter,
|
||||
infer_provider,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_TIMEOUT_SECONDS = 30.0
|
||||
|
||||
|
|
@ -60,6 +66,7 @@ def probe_count_tokens(
|
|||
model: str,
|
||||
message: str = "hello world",
|
||||
timeout: float = DEFAULT_TIMEOUT_SECONDS,
|
||||
rate_limiter: Optional[RateLimiter] = None,
|
||||
) -> ProbeResult:
|
||||
"""POST to `{base_url}/v1/messages/count_tokens` for `model` and return the parsed result.
|
||||
|
||||
|
|
@ -68,7 +75,18 @@ def probe_count_tokens(
|
|||
returns `{"input_tokens": N}` for a successful response. Anything
|
||||
else -- non-200 status, non-JSON body, missing/non-int
|
||||
`input_tokens` -- is a regression we want the cell to flip red on.
|
||||
|
||||
The same cross-process token-bucket limiter `cli_driver.run_claude`
|
||||
uses is acquired here too, so probe rows count against the
|
||||
aggregate per-provider budget. Without this, an HTTP-probe row
|
||||
would fire unthrottled requests in parallel with throttled CLI
|
||||
rows and silently violate the limiter's aggregate-rate guarantee.
|
||||
`rate_limiter` is an injection seam for unit tests; production
|
||||
callers should leave it unset to use the process-wide default.
|
||||
"""
|
||||
limiter = rate_limiter if rate_limiter is not None else get_default_limiter()
|
||||
limiter.acquire(infer_provider(model))
|
||||
|
||||
url = base_url.rstrip("/") + "/v1/messages/count_tokens"
|
||||
try:
|
||||
response = httpx.post(
|
||||
|
|
@ -107,6 +125,7 @@ def probe_tool_search(
|
|||
api_key: str,
|
||||
model: str,
|
||||
timeout: float = DEFAULT_TIMEOUT_SECONDS,
|
||||
rate_limiter: Optional[RateLimiter] = None,
|
||||
) -> ProbeResult:
|
||||
"""POST to `{base_url}/v1/messages` with a `tool_search_tool_regex_20251119`
|
||||
tool definition and return the result.
|
||||
|
|
@ -127,7 +146,15 @@ def probe_tool_search(
|
|||
response, not to test whether the model decided to invoke
|
||||
tool_search. That kind of behavior test would couple this row to
|
||||
Claude Code's model behavior heuristics, which change weekly.
|
||||
|
||||
Like `probe_count_tokens`, this acquires one token from the
|
||||
process-wide rate limiter so probe traffic counts against the
|
||||
same aggregate per-provider budget as the CLI rows. `rate_limiter`
|
||||
is a test seam; production callers should leave it unset.
|
||||
"""
|
||||
limiter = rate_limiter if rate_limiter is not None else get_default_limiter()
|
||||
limiter.acquire(infer_provider(model))
|
||||
|
||||
url = base_url.rstrip("/") + "/v1/messages"
|
||||
payload = {
|
||||
"model": model,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue