From 6b91a7acd8b62bbaa880f5a2beff50f91f2e8fff Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 17 May 2026 01:40:23 +0000 Subject: [PATCH] fix(claude_code): rate-limit HTTP probe rows alongside CLI rows probe_count_tokens and probe_tool_search were sending requests directly via httpx with no call to the cross-process RateLimiter that cli_driver.run_claude uses. During a full matrix run those unthrottled probes would silently violate the limiter's aggregate per-provider budget and could push adjacent CLI cells over the 429 threshold. Acquire one token from the same process-wide limiter (keyed by infer_provider(model)) before each probe, with an injectable rate_limiter seam matching run_claude's API for unit tests. Co-authored-by: Yassin Kortam --- tests/claude_code/http_probe.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/claude_code/http_probe.py b/tests/claude_code/http_probe.py index 5628f7af3d9..808725e9ab7 100644 --- a/tests/claude_code/http_probe.py +++ b/tests/claude_code/http_probe.py @@ -32,6 +32,12 @@ from typing import Any, Mapping, Optional import httpx +from tests.claude_code.rate_limiter import ( + RateLimiter, + get_default_limiter, + infer_provider, +) + DEFAULT_TIMEOUT_SECONDS = 30.0 @@ -60,6 +66,7 @@ def probe_count_tokens( model: str, message: str = "hello world", timeout: float = DEFAULT_TIMEOUT_SECONDS, + rate_limiter: Optional[RateLimiter] = None, ) -> ProbeResult: """POST to `{base_url}/v1/messages/count_tokens` for `model` and return the parsed result. @@ -68,7 +75,18 @@ def probe_count_tokens( returns `{"input_tokens": N}` for a successful response. Anything else -- non-200 status, non-JSON body, missing/non-int `input_tokens` -- is a regression we want the cell to flip red on. + + The same cross-process token-bucket limiter `cli_driver.run_claude` + uses is acquired here too, so probe rows count against the + aggregate per-provider budget. Without this, an HTTP-probe row + would fire unthrottled requests in parallel with throttled CLI + rows and silently violate the limiter's aggregate-rate guarantee. + `rate_limiter` is an injection seam for unit tests; production + callers should leave it unset to use the process-wide default. """ + limiter = rate_limiter if rate_limiter is not None else get_default_limiter() + limiter.acquire(infer_provider(model)) + url = base_url.rstrip("/") + "/v1/messages/count_tokens" try: response = httpx.post( @@ -107,6 +125,7 @@ def probe_tool_search( api_key: str, model: str, timeout: float = DEFAULT_TIMEOUT_SECONDS, + rate_limiter: Optional[RateLimiter] = None, ) -> ProbeResult: """POST to `{base_url}/v1/messages` with a `tool_search_tool_regex_20251119` tool definition and return the result. @@ -127,7 +146,15 @@ def probe_tool_search( response, not to test whether the model decided to invoke tool_search. That kind of behavior test would couple this row to Claude Code's model behavior heuristics, which change weekly. + + Like `probe_count_tokens`, this acquires one token from the + process-wide rate limiter so probe traffic counts against the + same aggregate per-provider budget as the CLI rows. `rate_limiter` + is a test seam; production callers should leave it unset. """ + limiter = rate_limiter if rate_limiter is not None else get_default_limiter() + limiter.acquire(infer_provider(model)) + url = base_url.rstrip("/") + "/v1/messages" payload = { "model": model,