test(spend): move the spend-counter scope churn repro to tests/integration

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kerry 2026-09-22 18:44:47 +00:00
parent 675de2f1f4
commit ec97ed69c2
4 changed files with 73 additions and 78 deletions

View file

@ -38,7 +38,6 @@
- {id: quota_management.budget.team_multi_window.blocks_then_resets, module: quota_management, tier: P1, behavior: budget, variant: team_multi_window, assertions: [blocks_then_resets], exercised_on: [chat_completions], source: "proxy/common_utils/reset_budget_job.py", rationale: "Team budget_limits enforce and reset per window"}
- {id: quota_management.budget.fallback.routes_to_fallback, module: quota_management, tier: P1, behavior: budget, variant: fallback, assertions: [routes_to_fallback], exercised_on: [messages], source: "proxy/hooks/model_max_budget_limiter.py", rationale: "budget_fallbacks reroute to the fallback model once the primary's budget is exhausted"}
- {id: quota_management.budget.spend_counter.reseed_matches_db, module: quota_management, tier: P2, behavior: budget, variant: spend_counter, assertions: [reseed_matches_db], exercised_on: [chat_completions], source: "proxy/spend_tracking/budget_reservation.py", rationale: "Concurrent cold-counter reseeds keep the enforcement counter equal to DB spend (#26829)"}
- {id: quota_management.budget.spend_counter.survives_scope_churn, module: quota_management, tier: P1, behavior: budget, variant: spend_counter, assertions: [survives_scope_churn], exercised_on: [chat_completions], source: "proxy/proxy_server.py:spend_counter_cache", rationale: "A key's in-memory spend counter is not evicted by traffic on hundreds of other budget scopes, so an over-budget key stays blocked instead of being re-admitted off lagging DB spend (#40221)"}
- {id: quota_management.spend_tracking.chat_completions.logs_cost, module: quota_management, tier: P0, behavior: spend_tracking, variant: chat_completions, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A paid chat call writes a nonzero spend row"}
- {id: quota_management.spend_tracking.stream.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Streaming responses aggregate token counts into a spend row"}
- {id: quota_management.spend_tracking.messages_bridge.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [logs_cost], exercised_on: [messages], source: "llms/anthropic/experimental_pass_through/responses_adapters/handler.py", rationale: "A streaming /v1/messages request served by an openai-provider model is bridged through the anthropic-messages -> Responses adapter and must aggregate the consumed SSE stream into one spend row with nonzero cost and token counts, attributed to custom_llm_provider openai under call_type anthropic_messages"}

View file

@ -1,77 +0,0 @@
"""Live e2e: an over-budget key stays blocked while hundreds of other budget scopes take traffic (#40221).
The proxy keeps one in-memory spend counter per budget scope (key, team, end user, tag, ...)
in ``spend_counter_cache``. Every request that names a new ``user`` creates a new end-user
counter. When that cache is sized like the generic in-memory cache (200 entries), the 201st
active scope evicts a still-valid counter, and the next request on the evicted key reseeds it
from the DB. The DB row lags the counter until ``proxy_batch_write_at`` flushes, so a key the
proxy had just rejected for being over budget is admitted again.
The test blocks a tiny-budget key with a real call, then drives more than 200 distinct end users
on a second key, then re-probes the blocked key. The invariant (blocked stays blocked) must hold
on every stack. The eviction itself is only reachable on a proxy without Redis whose DB row still
lags (``proxy_batch_write_at`` above the churn time); with Redis the cross-pod counter answers
first, and a flushed row reseeds correctly, so those stacks re-prove the plain budget block.
The probe key's DB spend at re-probe time is reported in the failure message so a reader can
tell which path was exercised.
"""
from concurrent.futures import ThreadPoolExecutor
import pytest
from budget_client import BudgetClient, is_budget_block
from e2e_config import unique_marker
from e2e_http import StreamingResponse
from lifecycle import ResourceManager
pytestmark = pytest.mark.e2e
MODEL = "claude-haiku-4-5"
TINY_BUDGET = 0.000001
GENERIC_CACHE_SIZE = 200
CHURN_SCOPES = 250
CHURN_WORKERS = 24
def _chat(client: BudgetClient, key: str, user: str) -> StreamingResponse:
return client.chat(key, MODEL, f"scope churn {unique_marker()}", max_tokens=1, user=user)
class TestSpendCounterScopeChurn:
@pytest.mark.covers("quota_management.budget.spend_counter.survives_scope_churn")
def test_over_budget_key_stays_blocked_after_hundreds_of_other_scopes(
self, client: BudgetClient, resources: ResourceManager
) -> None:
run = unique_marker()
probe_key = client.generate_key(max_budget=TINY_BUDGET, models=[MODEL])
resources.defer(lambda: client.delete_key(probe_key))
churn_key = client.generate_key(models=[MODEL])
resources.defer(lambda: client.delete_key(churn_key))
churn_users = [f"churn-{run}-{index}" for index in range(CHURN_SCOPES)]
resources.defer(lambda: client.delete_customers(churn_users))
first = _chat(client, probe_key, user=f"probe-{run}")
assert first.ok, f"first call on a fresh key must be admitted: {first.status_code} {first.body[:300]}"
blocked = _chat(client, probe_key, user=f"probe-{run}")
assert is_budget_block(blocked), (
f"precondition: the key must be over budget before the churn: {blocked.status_code} {blocked.body[:300]}"
)
def churn_call(user: str) -> StreamingResponse:
return _chat(client, churn_key, user=user)
with ThreadPoolExecutor(max_workers=CHURN_WORKERS) as pool:
churn = list(pool.map(churn_call, churn_users))
admitted = sum(1 for r in churn if r.ok)
assert admitted > GENERIC_CACHE_SIZE, (
f"only {admitted}/{CHURN_SCOPES} churn calls succeeded; cannot exceed {GENERIC_CACHE_SIZE} live scopes"
)
db_spend = client.proxy.key_info(probe_key).spend or 0.0
after = _chat(client, probe_key, user=f"probe-{run}")
assert is_budget_block(after), (
f"over-budget key was admitted after {admitted} other scopes took traffic (DB spend {db_spend}): "
f"{after.status_code} {after.body[:300]}; its spend counter was evicted and reseeded from lagging "
"DB spend (#40221)"
)

View file

@ -107,6 +107,9 @@
"tests/integration/spend/test_cache_and_quota.py::test_key_budget_at_boundary_blocks_provider_then_explicit_reset_restores": [
"quota_management.budget.key.boundary_blocks_before_provider_and_reset_restores"
],
"tests/integration/spend/test_spend_counter_scope_churn.py::test_over_budget_key_stays_blocked_while_hundreds_of_end_users_take_traffic": [
"quota_management.budget.spend_counter.survives_scope_churn"
],
"tests/integration/spend/test_cache_and_quota.py::test_different_system_messages_do_not_share_a_cached_response": [
"quota_management.response_cache.system_messages_partition_cache_identity"
],

View file

@ -0,0 +1,70 @@
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Final
import httpx
import pytest
import yaml
from integration._support.client import Gateway
from integration._support.process import owned_proxy
from pydantic import TypeAdapter
GENERIC_CACHE_SIZE: Final = 200
CHURN_SCOPES: Final = 250
def _chat(candidate: Gateway, model: str, key: str, user: str) -> httpx.Response:
return candidate.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": f"scope churn {uuid.uuid4().hex}"}], "user": user},
key=key,
)
@pytest.mark.covers("quota_management.budget.spend_counter.survives_scope_churn")
@pytest.mark.timeout(180)
def test_over_budget_key_stays_blocked_while_hundreds_of_end_users_take_traffic(
gateway: Gateway, tmp_path: Path
) -> None:
shared: Final = TypeAdapter(dict[str, object]).validate_python(
yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
)
general_settings: Final = TypeAdapter(dict[str, object]).validate_python(shared["general_settings"])
path: Final = tmp_path / "in-memory-counters.yaml"
path.write_text(
yaml.safe_dump(
{
"model_list": [],
"general_settings": {**general_settings, "proxy_batch_write_at": 600},
"router_settings": shared["router_settings"],
}
)
)
run: Final = uuid.uuid4().hex
with (
owned_proxy(gateway, tmp_path, {}, config=path, remove_environment=("REDIS_HOST", "REDIS_PORT")) as candidate,
candidate.scenario() as scenario,
httpx.Client(base_url=candidate.upstream_url, timeout=5, trust_env=False) as upstream,
):
model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002)
probe: Final = scenario.key(models=[model], max_budget=0.06)
churn: Final = scenario.key(models=[model])
def _churn_status(index: int) -> int:
return _chat(candidate, model, churn, f"churn-{run}-{index}").status_code
first: Final = _chat(candidate, model, probe, f"probe-{run}")
assert first.status_code == 200 and first.json()["usage"]["total_tokens"] == 40, first.text
blocked: Final = _chat(candidate, model, probe, f"probe-{run}")
assert blocked.status_code == 422 and blocked.json()["error"]["type"] == "budget_exceeded", blocked.text
with ThreadPoolExecutor(max_workers=8) as pool:
statuses: Final = tuple(pool.map(_churn_status, range(CHURN_SCOPES)))
assert statuses.count(200) > GENERIC_CACHE_SIZE, statuses
upstream.get("/__observations").raise_for_status()
after: Final = _chat(candidate, model, probe, f"probe-{run}")
assert after.status_code == 422 and after.json()["error"]["type"] == "budget_exceeded", (
f"over-budget key admitted after {statuses.count(200)} other scopes took traffic: {after.status_code} {after.text}"
)
assert upstream.get("/__observations").json()["requests"] == []