From ec97ed69c2390839a2a830093ea5b5652d3e6764 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 22 Sep 2026 18:44:47 +0000 Subject: [PATCH] test(spend): move the spend-counter scope churn repro to tests/integration Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../coverage_registry/quota_management.yaml | 1 - .../test_spend_counter_scope_churn_e2e.py | 77 ------------------- tests/integration/contracts.json | 3 + .../spend/test_spend_counter_scope_churn.py | 70 +++++++++++++++++ 4 files changed, 73 insertions(+), 78 deletions(-) delete mode 100644 tests/e2e/quota_management/budgets/test_spend_counter_scope_churn_e2e.py create mode 100644 tests/integration/spend/test_spend_counter_scope_churn.py diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index d7179b40ab0..ad0914d455b 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -38,7 +38,6 @@ - {id: quota_management.budget.team_multi_window.blocks_then_resets, module: quota_management, tier: P1, behavior: budget, variant: team_multi_window, assertions: [blocks_then_resets], exercised_on: [chat_completions], source: "proxy/common_utils/reset_budget_job.py", rationale: "Team budget_limits enforce and reset per window"} - {id: quota_management.budget.fallback.routes_to_fallback, module: quota_management, tier: P1, behavior: budget, variant: fallback, assertions: [routes_to_fallback], exercised_on: [messages], source: "proxy/hooks/model_max_budget_limiter.py", rationale: "budget_fallbacks reroute to the fallback model once the primary's budget is exhausted"} - {id: quota_management.budget.spend_counter.reseed_matches_db, module: quota_management, tier: P2, behavior: budget, variant: spend_counter, assertions: [reseed_matches_db], exercised_on: [chat_completions], source: "proxy/spend_tracking/budget_reservation.py", rationale: "Concurrent cold-counter reseeds keep the enforcement counter equal to DB spend (#26829)"} -- {id: quota_management.budget.spend_counter.survives_scope_churn, module: quota_management, tier: P1, behavior: budget, variant: spend_counter, assertions: [survives_scope_churn], exercised_on: [chat_completions], source: "proxy/proxy_server.py:spend_counter_cache", rationale: "A key's in-memory spend counter is not evicted by traffic on hundreds of other budget scopes, so an over-budget key stays blocked instead of being re-admitted off lagging DB spend (#40221)"} - {id: quota_management.spend_tracking.chat_completions.logs_cost, module: quota_management, tier: P0, behavior: spend_tracking, variant: chat_completions, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A paid chat call writes a nonzero spend row"} - {id: quota_management.spend_tracking.stream.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Streaming responses aggregate token counts into a spend row"} - {id: quota_management.spend_tracking.messages_bridge.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [logs_cost], exercised_on: [messages], source: "llms/anthropic/experimental_pass_through/responses_adapters/handler.py", rationale: "A streaming /v1/messages request served by an openai-provider model is bridged through the anthropic-messages -> Responses adapter and must aggregate the consumed SSE stream into one spend row with nonzero cost and token counts, attributed to custom_llm_provider openai under call_type anthropic_messages"} diff --git a/tests/e2e/quota_management/budgets/test_spend_counter_scope_churn_e2e.py b/tests/e2e/quota_management/budgets/test_spend_counter_scope_churn_e2e.py deleted file mode 100644 index c8a71285acc..00000000000 --- a/tests/e2e/quota_management/budgets/test_spend_counter_scope_churn_e2e.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Live e2e: an over-budget key stays blocked while hundreds of other budget scopes take traffic (#40221). - -The proxy keeps one in-memory spend counter per budget scope (key, team, end user, tag, ...) -in ``spend_counter_cache``. Every request that names a new ``user`` creates a new end-user -counter. When that cache is sized like the generic in-memory cache (200 entries), the 201st -active scope evicts a still-valid counter, and the next request on the evicted key reseeds it -from the DB. The DB row lags the counter until ``proxy_batch_write_at`` flushes, so a key the -proxy had just rejected for being over budget is admitted again. - -The test blocks a tiny-budget key with a real call, then drives more than 200 distinct end users -on a second key, then re-probes the blocked key. The invariant (blocked stays blocked) must hold -on every stack. The eviction itself is only reachable on a proxy without Redis whose DB row still -lags (``proxy_batch_write_at`` above the churn time); with Redis the cross-pod counter answers -first, and a flushed row reseeds correctly, so those stacks re-prove the plain budget block. -The probe key's DB spend at re-probe time is reported in the failure message so a reader can -tell which path was exercised. -""" - -from concurrent.futures import ThreadPoolExecutor - -import pytest - -from budget_client import BudgetClient, is_budget_block -from e2e_config import unique_marker -from e2e_http import StreamingResponse -from lifecycle import ResourceManager - -pytestmark = pytest.mark.e2e - -MODEL = "claude-haiku-4-5" -TINY_BUDGET = 0.000001 -GENERIC_CACHE_SIZE = 200 -CHURN_SCOPES = 250 -CHURN_WORKERS = 24 - - -def _chat(client: BudgetClient, key: str, user: str) -> StreamingResponse: - return client.chat(key, MODEL, f"scope churn {unique_marker()}", max_tokens=1, user=user) - - -class TestSpendCounterScopeChurn: - @pytest.mark.covers("quota_management.budget.spend_counter.survives_scope_churn") - def test_over_budget_key_stays_blocked_after_hundreds_of_other_scopes( - self, client: BudgetClient, resources: ResourceManager - ) -> None: - run = unique_marker() - probe_key = client.generate_key(max_budget=TINY_BUDGET, models=[MODEL]) - resources.defer(lambda: client.delete_key(probe_key)) - churn_key = client.generate_key(models=[MODEL]) - resources.defer(lambda: client.delete_key(churn_key)) - churn_users = [f"churn-{run}-{index}" for index in range(CHURN_SCOPES)] - resources.defer(lambda: client.delete_customers(churn_users)) - - first = _chat(client, probe_key, user=f"probe-{run}") - assert first.ok, f"first call on a fresh key must be admitted: {first.status_code} {first.body[:300]}" - blocked = _chat(client, probe_key, user=f"probe-{run}") - assert is_budget_block(blocked), ( - f"precondition: the key must be over budget before the churn: {blocked.status_code} {blocked.body[:300]}" - ) - - def churn_call(user: str) -> StreamingResponse: - return _chat(client, churn_key, user=user) - - with ThreadPoolExecutor(max_workers=CHURN_WORKERS) as pool: - churn = list(pool.map(churn_call, churn_users)) - admitted = sum(1 for r in churn if r.ok) - assert admitted > GENERIC_CACHE_SIZE, ( - f"only {admitted}/{CHURN_SCOPES} churn calls succeeded; cannot exceed {GENERIC_CACHE_SIZE} live scopes" - ) - - db_spend = client.proxy.key_info(probe_key).spend or 0.0 - after = _chat(client, probe_key, user=f"probe-{run}") - assert is_budget_block(after), ( - f"over-budget key was admitted after {admitted} other scopes took traffic (DB spend {db_spend}): " - f"{after.status_code} {after.body[:300]}; its spend counter was evicted and reseeded from lagging " - "DB spend (#40221)" - ) diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index e1b5940935f..d455c8b1da3 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -107,6 +107,9 @@ "tests/integration/spend/test_cache_and_quota.py::test_key_budget_at_boundary_blocks_provider_then_explicit_reset_restores": [ "quota_management.budget.key.boundary_blocks_before_provider_and_reset_restores" ], + "tests/integration/spend/test_spend_counter_scope_churn.py::test_over_budget_key_stays_blocked_while_hundreds_of_end_users_take_traffic": [ + "quota_management.budget.spend_counter.survives_scope_churn" + ], "tests/integration/spend/test_cache_and_quota.py::test_different_system_messages_do_not_share_a_cached_response": [ "quota_management.response_cache.system_messages_partition_cache_identity" ], diff --git a/tests/integration/spend/test_spend_counter_scope_churn.py b/tests/integration/spend/test_spend_counter_scope_churn.py new file mode 100644 index 00000000000..1dd0577b428 --- /dev/null +++ b/tests/integration/spend/test_spend_counter_scope_churn.py @@ -0,0 +1,70 @@ +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import Gateway +from integration._support.process import owned_proxy +from pydantic import TypeAdapter + +GENERIC_CACHE_SIZE: Final = 200 +CHURN_SCOPES: Final = 250 + + +def _chat(candidate: Gateway, model: str, key: str, user: str) -> httpx.Response: + return candidate.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": f"scope churn {uuid.uuid4().hex}"}], "user": user}, + key=key, + ) + + +@pytest.mark.covers("quota_management.budget.spend_counter.survives_scope_churn") +@pytest.mark.timeout(180) +def test_over_budget_key_stays_blocked_while_hundreds_of_end_users_take_traffic( + gateway: Gateway, tmp_path: Path +) -> None: + shared: Final = TypeAdapter(dict[str, object]).validate_python( + yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + ) + general_settings: Final = TypeAdapter(dict[str, object]).validate_python(shared["general_settings"]) + path: Final = tmp_path / "in-memory-counters.yaml" + path.write_text( + yaml.safe_dump( + { + "model_list": [], + "general_settings": {**general_settings, "proxy_batch_write_at": 600}, + "router_settings": shared["router_settings"], + } + ) + ) + run: Final = uuid.uuid4().hex + with ( + owned_proxy(gateway, tmp_path, {}, config=path, remove_environment=("REDIS_HOST", "REDIS_PORT")) as candidate, + candidate.scenario() as scenario, + httpx.Client(base_url=candidate.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + probe: Final = scenario.key(models=[model], max_budget=0.06) + churn: Final = scenario.key(models=[model]) + + def _churn_status(index: int) -> int: + return _chat(candidate, model, churn, f"churn-{run}-{index}").status_code + + first: Final = _chat(candidate, model, probe, f"probe-{run}") + assert first.status_code == 200 and first.json()["usage"]["total_tokens"] == 40, first.text + blocked: Final = _chat(candidate, model, probe, f"probe-{run}") + assert blocked.status_code == 422 and blocked.json()["error"]["type"] == "budget_exceeded", blocked.text + with ThreadPoolExecutor(max_workers=8) as pool: + statuses: Final = tuple(pool.map(_churn_status, range(CHURN_SCOPES))) + assert statuses.count(200) > GENERIC_CACHE_SIZE, statuses + upstream.get("/__observations").raise_for_status() + after: Final = _chat(candidate, model, probe, f"probe-{run}") + assert after.status_code == 422 and after.json()["error"]["type"] == "budget_exceeded", ( + f"over-budget key admitted after {statuses.count(200)} other scopes took traffic: {after.status_code} {after.text}" + ) + assert upstream.get("/__observations").json()["requests"] == []