diff --git a/tests/e2e/budgets/test_budget_reset_e2e.py b/tests/e2e/budgets/test_budget_reset_e2e.py index 4727bd8f9cf..dcf776db9a2 100644 --- a/tests/e2e/budgets/test_budget_reset_e2e.py +++ b/tests/e2e/budgets/test_budget_reset_e2e.py @@ -44,13 +44,16 @@ def test_key_budget_resets_after_duration( assert blocked, "key budget never enforced" # 2. once the 30s duration elapses + the reset job runs, key.spend zeroes and - # calls flow again, within a span only a short duration could produce. + # calls flow again. The window is wall-clock-aligned, so the reset lands up to + # a window later, then the rescheduler (~15-20s) zeroes the spend; allow + # generous headroom over that. A stuck rescheduler is caught by the wait-loop + # timeout, not this elapsed bound. start = time.monotonic() - while time.monotonic() < start + 90: + while time.monotonic() < start + 150: time.sleep(5) result = _call(client, key) if result.ok: - assert time.monotonic() - start < 75, "reset too slow for a 30s budget" + assert time.monotonic() - start < 120, "reset too slow for a 30s budget" return assert is_budget_block(result), f"non-budget error: {result.body[:200]}" - pytest.fail("key budget never reset within 90s") + pytest.fail("key budget never reset within 150s") diff --git a/tests/e2e/budgets/test_multi_window_budget_e2e.py b/tests/e2e/budgets/test_multi_window_budget_e2e.py index 3a3720302c8..553ad1ce701 100644 --- a/tests/e2e/budgets/test_multi_window_budget_e2e.py +++ b/tests/e2e/budgets/test_multi_window_budget_e2e.py @@ -52,19 +52,19 @@ def test_short_window_blocks_then_resets( time.sleep(2) assert blocked, f"{WINDOW_SECONDS}s window never enforced" - # 2. the window resets at the next wall-clock-aligned boundary (so it can land - # a little under WINDOW_SECONDS from creation) + the reset job. When a call - # flows again the window has reset; the elapsed clock must be short enough - # that this is the 30s window resetting, not the roomy 1m one. - deadline = time.monotonic() + 90 + # 2. the window resets at the next wall-clock-aligned boundary (up to a window + # after start), then the reset job (~15-20s rescheduler) zeroes the spend. + # Allow generous headroom for that alignment + rescheduler latency; a stuck + # rescheduler is caught by the wait-loop timeout, not this elapsed bound. + deadline = time.monotonic() + 150 while time.monotonic() < deadline: time.sleep(5) result = _call(client, key) if result.ok: elapsed = time.monotonic() - start - assert elapsed < WINDOW_SECONDS + 45, ( + assert elapsed < WINDOW_SECONDS + 90, ( f"reset took {elapsed:.0f}s - too long for a {WINDOW_SECONDS}s window" ) return assert is_budget_block(result), f"non-budget error during reset wait: {result.body[:200]}" - pytest.fail(f"{WINDOW_SECONDS}s window never reset within 90s") + pytest.fail(f"{WINDOW_SECONDS}s window never reset within 150s") diff --git a/tests/e2e/budgets/test_team_member_budget_reset_e2e.py b/tests/e2e/budgets/test_team_member_budget_reset_e2e.py index 8f42550be38..2749f16a26e 100644 --- a/tests/e2e/budgets/test_team_member_budget_reset_e2e.py +++ b/tests/e2e/budgets/test_team_member_budget_reset_e2e.py @@ -38,10 +38,10 @@ def test_team_member_budget_reset_keeps_advancing(client: BudgetClient, resource # once the window elapses the reset job must move budget_reset_at forward; a job # that skips the member's budget row (the #25109 regression) leaves it pinned at # first_reset forever - deadline = time.monotonic() + 90 + deadline = time.monotonic() + 150 while time.monotonic() < deadline: time.sleep(5) current = client.member_budget_reset_at(team_id, user_id) if current and _as_datetime(current) > first_reset: return - pytest.fail(f"member budget_reset_at never advanced past {first_reset.isoformat()} in 90s") + pytest.fail(f"member budget_reset_at never advanced past {first_reset.isoformat()} in 150s") diff --git a/tests/e2e/budgets/test_team_multi_window_budget_e2e.py b/tests/e2e/budgets/test_team_multi_window_budget_e2e.py index 2d4fb860045..ad79aebf5a0 100644 --- a/tests/e2e/budgets/test_team_multi_window_budget_e2e.py +++ b/tests/e2e/budgets/test_team_multi_window_budget_e2e.py @@ -56,16 +56,17 @@ def test_team_short_window_blocks_then_resets(client: BudgetClient, resources: R time.sleep(2) assert blocked, f"team {WINDOW_SECONDS}s window never enforced" - # 2. the window resets at the next wall-clock-aligned boundary + the reset job. When - # a call flows again the window has reset; the elapsed clock must be short enough - # that this is the 30s window resetting, not the roomy 1m one. - deadline = time.monotonic() + 90 + # 2. the window resets at the next wall-clock-aligned boundary (up to a window + # after start), then the reset job (~15-20s rescheduler) zeroes the spend. + # Allow generous headroom for that alignment + rescheduler latency; a stuck + # rescheduler is caught by the wait-loop timeout, not this elapsed bound. + deadline = time.monotonic() + 150 while time.monotonic() < deadline: time.sleep(5) result = _call(client, key) if result.ok: elapsed = time.monotonic() - start - assert elapsed < WINDOW_SECONDS + 45, f"reset took {elapsed:.0f}s - too long for a {WINDOW_SECONDS}s window" + assert elapsed < WINDOW_SECONDS + 90, f"reset took {elapsed:.0f}s - too long for a {WINDOW_SECONDS}s window" return assert is_budget_block(result), f"non-budget error during reset wait: {result.body[:200]}" - pytest.fail(f"team {WINDOW_SECONDS}s window never reset within 90s") + pytest.fail(f"team {WINDOW_SECONDS}s window never reset within 150s")