From 7c973d895e0ce8c8dcdfd86de614bae72c9fa689 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 08:43:51 -0700 Subject: [PATCH 1/4] test(integration): bound crafted burst 400s against a benign control burst The two mid-burst normalized error tests required every crafted 288 KB model name to get its 400 in under a fixed 10 s. On a loaded CI box a burst of 28 requests takes that long whatever the model name, so main went red with every crafted request correctly refused Each test now sends the same burst with a benign 288 KB model name first and bounds the slowest crafted 400 within CRAFTED_SLOWDOWN_FACTOR of the slowest benign one, the bar the Anthropic SDK test in this file already uses --- .../test_normalized_error_long_message.py | 53 ++++++++++++++----- 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/tests/integration/spend/test_normalized_error_long_message.py b/tests/integration/spend/test_normalized_error_long_message.py index a364327290f..3f185173fb2 100644 --- a/tests/integration/spend/test_normalized_error_long_message.py +++ b/tests/integration/spend/test_normalized_error_long_message.py @@ -349,10 +349,20 @@ def _burst_call(client: httpx.Client, label: str, path: str, body: Mapping[str, def _burst( - client: httpx.Client, happy_model: str, happy_key: str, open_key: str, during: Callable[[], None] + client: httpx.Client, + oversized_model: str, + happy_model: str, + happy_key: str, + open_key: str, + during: Callable[[], None], ) -> tuple[_BurstResult, ...]: - crafted: Final = tuple( - (f"crafted-{path}-{index}", path, _body(path, CRAFTED_MODEL, uuid.uuid4().hex, stream=index % 2 == 1), open_key) + oversized: Final = tuple( + ( + f"oversized-{path}-{index}", + path, + _body(path, oversized_model, uuid.uuid4().hex, stream=index % 2 == 1), + open_key, + ) for path in (CHAT, MESSAGES, RESPONSES) for index in range(4) ) @@ -367,7 +377,7 @@ def _burst( httpx.Client(base_url=client.base_url, timeout=client.timeout, trust_env=False) as fresh, ): first: Final = tuple( - pool.submit(_burst_call, client, label, path, body, key) for label, path, body, key in crafted + happy + pool.submit(_burst_call, client, label, path, body, key) for label, path, body, key in oversized + happy ) wait(first, return_when=FIRST_COMPLETED) during() @@ -377,6 +387,12 @@ def _burst( return tuple(future.result() for future in first + second) +def _slowest_oversized_refusal(results: tuple[_BurstResult, ...]) -> float: + refused: Final = tuple(result for result in results if result.label.startswith("oversized") and result.status) + assert refused and all(result.status == 400 for result in refused), refused + return max(result.seconds for result in refused) + + def _assert_rows_land_exactly_once(results: tuple[_BurstResult, ...], prefix: str) -> None: landed: Final = tuple(result for result in results if result.label.startswith(prefix) and result.status == 200) assert landed, results @@ -405,20 +421,21 @@ def test_killing_one_worker_mid_burst_leaves_the_other_serving_crafted_and_happy ): model: Final = scenario.model(api_base=wire.url + "/v1", num_retries=0) key: Final = scenario.key(models=[model]) + open_key: Final = scenario.key() workers: Final = owned.worker_pids() assert len(workers) == 2, workers victim: Final = min(workers) + control: Final = _slowest_oversized_refusal( + _burst(client, BENIGN_288KB_MODEL, model, key, open_key, lambda: None) + ) with owned.replacing_worker(victim) as kill: - results: Final = _burst(client, model, key, scenario.key(), kill) + results: Final = _burst(client, CRAFTED_MODEL, model, key, open_key, kill) dropped: Final = tuple(result for result in results if result.status is None) assert len(dropped) < len(results), results - crafted: Final = tuple( - result for result in results if result.label.startswith("crafted") and result.status + assert_within_factor_of_control( + _slowest_oversized_refusal(results), control=control, factor=CRAFTED_SLOWDOWN_FACTOR ) - assert crafted and all( - result.status == 400 and result.seconds < FAST_SECONDS for result in crafted - ), crafted late: Final = tuple(result for result in results if result.label.startswith("late")) assert all(result.status == 200 for result in late), late _assert_rows_land_exactly_once(results, "late") @@ -446,7 +463,12 @@ def test_upstream_returning_503_mid_burst_logs_every_failure_with_its_own_cluste ): model: Final = scenario.model(api_base=wire.url + "/v1", num_retries=0) key: Final = scenario.key(models=[model]) - results: Final = _burst(client, model, key, scenario.key(), lambda: None) + open_key: Final = scenario.key() + control: Final = _slowest_oversized_refusal( + _burst(client, BENIGN_288KB_MODEL, model, key, open_key, lambda: None) + ) + wire.drain() + results: Final = _burst(client, CRAFTED_MODEL, model, key, open_key, lambda: None) assert all(result.status is not None for result in results), results happy: Final = tuple(result for result in results if result.label.startswith("happy")) assert all(result.status == 503 for result in happy), happy @@ -456,13 +478,16 @@ def test_upstream_returning_503_mid_burst_logs_every_failure_with_its_own_cluste assert all(any(f"normalized error audit {result.label}" in body for body in seen) for result in happy + late), ( seen ) - crafted: Final = tuple(result for result in results if result.label.startswith("crafted")) - assert all(result.status == 400 and result.seconds < FAST_SECONDS for result in crafted), crafted + assert_within_factor_of_control( + _slowest_oversized_refusal(results), control=control, factor=CRAFTED_SLOWDOWN_FACTOR + ) codes: Final = { result.label: _error_information(string_value(result.call_id))["normalized_error"] for result in results } assert all(code == "503_PROVIDER_OVERLOADED" for label, code in codes.items() if label.startswith("happy")), ( codes ) - assert all(code == "400_INVALID_REQUEST" for label, code in codes.items() if label.startswith("crafted")), codes + assert all(code == "400_INVALID_REQUEST" for label, code in codes.items() if label.startswith("oversized")), ( + codes + ) assert client.get("/health/liveliness").status_code == 200 From 00346b56073000c072d64b9fefd5dccc0d2e7969 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:09:18 -0700 Subject: [PATCH 2/4] test(integration): hold crafted burst 400s to 3x a benign control burst A 1 s crafted-only slowdown passed the factor-10 bar, so the burst tests now use a factor of 3. Clean runs stay under 1.6x and the slowdown lands at 3.6x to 5.4x. The bursts are labelled crafted and benign so a failure names the burst. --- .../test_normalized_error_long_message.py | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/tests/integration/spend/test_normalized_error_long_message.py b/tests/integration/spend/test_normalized_error_long_message.py index 3f185173fb2..5b8e50c2593 100644 --- a/tests/integration/spend/test_normalized_error_long_message.py +++ b/tests/integration/spend/test_normalized_error_long_message.py @@ -9,7 +9,8 @@ from contextlib import contextmanager from dataclasses import dataclass from hashlib import sha256 from pathlib import Path -from typing import Final +from types import MappingProxyType +from typing import Final, Literal, TypeAlias import anthropic import httpx @@ -27,7 +28,12 @@ pytestmark: Final = pytest.mark.shares_worker_state CRAFTED_MODEL: Final = ("exceeded " * 32_000)[:288_000] HOSTILE_5KB_MODEL: Final = ("exceeded budget " * 400)[:5_000] BENIGN_288KB_MODEL: Final = "x" * len(CRAFTED_MODEL) +_Oversized: TypeAlias = Literal["crafted", "benign"] +OVERSIZED_MODELS: Final[Mapping[_Oversized, str]] = MappingProxyType( + {"crafted": CRAFTED_MODEL, "benign": BENIGN_288KB_MODEL} +) CRAFTED_SLOWDOWN_FACTOR: Final = 10 +CRAFTED_BURST_SLOWDOWN_FACTOR: Final = 3 FAST_SECONDS: Final = 10.0 LIVELINESS_STALL_BUDGET_SECONDS: Final = 5.0 ROW_SECONDS: Final = 70 @@ -350,17 +356,17 @@ def _burst_call(client: httpx.Client, label: str, path: str, body: Mapping[str, def _burst( client: httpx.Client, - oversized_model: str, + oversized: _Oversized, happy_model: str, happy_key: str, open_key: str, during: Callable[[], None], ) -> tuple[_BurstResult, ...]: - oversized: Final = tuple( + oversized_calls: Final = tuple( ( - f"oversized-{path}-{index}", + f"{oversized}-{path}-{index}", path, - _body(path, oversized_model, uuid.uuid4().hex, stream=index % 2 == 1), + _body(path, OVERSIZED_MODELS[oversized], uuid.uuid4().hex, stream=index % 2 == 1), open_key, ) for path in (CHAT, MESSAGES, RESPONSES) @@ -377,7 +383,8 @@ def _burst( httpx.Client(base_url=client.base_url, timeout=client.timeout, trust_env=False) as fresh, ): first: Final = tuple( - pool.submit(_burst_call, client, label, path, body, key) for label, path, body, key in oversized + happy + pool.submit(_burst_call, client, label, path, body, key) + for label, path, body, key in oversized_calls + happy ) wait(first, return_when=FIRST_COMPLETED) during() @@ -387,9 +394,10 @@ def _burst( return tuple(future.result() for future in first + second) -def _slowest_oversized_refusal(results: tuple[_BurstResult, ...]) -> float: - refused: Final = tuple(result for result in results if result.label.startswith("oversized") and result.status) - assert refused and all(result.status == 400 for result in refused), refused +def _slowest_refusal(results: tuple[_BurstResult, ...], oversized: _Oversized) -> float: + refused: Final = tuple(result for result in results if result.label.startswith(oversized) and result.status) + assert refused, f"every {oversized} request dropped: {results}" + assert all(result.status == 400 for result in refused), refused return max(result.seconds for result in refused) @@ -425,16 +433,14 @@ def test_killing_one_worker_mid_burst_leaves_the_other_serving_crafted_and_happy workers: Final = owned.worker_pids() assert len(workers) == 2, workers victim: Final = min(workers) - control: Final = _slowest_oversized_refusal( - _burst(client, BENIGN_288KB_MODEL, model, key, open_key, lambda: None) - ) + control: Final = _slowest_refusal(_burst(client, "benign", model, key, open_key, lambda: None), "benign") with owned.replacing_worker(victim) as kill: - results: Final = _burst(client, CRAFTED_MODEL, model, key, open_key, kill) + results: Final = _burst(client, "crafted", model, key, open_key, kill) dropped: Final = tuple(result for result in results if result.status is None) assert len(dropped) < len(results), results assert_within_factor_of_control( - _slowest_oversized_refusal(results), control=control, factor=CRAFTED_SLOWDOWN_FACTOR + _slowest_refusal(results, "crafted"), control=control, factor=CRAFTED_BURST_SLOWDOWN_FACTOR ) late: Final = tuple(result for result in results if result.label.startswith("late")) assert all(result.status == 200 for result in late), late @@ -464,11 +470,9 @@ def test_upstream_returning_503_mid_burst_logs_every_failure_with_its_own_cluste model: Final = scenario.model(api_base=wire.url + "/v1", num_retries=0) key: Final = scenario.key(models=[model]) open_key: Final = scenario.key() - control: Final = _slowest_oversized_refusal( - _burst(client, BENIGN_288KB_MODEL, model, key, open_key, lambda: None) - ) + control: Final = _slowest_refusal(_burst(client, "benign", model, key, open_key, lambda: None), "benign") wire.drain() - results: Final = _burst(client, CRAFTED_MODEL, model, key, open_key, lambda: None) + results: Final = _burst(client, "crafted", model, key, open_key, lambda: None) assert all(result.status is not None for result in results), results happy: Final = tuple(result for result in results if result.label.startswith("happy")) assert all(result.status == 503 for result in happy), happy @@ -479,7 +483,7 @@ def test_upstream_returning_503_mid_burst_logs_every_failure_with_its_own_cluste seen ) assert_within_factor_of_control( - _slowest_oversized_refusal(results), control=control, factor=CRAFTED_SLOWDOWN_FACTOR + _slowest_refusal(results, "crafted"), control=control, factor=CRAFTED_BURST_SLOWDOWN_FACTOR ) codes: Final = { result.label: _error_information(string_value(result.call_id))["normalized_error"] for result in results @@ -487,7 +491,5 @@ def test_upstream_returning_503_mid_burst_logs_every_failure_with_its_own_cluste assert all(code == "503_PROVIDER_OVERLOADED" for label, code in codes.items() if label.startswith("happy")), ( codes ) - assert all(code == "400_INVALID_REQUEST" for label, code in codes.items() if label.startswith("oversized")), ( - codes - ) + assert all(code == "400_INVALID_REQUEST" for label, code in codes.items() if label.startswith("crafted")), codes assert client.get("/health/liveliness").status_code == 200 From 5a08b8e12d5bd450deeebad01cb49b5cb89366f3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:32:18 -0700 Subject: [PATCH 3/4] test(integration): compare crafted burst 400s with a short-name control burst A harmless 288 KB control paid any slowdown that every long model name pays, so the ratio hid it. A short unknown name keeps the control free of that cost, so the bar catches it. --- .../test_normalized_error_long_message.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/integration/spend/test_normalized_error_long_message.py b/tests/integration/spend/test_normalized_error_long_message.py index 5b8e50c2593..3a7a8c6b95e 100644 --- a/tests/integration/spend/test_normalized_error_long_message.py +++ b/tests/integration/spend/test_normalized_error_long_message.py @@ -28,9 +28,10 @@ pytestmark: Final = pytest.mark.shares_worker_state CRAFTED_MODEL: Final = ("exceeded " * 32_000)[:288_000] HOSTILE_5KB_MODEL: Final = ("exceeded budget " * 400)[:5_000] BENIGN_288KB_MODEL: Final = "x" * len(CRAFTED_MODEL) -_Oversized: TypeAlias = Literal["crafted", "benign"] -OVERSIZED_MODELS: Final[Mapping[_Oversized, str]] = MappingProxyType( - {"crafted": CRAFTED_MODEL, "benign": BENIGN_288KB_MODEL} +SHORT_UNKNOWN_MODEL: Final = "x" * 16 +_Refused: TypeAlias = Literal["crafted", "short"] +REFUSED_MODELS: Final[Mapping[_Refused, str]] = MappingProxyType( + {"crafted": CRAFTED_MODEL, "short": SHORT_UNKNOWN_MODEL} ) CRAFTED_SLOWDOWN_FACTOR: Final = 10 CRAFTED_BURST_SLOWDOWN_FACTOR: Final = 3 @@ -356,17 +357,17 @@ def _burst_call(client: httpx.Client, label: str, path: str, body: Mapping[str, def _burst( client: httpx.Client, - oversized: _Oversized, + refused: _Refused, happy_model: str, happy_key: str, open_key: str, during: Callable[[], None], ) -> tuple[_BurstResult, ...]: - oversized_calls: Final = tuple( + refused_calls: Final = tuple( ( - f"{oversized}-{path}-{index}", + f"{refused}-{path}-{index}", path, - _body(path, OVERSIZED_MODELS[oversized], uuid.uuid4().hex, stream=index % 2 == 1), + _body(path, REFUSED_MODELS[refused], uuid.uuid4().hex, stream=index % 2 == 1), open_key, ) for path in (CHAT, MESSAGES, RESPONSES) @@ -383,8 +384,7 @@ def _burst( httpx.Client(base_url=client.base_url, timeout=client.timeout, trust_env=False) as fresh, ): first: Final = tuple( - pool.submit(_burst_call, client, label, path, body, key) - for label, path, body, key in oversized_calls + happy + pool.submit(_burst_call, client, label, path, body, key) for label, path, body, key in refused_calls + happy ) wait(first, return_when=FIRST_COMPLETED) during() @@ -394,11 +394,11 @@ def _burst( return tuple(future.result() for future in first + second) -def _slowest_refusal(results: tuple[_BurstResult, ...], oversized: _Oversized) -> float: - refused: Final = tuple(result for result in results if result.label.startswith(oversized) and result.status) - assert refused, f"every {oversized} request dropped: {results}" - assert all(result.status == 400 for result in refused), refused - return max(result.seconds for result in refused) +def _slowest_refusal(results: tuple[_BurstResult, ...], refused: _Refused) -> float: + answered: Final = tuple(result for result in results if result.label.startswith(refused) and result.status) + assert answered, f"every {refused} request dropped: {results}" + assert all(result.status == 400 for result in answered), answered + return max(result.seconds for result in answered) def _assert_rows_land_exactly_once(results: tuple[_BurstResult, ...], prefix: str) -> None: @@ -433,7 +433,7 @@ def test_killing_one_worker_mid_burst_leaves_the_other_serving_crafted_and_happy workers: Final = owned.worker_pids() assert len(workers) == 2, workers victim: Final = min(workers) - control: Final = _slowest_refusal(_burst(client, "benign", model, key, open_key, lambda: None), "benign") + control: Final = _slowest_refusal(_burst(client, "short", model, key, open_key, lambda: None), "short") with owned.replacing_worker(victim) as kill: results: Final = _burst(client, "crafted", model, key, open_key, kill) @@ -470,7 +470,7 @@ def test_upstream_returning_503_mid_burst_logs_every_failure_with_its_own_cluste model: Final = scenario.model(api_base=wire.url + "/v1", num_retries=0) key: Final = scenario.key(models=[model]) open_key: Final = scenario.key() - control: Final = _slowest_refusal(_burst(client, "benign", model, key, open_key, lambda: None), "benign") + control: Final = _slowest_refusal(_burst(client, "short", model, key, open_key, lambda: None), "short") wire.drain() results: Final = _burst(client, "crafted", model, key, open_key, lambda: None) assert all(result.status is not None for result in results), results From 6c43244d3b4ffe94da8a8f473084fe848774b9d3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:41:54 -0700 Subject: [PATCH 4/4] test(integration): go back to the 288 KB control burst A short-name control burst answers in 0.2 to 0.4 s, while a benign 288 KB name alone costs about 4 s, so the crafted burst failed a 3x bound against it on an idle box. --- .../test_normalized_error_long_message.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/integration/spend/test_normalized_error_long_message.py b/tests/integration/spend/test_normalized_error_long_message.py index 3a7a8c6b95e..5b8e50c2593 100644 --- a/tests/integration/spend/test_normalized_error_long_message.py +++ b/tests/integration/spend/test_normalized_error_long_message.py @@ -28,10 +28,9 @@ pytestmark: Final = pytest.mark.shares_worker_state CRAFTED_MODEL: Final = ("exceeded " * 32_000)[:288_000] HOSTILE_5KB_MODEL: Final = ("exceeded budget " * 400)[:5_000] BENIGN_288KB_MODEL: Final = "x" * len(CRAFTED_MODEL) -SHORT_UNKNOWN_MODEL: Final = "x" * 16 -_Refused: TypeAlias = Literal["crafted", "short"] -REFUSED_MODELS: Final[Mapping[_Refused, str]] = MappingProxyType( - {"crafted": CRAFTED_MODEL, "short": SHORT_UNKNOWN_MODEL} +_Oversized: TypeAlias = Literal["crafted", "benign"] +OVERSIZED_MODELS: Final[Mapping[_Oversized, str]] = MappingProxyType( + {"crafted": CRAFTED_MODEL, "benign": BENIGN_288KB_MODEL} ) CRAFTED_SLOWDOWN_FACTOR: Final = 10 CRAFTED_BURST_SLOWDOWN_FACTOR: Final = 3 @@ -357,17 +356,17 @@ def _burst_call(client: httpx.Client, label: str, path: str, body: Mapping[str, def _burst( client: httpx.Client, - refused: _Refused, + oversized: _Oversized, happy_model: str, happy_key: str, open_key: str, during: Callable[[], None], ) -> tuple[_BurstResult, ...]: - refused_calls: Final = tuple( + oversized_calls: Final = tuple( ( - f"{refused}-{path}-{index}", + f"{oversized}-{path}-{index}", path, - _body(path, REFUSED_MODELS[refused], uuid.uuid4().hex, stream=index % 2 == 1), + _body(path, OVERSIZED_MODELS[oversized], uuid.uuid4().hex, stream=index % 2 == 1), open_key, ) for path in (CHAT, MESSAGES, RESPONSES) @@ -384,7 +383,8 @@ def _burst( httpx.Client(base_url=client.base_url, timeout=client.timeout, trust_env=False) as fresh, ): first: Final = tuple( - pool.submit(_burst_call, client, label, path, body, key) for label, path, body, key in refused_calls + happy + pool.submit(_burst_call, client, label, path, body, key) + for label, path, body, key in oversized_calls + happy ) wait(first, return_when=FIRST_COMPLETED) during() @@ -394,11 +394,11 @@ def _burst( return tuple(future.result() for future in first + second) -def _slowest_refusal(results: tuple[_BurstResult, ...], refused: _Refused) -> float: - answered: Final = tuple(result for result in results if result.label.startswith(refused) and result.status) - assert answered, f"every {refused} request dropped: {results}" - assert all(result.status == 400 for result in answered), answered - return max(result.seconds for result in answered) +def _slowest_refusal(results: tuple[_BurstResult, ...], oversized: _Oversized) -> float: + refused: Final = tuple(result for result in results if result.label.startswith(oversized) and result.status) + assert refused, f"every {oversized} request dropped: {results}" + assert all(result.status == 400 for result in refused), refused + return max(result.seconds for result in refused) def _assert_rows_land_exactly_once(results: tuple[_BurstResult, ...], prefix: str) -> None: @@ -433,7 +433,7 @@ def test_killing_one_worker_mid_burst_leaves_the_other_serving_crafted_and_happy workers: Final = owned.worker_pids() assert len(workers) == 2, workers victim: Final = min(workers) - control: Final = _slowest_refusal(_burst(client, "short", model, key, open_key, lambda: None), "short") + control: Final = _slowest_refusal(_burst(client, "benign", model, key, open_key, lambda: None), "benign") with owned.replacing_worker(victim) as kill: results: Final = _burst(client, "crafted", model, key, open_key, kill) @@ -470,7 +470,7 @@ def test_upstream_returning_503_mid_burst_logs_every_failure_with_its_own_cluste model: Final = scenario.model(api_base=wire.url + "/v1", num_retries=0) key: Final = scenario.key(models=[model]) open_key: Final = scenario.key() - control: Final = _slowest_refusal(_burst(client, "short", model, key, open_key, lambda: None), "short") + control: Final = _slowest_refusal(_burst(client, "benign", model, key, open_key, lambda: None), "benign") wire.drain() results: Final = _burst(client, "crafted", model, key, open_key, lambda: None) assert all(result.status is not None for result in results), results