diff --git a/tests/integration/observability/conftest.py b/tests/integration/observability/conftest.py index c5150877857..fcac8eaba3c 100644 --- a/tests/integration/observability/conftest.py +++ b/tests/integration/observability/conftest.py @@ -8,8 +8,9 @@ from urllib.parse import urlparse import pytest import yaml +from integration._support.client import eventually from integration._support.otlp_sink import SpanSinks, owned_sinks -from integration._support.prometheus_series import CapRig, series_cap_rig +from integration._support.prometheus_series import CapRig, series_cap_rig, spend_rows from pydantic import JsonValue AuditConfigWriter = Callable[[Path, Mapping[str, JsonValue]], Path] @@ -63,4 +64,9 @@ def capped(tmp_path_factory: pytest.TempPathFactory) -> Iterator[CapRig]: workers=2, warm_keys=3, ) as rig: + eventually( + lambda: tuple(len(spend_rows(key.alias)) for key in rig.warm), + lambda counts: all(count == 1 for count in counts), + seconds=70, + ) yield rig diff --git a/tests/integration/observability/test_cache_hit_guardrail_metrics_chaos.py b/tests/integration/observability/test_cache_hit_guardrail_metrics_chaos.py index c77b8eebe33..7e3b479e5f0 100644 --- a/tests/integration/observability/test_cache_hit_guardrail_metrics_chaos.py +++ b/tests/integration/observability/test_cache_hit_guardrail_metrics_chaos.py @@ -263,7 +263,7 @@ def test_worker_kill_mid_burst_keeps_counting(gateway: Gateway, tmp_path: Path) def test_proxy_restart_mid_burst_keeps_counting(gateway: Gateway, tmp_path: Path) -> None: - """X4: restart the owned proxy between the two halves; pre-restart count asserted, then recounted.""" + """X4: the boot wipes the kept directory, so the second proxy counts only the second half.""" marker: Final = uuid.uuid4().hex prom_dir: Final = tmp_path / "prom" prom_dir.mkdir() @@ -321,8 +321,8 @@ def test_proxy_restart_mid_burst_keeps_counting(gateway: Gateway, tmp_path: Path _populated(_samples(owned_two.gateway, (model,)), deployment), _blank(_samples(owned_two.gateway, (model,))), ), - lambda observed: observed[0] == len(named) and observed[1] == 0, + lambda observed: observed[0] == len(second_half) and observed[1] == 0, seconds=70, ) - assert post[0] == len(named), (pre, post, outcomes_two) + assert post[0] == len(second_half), (pre, post, outcomes_two) owned_two.gateway.post("/model/delete", {"id": deployment})