From 89ac9b3ce4d2fa866d05e34a5b7fb3ade339aed7 Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:50:42 -0700 Subject: [PATCH] test(integration): read failure spend rows from the owned proxy database and keep the restart cell on shared state Main now gives each owned proxy a private database, so the spend row reads go through the proxy gateway. The restart cell boots both proxies on the worker state so the second one keeps the stored model, and it waits for the post-restart events while that proxy is still up since batch loggers do not flush on shutdown --- .../observability/test_failure_redaction.py | 17 +++-- .../test_failure_redaction_chaos.py | 65 ++++++++++--------- 2 files changed, 44 insertions(+), 38 deletions(-) diff --git a/tests/integration/observability/test_failure_redaction.py b/tests/integration/observability/test_failure_redaction.py index 87660dfbd2a..d3719ac49fc 100644 --- a/tests/integration/observability/test_failure_redaction.py +++ b/tests/integration/observability/test_failure_redaction.py @@ -15,7 +15,6 @@ import openai import pytest import yaml from integration._support.client import Gateway, JsonValue, eventually, gateway_from_environment, object_value -from integration._support.database import read_rows from integration._support.process import owned_proxy from integration._support.wire import Reply, Request, Wire, wire_server @@ -227,9 +226,9 @@ def _single_failure_event(rig: Rig, model: str) -> dict[str, JsonValue]: return events[0] -def _spend_row(call_id: str) -> dict[str, JsonValue]: +def _spend_row(proxy: Gateway, call_id: str) -> dict[str, JsonValue]: rows: Final = eventually( - lambda: read_rows( + lambda: proxy.rows( 'SELECT status, messages, response, proxy_server_request, metadata FROM "LiteLLM_SpendLogs" ' "WHERE request_id=%s", (call_id,), @@ -264,7 +263,7 @@ def test_provider_error_echoing_the_prompt_is_redacted_in_callbacks_and_spend_lo assert error_information["error_code"] == "400", error_information assert error_information["llm_provider"] == "openai", error_information assert "hidden_params" not in event, sorted(event) - row: Final = _spend_row(response.headers["x-litellm-call-id"]) + row: Final = _spend_row(rig.proxy, response.headers["x-litellm-call-id"]) assert row["status"] == "failure", row assert secret not in json.dumps(row, default=str), row persisted: Final = object_value( @@ -361,7 +360,7 @@ def _assert_failure_redacted(rig: Rig, model: str, secret: str, call_id: str) -> error_information: Final = _error_information(event) assert error_information["error_class"], error_information assert error_information["llm_provider"], error_information - row: Final = _spend_row(call_id) + row: Final = _spend_row(rig.proxy, call_id) assert row["status"] == "failure", row assert secret not in json.dumps(row, default=str), row return event @@ -371,7 +370,7 @@ def _assert_failure_raw(rig: Rig, model: str, secret: str, call_id: str | None = event: Final = _single_failure_event(rig, model) assert secret in json.dumps(event), json.dumps(event) if call_id is not None: - row: Final = _spend_row(call_id) + row: Final = _spend_row(rig.proxy, call_id) assert secret in json.dumps(row, default=str), row return event @@ -601,7 +600,7 @@ def test_a9_unknown_model_failure_redacted(rig: Rig) -> None: assert rig.provider.drain() == () event: Final = eventually(lambda: rig.failure_events(model), lambda values: len(values) >= 1, seconds=20)[0] assert secret not in json.dumps(event), json.dumps(event) - row: Final = _spend_row(_call_id(response)) + row: Final = _spend_row(rig.proxy, _call_id(response)) assert secret not in json.dumps(row, default=str), row @@ -904,7 +903,7 @@ def test_e4_header_only_unknown_model_spend_row_never_carries_marker(rig_off: Ri headers={"x-litellm-enable-message-redaction": "true"}, ) assert response.status_code == 400, response.text - row: Final = _spend_row(_call_id(response)) + row: Final = _spend_row(rig_off.proxy, _call_id(response)) assert secret not in json.dumps(row, default=str), row @@ -1107,4 +1106,4 @@ def test_f4_five_identical_failures_log_once_each(rig: Rig) -> None: lambda values: len(values) == 1, seconds=30, ) - assert _spend_row(call_id)["status"] == "failure", call_id + assert _spend_row(rig.proxy, call_id)["status"] == "failure", call_id diff --git a/tests/integration/observability/test_failure_redaction_chaos.py b/tests/integration/observability/test_failure_redaction_chaos.py index a925566dde6..3a8e8d1c773 100644 --- a/tests/integration/observability/test_failure_redaction_chaos.py +++ b/tests/integration/observability/test_failure_redaction_chaos.py @@ -11,7 +11,7 @@ import httpx import pytest import yaml from integration._support.client import Gateway, JsonValue, eventually, gateway_from_environment, object_value -from integration._support.process import OwnedProxy, owned_proxy_process +from integration._support.process import OwnedProxy, OwnedState, owned_proxy_process from integration._support.wire import Reply, Request, Wire, wire_server BURST: Final = 30 @@ -190,6 +190,7 @@ def test_g2_slow_sink_no_deadlock_no_duplicates(tmp_path: Path) -> None: @pytest.mark.timeout(280) def test_g3_proxy_restart_mid_burst(tmp_path: Path) -> None: marker: Final = uuid.uuid4().hex + model: Final = f"restart-model-{uuid.uuid4().hex}" with wire_server(_provider) as provider, wire_server(lambda _: Reply()) as endpoint: config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) config["litellm_settings"].update( @@ -199,12 +200,14 @@ def test_g3_proxy_restart_mid_burst(tmp_path: Path) -> None: path.write_text(yaml.safe_dump(config)) overrides: Final = {"GENERIC_LOGGER_ENDPOINT": endpoint.url} with gateway_from_environment() as gateway: - bodies: Final = _bodies("restart-model", marker) - with owned_proxy_process(gateway, tmp_path, overrides, config=path, workers=2) as owned_one: + bodies: Final = _bodies(model, marker) + with owned_proxy_process( + gateway, tmp_path, overrides, config=path, workers=2, state=OwnedState.SHARED_WITH_WORKER + ) as owned_one: owned_one.gateway.post( "/model/new", { - "model_name": "restart-model", + "model_name": model, "litellm_params": { "model": "openai/gpt-4o-mini", "api_base": provider.url + "/v1", @@ -218,20 +221,22 @@ def test_g3_proxy_restart_mid_burst(tmp_path: Path) -> None: owned_one, provider, endpoint, - "restart-model", + model, threading.Event(), threading.Event(), [], ), bodies[: BURST // 2], ) - with owned_proxy_process(gateway, tmp_path, overrides, config=path, workers=2) as owned_two: + with owned_proxy_process( + gateway, tmp_path, overrides, config=path, workers=2, state=OwnedState.SHARED_WITH_WORKER + ) as owned_two: rig_two: Final = Rig( owned_two.gateway, owned_two, provider, endpoint, - "restart-model", + model, threading.Event(), threading.Event(), [], @@ -249,33 +254,35 @@ def test_g3_proxy_restart_mid_burst(tmp_path: Path) -> None: eventually(served, lambda statuses: statuses[0] == 400, seconds=60) second: Final = _fire(rig_two, bodies[BURST // 2 :]) - answered: Final = first + second + tuple(probes) - assert all(status in (400, 500) for status, _ in answered), answered - post_restart_ids: Final = frozenset(cid for status, cid in second + tuple(probes) if status == 400 and cid) - assert post_restart_ids, answered - assert any(marker.encode() in request.body for request in provider.drain()) - collected: list[Request] = [] # mutable-ok: drain consumes batches, later polls keep earlier ones + answered: Final = first + second + tuple(probes) + assert all(status in (400, 500) for status, _ in answered), answered + post_restart_ids: Final = frozenset( + cid for status, cid in second + tuple(probes) if status == 400 and cid + ) + assert post_restart_ids, answered + assert any(marker.encode() in request.body for request in provider.drain()) + collected: list[Request] = [] # mutable-ok: drain consumes batches, later polls keep earlier ones - def landed() -> tuple[str, ...]: - collected.extend(endpoint.drain()) - return tuple( - str(event.get("litellm_call_id")) + def landed() -> tuple[str, ...]: + collected.extend(endpoint.drain()) + return tuple( + str(event.get("litellm_call_id")) + for batch in collected + if _json_body_ok(batch.body) + for event in json.loads(batch.body) + ) + + landed_ids: Final = eventually(landed, lambda ids: post_restart_ids <= set(ids), seconds=60) + assert all(not batch.body.strip() for batch in collected if not _json_body_ok(batch.body)), collected + assert len(landed_ids) == len(set(landed_ids)), ("duplicate events after restart", landed_ids) + events: Final = tuple( + object_value(event) for batch in collected if _json_body_ok(batch.body) for event in json.loads(batch.body) ) - - landed_ids: Final = eventually(landed, lambda ids: post_restart_ids <= set(ids), seconds=60) - assert all(not batch.body.strip() for batch in collected if not _json_body_ok(batch.body)), collected - assert len(landed_ids) == len(set(landed_ids)), ("duplicate events after restart", landed_ids) - events: Final = tuple( - object_value(event) - for batch in collected - if _json_body_ok(batch.body) - for event in json.loads(batch.body) - ) - for event in events: - assert marker not in json.dumps(event), json.dumps(event)[:400] + for event in events: + assert marker not in json.dumps(event), json.dumps(event)[:400] @pytest.mark.timeout(280)