diff --git a/tests/integration/observability/test_failure_redaction.py b/tests/integration/observability/test_failure_redaction.py index 87660dfbd2a..d3719ac49fc 100644 --- a/tests/integration/observability/test_failure_redaction.py +++ b/tests/integration/observability/test_failure_redaction.py @@ -15,7 +15,6 @@ import openai import pytest import yaml from integration._support.client import Gateway, JsonValue, eventually, gateway_from_environment, object_value -from integration._support.database import read_rows from integration._support.process import owned_proxy from integration._support.wire import Reply, Request, Wire, wire_server @@ -227,9 +226,9 @@ def _single_failure_event(rig: Rig, model: str) -> dict[str, JsonValue]: return events[0] -def _spend_row(call_id: str) -> dict[str, JsonValue]: +def _spend_row(proxy: Gateway, call_id: str) -> dict[str, JsonValue]: rows: Final = eventually( - lambda: read_rows( + lambda: proxy.rows( 'SELECT status, messages, response, proxy_server_request, metadata FROM "LiteLLM_SpendLogs" ' "WHERE request_id=%s", (call_id,), @@ -264,7 +263,7 @@ def test_provider_error_echoing_the_prompt_is_redacted_in_callbacks_and_spend_lo assert error_information["error_code"] == "400", error_information assert error_information["llm_provider"] == "openai", error_information assert "hidden_params" not in event, sorted(event) - row: Final = _spend_row(response.headers["x-litellm-call-id"]) + row: Final = _spend_row(rig.proxy, response.headers["x-litellm-call-id"]) assert row["status"] == "failure", row assert secret not in json.dumps(row, default=str), row persisted: Final = object_value( @@ -361,7 +360,7 @@ def _assert_failure_redacted(rig: Rig, model: str, secret: str, call_id: str) -> error_information: Final = _error_information(event) assert error_information["error_class"], error_information assert error_information["llm_provider"], error_information - row: Final = _spend_row(call_id) + row: Final = _spend_row(rig.proxy, call_id) assert row["status"] == "failure", row assert secret not in json.dumps(row, default=str), row return event @@ -371,7 +370,7 @@ def _assert_failure_raw(rig: Rig, model: str, secret: str, call_id: str | None = event: Final = _single_failure_event(rig, model) assert secret in json.dumps(event), json.dumps(event) if call_id is not None: - row: Final = _spend_row(call_id) + row: Final = _spend_row(rig.proxy, call_id) assert secret in json.dumps(row, default=str), row return event @@ -601,7 +600,7 @@ def test_a9_unknown_model_failure_redacted(rig: Rig) -> None: assert rig.provider.drain() == () event: Final = eventually(lambda: rig.failure_events(model), lambda values: len(values) >= 1, seconds=20)[0] assert secret not in json.dumps(event), json.dumps(event) - row: Final = _spend_row(_call_id(response)) + row: Final = _spend_row(rig.proxy, _call_id(response)) assert secret not in json.dumps(row, default=str), row @@ -904,7 +903,7 @@ def test_e4_header_only_unknown_model_spend_row_never_carries_marker(rig_off: Ri headers={"x-litellm-enable-message-redaction": "true"}, ) assert response.status_code == 400, response.text - row: Final = _spend_row(_call_id(response)) + row: Final = _spend_row(rig_off.proxy, _call_id(response)) assert secret not in json.dumps(row, default=str), row @@ -1107,4 +1106,4 @@ def test_f4_five_identical_failures_log_once_each(rig: Rig) -> None: lambda values: len(values) == 1, seconds=30, ) - assert _spend_row(call_id)["status"] == "failure", call_id + assert _spend_row(rig.proxy, call_id)["status"] == "failure", call_id diff --git a/tests/integration/observability/test_failure_redaction_chaos.py b/tests/integration/observability/test_failure_redaction_chaos.py index a925566dde6..3a8e8d1c773 100644 --- a/tests/integration/observability/test_failure_redaction_chaos.py +++ b/tests/integration/observability/test_failure_redaction_chaos.py @@ -11,7 +11,7 @@ import httpx import pytest import yaml from integration._support.client import Gateway, JsonValue, eventually, gateway_from_environment, object_value -from integration._support.process import OwnedProxy, owned_proxy_process +from integration._support.process import OwnedProxy, OwnedState, owned_proxy_process from integration._support.wire import Reply, Request, Wire, wire_server BURST: Final = 30 @@ -190,6 +190,7 @@ def test_g2_slow_sink_no_deadlock_no_duplicates(tmp_path: Path) -> None: @pytest.mark.timeout(280) def test_g3_proxy_restart_mid_burst(tmp_path: Path) -> None: marker: Final = uuid.uuid4().hex + model: Final = f"restart-model-{uuid.uuid4().hex}" with wire_server(_provider) as provider, wire_server(lambda _: Reply()) as endpoint: config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) config["litellm_settings"].update( @@ -199,12 +200,14 @@ def test_g3_proxy_restart_mid_burst(tmp_path: Path) -> None: path.write_text(yaml.safe_dump(config)) overrides: Final = {"GENERIC_LOGGER_ENDPOINT": endpoint.url} with gateway_from_environment() as gateway: - bodies: Final = _bodies("restart-model", marker) - with owned_proxy_process(gateway, tmp_path, overrides, config=path, workers=2) as owned_one: + bodies: Final = _bodies(model, marker) + with owned_proxy_process( + gateway, tmp_path, overrides, config=path, workers=2, state=OwnedState.SHARED_WITH_WORKER + ) as owned_one: owned_one.gateway.post( "/model/new", { - "model_name": "restart-model", + "model_name": model, "litellm_params": { "model": "openai/gpt-4o-mini", "api_base": provider.url + "/v1", @@ -218,20 +221,22 @@ def test_g3_proxy_restart_mid_burst(tmp_path: Path) -> None: owned_one, provider, endpoint, - "restart-model", + model, threading.Event(), threading.Event(), [], ), bodies[: BURST // 2], ) - with owned_proxy_process(gateway, tmp_path, overrides, config=path, workers=2) as owned_two: + with owned_proxy_process( + gateway, tmp_path, overrides, config=path, workers=2, state=OwnedState.SHARED_WITH_WORKER + ) as owned_two: rig_two: Final = Rig( owned_two.gateway, owned_two, provider, endpoint, - "restart-model", + model, threading.Event(), threading.Event(), [], @@ -249,33 +254,35 @@ def test_g3_proxy_restart_mid_burst(tmp_path: Path) -> None: eventually(served, lambda statuses: statuses[0] == 400, seconds=60) second: Final = _fire(rig_two, bodies[BURST // 2 :]) - answered: Final = first + second + tuple(probes) - assert all(status in (400, 500) for status, _ in answered), answered - post_restart_ids: Final = frozenset(cid for status, cid in second + tuple(probes) if status == 400 and cid) - assert post_restart_ids, answered - assert any(marker.encode() in request.body for request in provider.drain()) - collected: list[Request] = [] # mutable-ok: drain consumes batches, later polls keep earlier ones + answered: Final = first + second + tuple(probes) + assert all(status in (400, 500) for status, _ in answered), answered + post_restart_ids: Final = frozenset( + cid for status, cid in second + tuple(probes) if status == 400 and cid + ) + assert post_restart_ids, answered + assert any(marker.encode() in request.body for request in provider.drain()) + collected: list[Request] = [] # mutable-ok: drain consumes batches, later polls keep earlier ones - def landed() -> tuple[str, ...]: - collected.extend(endpoint.drain()) - return tuple( - str(event.get("litellm_call_id")) + def landed() -> tuple[str, ...]: + collected.extend(endpoint.drain()) + return tuple( + str(event.get("litellm_call_id")) + for batch in collected + if _json_body_ok(batch.body) + for event in json.loads(batch.body) + ) + + landed_ids: Final = eventually(landed, lambda ids: post_restart_ids <= set(ids), seconds=60) + assert all(not batch.body.strip() for batch in collected if not _json_body_ok(batch.body)), collected + assert len(landed_ids) == len(set(landed_ids)), ("duplicate events after restart", landed_ids) + events: Final = tuple( + object_value(event) for batch in collected if _json_body_ok(batch.body) for event in json.loads(batch.body) ) - - landed_ids: Final = eventually(landed, lambda ids: post_restart_ids <= set(ids), seconds=60) - assert all(not batch.body.strip() for batch in collected if not _json_body_ok(batch.body)), collected - assert len(landed_ids) == len(set(landed_ids)), ("duplicate events after restart", landed_ids) - events: Final = tuple( - object_value(event) - for batch in collected - if _json_body_ok(batch.body) - for event in json.loads(batch.body) - ) - for event in events: - assert marker not in json.dumps(event), json.dumps(event)[:400] + for event in events: + assert marker not in json.dumps(event), json.dumps(event)[:400] @pytest.mark.timeout(280)