diff --git a/tests/integration/observability/test_failure_redaction.py b/tests/integration/observability/test_failure_redaction.py index 2b745d8836d..87660dfbd2a 100644 --- a/tests/integration/observability/test_failure_redaction.py +++ b/tests/integration/observability/test_failure_redaction.py @@ -392,7 +392,10 @@ _SDK_CLIENTS: Final = ("openai_sync", "openai_async", "httpx") def _chat_call(rig: Rig, client: str, model: str, secret: str, stream: bool) -> str: messages: Final = [{"role": "user", "content": secret}] if client == "httpx": - response: Final = _chat(rig, model, messages) + response: Final = rig.proxy.request( + "POST", "/v1/chat/completions", {"model": model, "messages": messages, "stream": stream} + ) + response.read() assert response.status_code == 400, response.text return _call_id(response) if client == "openai_sync": @@ -448,8 +451,11 @@ def _messages_call(rig: Rig, client: str, model: str, secret: str, stream: bool) messages: Final = [{"role": "user", "content": secret}] if client == "httpx": response: Final = rig.proxy.request( - "POST", "/v1/messages", {"model": model, "max_tokens": 16, "messages": messages} + "POST", + "/v1/messages", + {"model": model, "max_tokens": 16, "messages": messages, "stream": stream}, ) + response.read() assert response.status_code == 400, response.text return _call_id(response) if client == "openai_sync": diff --git a/tests/integration/observability/test_failure_redaction_chaos.py b/tests/integration/observability/test_failure_redaction_chaos.py index fdd4056acf9..d49f23ac7c6 100644 --- a/tests/integration/observability/test_failure_redaction_chaos.py +++ b/tests/integration/observability/test_failure_redaction_chaos.py @@ -253,18 +253,29 @@ def test_g3_proxy_restart_mid_burst(tmp_path: Path) -> None: second: Final = _fire(rig_two, bodies[BURST // 2 :]) answered: Final = first + second + tuple(probes) assert all(status in (400, 500) for status, _ in answered), answered - assert any(status == 400 for status, _ in answered), answered - batches: Final = endpoint.drain() - empty: Final = tuple(batch for batch in batches if not _json_body_ok(batch.body)) - assert all(not batch.body.strip() for batch in empty), empty + post_restart_ids: Final = frozenset(cid for status, cid in second + tuple(probes) if status == 400 and cid) + assert post_restart_ids, answered + assert any(marker.encode() in request.body for request in provider.drain()) + collected: list[Request] = [] # mutable-ok: drain consumes batches, later polls keep earlier ones + + def landed() -> tuple[str, ...]: + collected.extend(endpoint.drain()) + return tuple( + str(event.get("litellm_call_id")) + for batch in collected + if _json_body_ok(batch.body) + for event in json.loads(batch.body) + ) + + landed_ids: Final = eventually(landed, lambda ids: post_restart_ids <= set(ids), seconds=60) + assert all(not batch.body.strip() for batch in collected if not _json_body_ok(batch.body)), collected + assert len(landed_ids) == len(set(landed_ids)), ("duplicate events after restart", landed_ids) events: Final = tuple( object_value(event) - for batch in batches + for batch in collected if _json_body_ok(batch.body) for event in json.loads(batch.body) ) - call_ids: Final = tuple(str(event.get("litellm_call_id")) for event in events if event) - assert len(call_ids) == len(set(call_ids)), ("duplicate events after restart", call_ids) for event in events: assert marker not in json.dumps(event), json.dumps(event)[:400] diff --git a/tests/integration/observability/test_failure_redaction_otel.py b/tests/integration/observability/test_failure_redaction_otel.py index 6e633fcf3c2..840650a4ede 100644 --- a/tests/integration/observability/test_failure_redaction_otel.py +++ b/tests/integration/observability/test_failure_redaction_otel.py @@ -202,10 +202,22 @@ def _span_for(rig: Rig, name: str, model: str) -> dict[str, JsonValue]: return spans[0] -def _auth_exception_span(rig: Rig) -> dict[str, JsonValue]: +def _auth_exception_span_ids(rig: Rig) -> frozenset[str]: + return frozenset( + str(span.get("spanId")) + for span in rig.sink.all() + if str(span.get("name", "")).startswith("auth") and _exception_events(span) + ) + + +def _auth_exception_span(rig: Rig, exclude: frozenset[str]) -> dict[str, JsonValue]: def found() -> tuple[dict[str, JsonValue], ...]: return tuple( - span for span in rig.sink.all() if str(span.get("name", "")).startswith("auth") and _exception_events(span) + span + for span in rig.sink.all() + if str(span.get("name", "")).startswith("auth") + and _exception_events(span) + and str(span.get("spanId")) not in exclude ) return eventually(found, lambda values: len(values) >= 1, seconds=260)[0] @@ -280,6 +292,7 @@ def test_c10_v2_auth_span_honors_header_opt_in(rig_v2_off: Rig) -> None: allowed: Final = scenario.model(api_base=rig_v2_off.provider.url + "/v1", api_key="synthetic-provider-key") denied: Final = scenario.model(api_base=rig_v2_off.provider.url + "/v1", api_key="synthetic-provider-key") key: Final = scenario.key(models=[allowed]) + before: Final = _auth_exception_span_ids(rig_v2_off) response: Final = rig_v2_off.proxy.request( "POST", "/v1/chat/completions", @@ -288,7 +301,7 @@ def test_c10_v2_auth_span_honors_header_opt_in(rig_v2_off: Rig) -> None: headers={"x-litellm-enable-message-redaction": "true"}, ) assert response.status_code in (400, 401, 403, 404), response.text - auth: Final = _auth_exception_span(rig_v2_off) + auth: Final = _auth_exception_span(rig_v2_off, before) assert allowed not in _exception_texts(auth), _exception_texts(auth)[:600] assert "redacted-by-litellm" in _exception_texts(auth), _exception_texts(auth)[:600] @@ -300,6 +313,7 @@ def test_c11_v2_auth_span_redacts_under_global_on(rig_v2_on: Rig) -> None: allowed: Final = scenario.model(api_base=rig_v2_on.provider.url + "/v1", api_key="synthetic-provider-key") denied: Final = scenario.model(api_base=rig_v2_on.provider.url + "/v1", api_key="synthetic-provider-key") key: Final = scenario.key(models=[allowed]) + before: Final = _auth_exception_span_ids(rig_v2_on) response: Final = rig_v2_on.proxy.request( "POST", "/v1/chat/completions", @@ -308,6 +322,6 @@ def test_c11_v2_auth_span_redacts_under_global_on(rig_v2_on: Rig) -> None: headers={"litellm-disable-message-redaction": "true"}, ) assert response.status_code in (400, 401, 403, 404), response.text - auth: Final = _auth_exception_span(rig_v2_on) + auth: Final = _auth_exception_span(rig_v2_on, before) assert allowed not in _exception_texts(auth), _exception_texts(auth)[:600] assert "redacted-by-litellm" in _exception_texts(auth), _exception_texts(auth)[:600]