From 0a813aecee081a8210e247ec694d8ead479fd8e5 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 27 Oct 2026 01:31:51 -0700 Subject: [PATCH] test(integration): tie the auth span to its request trace and prove post-restart calls reach the provider --- .../test_failure_redaction_chaos.py | 1 + .../test_failure_redaction_otel.py | 34 +++++++++---------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/tests/integration/observability/test_failure_redaction_chaos.py b/tests/integration/observability/test_failure_redaction_chaos.py index 3a8e8d1c773..e58bad18683 100644 --- a/tests/integration/observability/test_failure_redaction_chaos.py +++ b/tests/integration/observability/test_failure_redaction_chaos.py @@ -253,6 +253,7 @@ def test_g3_proxy_restart_mid_burst(tmp_path: Path) -> None: return (probe.status_code,) eventually(served, lambda statuses: statuses[0] == 400, seconds=60) + provider.drain() second: Final = _fire(rig_two, bodies[BURST // 2 :]) answered: Final = first + second + tuple(probes) assert all(status in (400, 500) for status, _ in answered), answered diff --git a/tests/integration/observability/test_failure_redaction_otel.py b/tests/integration/observability/test_failure_redaction_otel.py index 64baaa67687..6ce8aed69a5 100644 --- a/tests/integration/observability/test_failure_redaction_otel.py +++ b/tests/integration/observability/test_failure_redaction_otel.py @@ -202,22 +202,14 @@ def _span_for(rig: Rig, name: str, model: str) -> dict[str, JsonValue]: return spans[0] -def _auth_exception_span_ids(rig: Rig) -> frozenset[str]: - return frozenset( - str(span.get("spanId")) - for span in rig.sink.all() - if str(span.get("name", "")).startswith("auth") and _exception_events(span) - ) - - -def _auth_exception_span(rig: Rig, exclude: frozenset[str]) -> dict[str, JsonValue]: +def _auth_exception_span(rig: Rig, trace_id: str) -> dict[str, JsonValue]: def found() -> tuple[dict[str, JsonValue], ...]: return tuple( span for span in rig.sink.all() - if str(span.get("name", "")).startswith("auth") + if span.get("traceId") == trace_id + and str(span.get("name", "")).startswith("auth") and _exception_events(span) - and str(span.get("spanId")) not in exclude ) return eventually(found, lambda values: len(values) >= 1, seconds=260)[0] @@ -295,17 +287,21 @@ def test_c10_v2_auth_span_honors_header_opt_in(rig_v2_off: Rig) -> None: allowed: Final = scenario.model(api_base=rig_v2_off.provider.url + "/v1", api_key="synthetic-provider-key") denied: Final = scenario.model(api_base=rig_v2_off.provider.url + "/v1", api_key="synthetic-provider-key") key: Final = scenario.key(models=[allowed]) - before: Final = _auth_exception_span_ids(rig_v2_off) + trace_id: Final = uuid.uuid4().hex response: Final = rig_v2_off.proxy.request( "POST", "/v1/chat/completions", {"model": denied, "messages": [{"role": "user", "content": "hi"}]}, key=key, - headers={"x-litellm-enable-message-redaction": "true"}, + headers={ + "x-litellm-enable-message-redaction": "true", + "traceparent": f"00-{trace_id}-{uuid.uuid4().hex[:16]}-01", + }, ) assert response.status_code in (400, 401, 403, 404), response.text - auth: Final = _auth_exception_span(rig_v2_off, before) + auth: Final = _auth_exception_span(rig_v2_off, trace_id) assert allowed not in _exception_texts(auth), _exception_texts(auth)[:600] + assert denied not in _exception_texts(auth), _exception_texts(auth)[:600] assert "redacted-by-litellm" in _exception_texts(auth), _exception_texts(auth)[:600] @@ -316,15 +312,19 @@ def test_c11_v2_auth_span_redacts_under_global_on(rig_v2_on: Rig) -> None: allowed: Final = scenario.model(api_base=rig_v2_on.provider.url + "/v1", api_key="synthetic-provider-key") denied: Final = scenario.model(api_base=rig_v2_on.provider.url + "/v1", api_key="synthetic-provider-key") key: Final = scenario.key(models=[allowed]) - before: Final = _auth_exception_span_ids(rig_v2_on) + trace_id: Final = uuid.uuid4().hex response: Final = rig_v2_on.proxy.request( "POST", "/v1/chat/completions", {"model": denied, "messages": [{"role": "user", "content": "hi"}]}, key=key, - headers={"litellm-disable-message-redaction": "true"}, + headers={ + "litellm-disable-message-redaction": "true", + "traceparent": f"00-{trace_id}-{uuid.uuid4().hex[:16]}-01", + }, ) assert response.status_code in (400, 401, 403, 404), response.text - auth: Final = _auth_exception_span(rig_v2_on, before) + auth: Final = _auth_exception_span(rig_v2_on, trace_id) assert allowed not in _exception_texts(auth), _exception_texts(auth)[:600] + assert denied not in _exception_texts(auth), _exception_texts(auth)[:600] assert "redacted-by-litellm" in _exception_texts(auth), _exception_texts(auth)[:600]