scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

test(integration): tighten audit cells per review findings

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yucheng 2026-09-29 21:21:38 +00:00
parent db04c2fd1b
commit 8aee22325a
3 changed files with 44 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
tests/integration/observability/test_failure_redaction.py
View file

@ -392,7 +392,10 @@ _SDK_CLIENTS: Final = ("openai_sync", "openai_async", "httpx")
def _chat_call(rig: Rig, client: str, model: str, secret: str, stream: bool) -> str:
messages: Final = [{"role": "user", "content": secret}]
if client == "httpx":
response: Final = _chat(rig, model, messages)
response: Final = rig.proxy.request(
"POST", "/v1/chat/completions", {"model": model, "messages": messages, "stream": stream}
)
response.read()
assert response.status_code == 400, response.text
return _call_id(response)
if client == "openai_sync":
@ -448,8 +451,11 @@ def _messages_call(rig: Rig, client: str, model: str, secret: str, stream: bool)
messages: Final = [{"role": "user", "content": secret}]
if client == "httpx":
response: Final = rig.proxy.request(
"POST", "/v1/messages", {"model": model, "max_tokens": 16, "messages": messages}
"POST",
"/v1/messages",
{"model": model, "max_tokens": 16, "messages": messages, "stream": stream},
)
response.read()
assert response.status_code == 400, response.text
return _call_id(response)
if client == "openai_sync":

25
tests/integration/observability/test_failure_redaction_chaos.py
View file

@ -253,18 +253,29 @@ def test_g3_proxy_restart_mid_burst(tmp_path: Path) -> None:
second: Final = _fire(rig_two, bodies[BURST // 2 :])
answered: Final = first + second + tuple(probes)
assert all(status in (400, 500) for status, _ in answered), answered
assert any(status == 400 for status, _ in answered), answered
batches: Final = endpoint.drain()
empty: Final = tuple(batch for batch in batches if not _json_body_ok(batch.body))
assert all(not batch.body.strip() for batch in empty), empty
post_restart_ids: Final = frozenset(cid for status, cid in second + tuple(probes) if status == 400 and cid)
assert post_restart_ids, answered
assert any(marker.encode() in request.body for request in provider.drain())
collected: list[Request] = [] # mutable-ok: drain consumes batches, later polls keep earlier ones
def landed() -> tuple[str, ...]:
collected.extend(endpoint.drain())
return tuple(
str(event.get("litellm_call_id"))
for batch in collected
if _json_body_ok(batch.body)
for event in json.loads(batch.body)
)
landed_ids: Final = eventually(landed, lambda ids: post_restart_ids <= set(ids), seconds=60)
assert all(not batch.body.strip() for batch in collected if not _json_body_ok(batch.body)), collected
assert len(landed_ids) == len(set(landed_ids)), ("duplicate events after restart", landed_ids)
events: Final = tuple(
object_value(event)
for batch in batches
for batch in collected
if _json_body_ok(batch.body)
for event in json.loads(batch.body)
)
call_ids: Final = tuple(str(event.get("litellm_call_id")) for event in events if event)
assert len(call_ids) == len(set(call_ids)), ("duplicate events after restart", call_ids)
for event in events:
assert marker not in json.dumps(event), json.dumps(event)[:400]

22
tests/integration/observability/test_failure_redaction_otel.py
View file

@ -202,10 +202,22 @@ def _span_for(rig: Rig, name: str, model: str) -> dict[str, JsonValue]:
return spans[0]
def _auth_exception_span(rig: Rig) -> dict[str, JsonValue]:
def _auth_exception_span_ids(rig: Rig) -> frozenset[str]:
return frozenset(
str(span.get("spanId"))
for span in rig.sink.all()
if str(span.get("name", "")).startswith("auth") and _exception_events(span)
)
def _auth_exception_span(rig: Rig, exclude: frozenset[str]) -> dict[str, JsonValue]:
def found() -> tuple[dict[str, JsonValue], ...]:
return tuple(
span for span in rig.sink.all() if str(span.get("name", "")).startswith("auth") and _exception_events(span)
span
for span in rig.sink.all()
if str(span.get("name", "")).startswith("auth")
and _exception_events(span)
and str(span.get("spanId")) not in exclude
)
return eventually(found, lambda values: len(values) >= 1, seconds=260)[0]
@ -280,6 +292,7 @@ def test_c10_v2_auth_span_honors_header_opt_in(rig_v2_off: Rig) -> None:
allowed: Final = scenario.model(api_base=rig_v2_off.provider.url + "/v1", api_key="synthetic-provider-key")
denied: Final = scenario.model(api_base=rig_v2_off.provider.url + "/v1", api_key="synthetic-provider-key")
key: Final = scenario.key(models=[allowed])
before: Final = _auth_exception_span_ids(rig_v2_off)
response: Final = rig_v2_off.proxy.request(
"POST",
"/v1/chat/completions",
@ -288,7 +301,7 @@ def test_c10_v2_auth_span_honors_header_opt_in(rig_v2_off: Rig) -> None:
headers={"x-litellm-enable-message-redaction": "true"},
)
assert response.status_code in (400, 401, 403, 404), response.text
auth: Final = _auth_exception_span(rig_v2_off)
auth: Final = _auth_exception_span(rig_v2_off, before)
assert allowed not in _exception_texts(auth), _exception_texts(auth)[:600]
assert "redacted-by-litellm" in _exception_texts(auth), _exception_texts(auth)[:600]
@ -300,6 +313,7 @@ def test_c11_v2_auth_span_redacts_under_global_on(rig_v2_on: Rig) -> None:
allowed: Final = scenario.model(api_base=rig_v2_on.provider.url + "/v1", api_key="synthetic-provider-key")
denied: Final = scenario.model(api_base=rig_v2_on.provider.url + "/v1", api_key="synthetic-provider-key")
key: Final = scenario.key(models=[allowed])
before: Final = _auth_exception_span_ids(rig_v2_on)
response: Final = rig_v2_on.proxy.request(
"POST",
"/v1/chat/completions",
@ -308,6 +322,6 @@ def test_c11_v2_auth_span_redacts_under_global_on(rig_v2_on: Rig) -> None:
headers={"litellm-disable-message-redaction": "true"},
)
assert response.status_code in (400, 401, 403, 404), response.text
auth: Final = _auth_exception_span(rig_v2_on)
auth: Final = _auth_exception_span(rig_v2_on, before)
assert allowed not in _exception_texts(auth), _exception_texts(auth)[:600]
assert "redacted-by-litellm" in _exception_texts(auth), _exception_texts(auth)[:600]