mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-12 03:48:56 +00:00
048a1500df
* test(e2e): move the harness self-tests out of tests/e2e The nightly Buildkite run copies tests/e2e into the runner image and runs bare pytest, so the 672 tests of the harness itself (fixture parsing, JUnit properties, the stack lock, the load aggregators, the Claude Code driver) counted as e2e tests on the status page even though none of them reaches a proxy. They now live in tests/e2e_harness, mirroring the tests/e2e layout, and run in the GitHub Actions lint job and the CircleCI provider_replay_harness job instead * fix(ci): point the providers replay controls at tests/e2e_harness The providers integration job still selected the four replay-control tests under tests/e2e/test_provider_edge.py, so pytest exited before they ran. The raw-HTTP check's file walk also drops to one loop per comprehension * style(tests): mark the raw-HTTP check's bindings Final
331 lines
17 KiB
Python
331 lines
17 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
from pathlib import Path
|
|
from typing import Final
|
|
|
|
import pytest
|
|
from fixture_bundle import BundleRecorder, LoadedBundle, load_bundle, prepare_bundle
|
|
from fixture_mode import current_test_key
|
|
from fixture_profile import MatchProfile
|
|
from provider_edge import REPLAY_MISS_STATUS, RecordEdge, ReplayEdge, ReplaySource
|
|
from test_provider_edge import (
|
|
CHAT_PATH,
|
|
SSE_CHUNKS,
|
|
STREAM_BODY,
|
|
UPLOAD_PATH,
|
|
call_edge,
|
|
chunked_provider,
|
|
fake_provider,
|
|
json_object,
|
|
provider_url,
|
|
raw_stream_post,
|
|
running_edge,
|
|
this_tests_files,
|
|
)
|
|
|
|
|
|
class TestStrictIdentity:
|
|
@pytest.mark.parametrize("path", [CHAT_PATH, "/anthropic/v1/messages"])
|
|
def test_roundtrip_rejects_semantic_changes(self, tmp_path: Path, path: str) -> None:
|
|
recorder: Final = prepare_bundle(tmp_path / "strict", profile="stateless_v1")
|
|
assert isinstance(recorder, BundleRecorder)
|
|
original: Final = (
|
|
b'{"model":"synthetic","messages":[{"role":"user",'
|
|
b'"content":"2031-04-05 00000000-0000-0000-0000-000000000001"}],"options":[1,2]}'
|
|
)
|
|
headers: Final = {
|
|
"content-type": "application/json",
|
|
"accept": "application/json",
|
|
"anthropic-version": "2023-06-01",
|
|
"anthropic-beta": "feature-a",
|
|
"openai-beta": "feature-b",
|
|
"authorization": "Bearer synthetic-secret-one",
|
|
}
|
|
query: Final = "?part=one&part=two&blank="
|
|
with fake_provider() as provider:
|
|
mounts: Final = {"openai": provider_url(provider), "anthropic": provider_url(provider)}
|
|
with running_edge(RecordEdge(recorder, threading.Lock()), mounts) as edge:
|
|
captured: Final = call_edge(edge, "POST", path + query, body=original, headers=headers)
|
|
assert captured.status_code == 200
|
|
assert json_object(captured.body)["echo"] == original.decode()
|
|
loaded: Final = load_bundle(recorder.root, profile="stateless_v1")
|
|
assert isinstance(loaded, LoadedBundle)
|
|
assert loaded.manifest.match_profile == "stateless_v1"
|
|
source: Final = ReplaySource(loaded)
|
|
with running_edge(ReplayEdge(source), mounts) as edge:
|
|
cases: Final = (
|
|
(original.replace(b"2031-04-05", b"2032-06-07"), headers, query, "body"),
|
|
(original.replace(b"000000000001", b"000000000002"), headers, query, "body"),
|
|
(original.replace(b"[1,2]", b"[2,1]"), headers, query, "body"),
|
|
(original.replace(b"synthetic", b"other"), headers, query, "body"),
|
|
(original, headers, "?part=three&part=two&blank=", "query"),
|
|
(original, headers, "?part=two&part=one&blank=", "query"),
|
|
*(
|
|
(original, {k: v for k, v in headers.items() if k != name}, query, "headers")
|
|
for name in ("accept", "anthropic-version", "anthropic-beta", "openai-beta")
|
|
),
|
|
*(
|
|
(original, {**headers, name: value}, query, "headers")
|
|
for name in ("accept", "anthropic-version", "anthropic-beta", "openai-beta")
|
|
for value in ("different", "")
|
|
),
|
|
(original, {**headers, "authorization": "Basic synthetic-secret-two"}, query, "auth"),
|
|
(original, {k: v for k, v in headers.items() if k != "authorization"}, query, "auth"),
|
|
)
|
|
for rejected, reason in (
|
|
(call_edge(edge, "POST", path + changed_query, body=body, headers=changed_headers), reason)
|
|
for body, changed_headers, changed_query, reason in cases
|
|
):
|
|
assert rejected.status_code == REPLAY_MISS_STATUS
|
|
assert reason in rejected.body.decode()
|
|
assert b"synthetic-secret" not in rejected.body
|
|
reordered: Final = json.dumps(dict(reversed(list(json_object(original).items())))).encode()
|
|
accepted: Final = call_edge(
|
|
edge, "POST", path + query, body=reordered, headers={k.upper(): v for k, v in headers.items()}
|
|
)
|
|
assert accepted.status_code == 200
|
|
assert accepted.body == captured.body
|
|
assert source.leftover_error(current_test_key()) is None
|
|
assert len(provider.hits) == 1
|
|
assert "synthetic-secret" not in "".join(file.read_text() for file in recorder.root.rglob("*.json"))
|
|
|
|
@pytest.mark.parametrize(
|
|
"body",
|
|
[
|
|
b'{"value":null}',
|
|
b'{"value":""}',
|
|
b'{"value":false}',
|
|
b'{"value":0}',
|
|
b'{"value":[]}',
|
|
b'{"value":{}}',
|
|
b'{"value":0.123456789012345678901}',
|
|
b'{"value":0.123456789012345678902}',
|
|
b'{"value":1e400}',
|
|
b'{"value":1}',
|
|
b'{"value":1e0}',
|
|
b'{"value":-0}',
|
|
b'{"value":1e9999999999999999999}',
|
|
],
|
|
)
|
|
def test_json_values_remain_distinct(self, tmp_path: Path, body: bytes) -> None:
|
|
recorder: Final = prepare_bundle(tmp_path / "strict", profile="stateless_v1")
|
|
assert isinstance(recorder, BundleRecorder)
|
|
with fake_provider() as provider:
|
|
mounts: Final = {"openai": provider_url(provider)}
|
|
with running_edge(RecordEdge(recorder, threading.Lock()), mounts) as edge:
|
|
assert (
|
|
call_edge(
|
|
edge, "POST", CHAT_PATH, body=body, headers={"content-type": "application/json"}
|
|
).status_code
|
|
== 200
|
|
)
|
|
loaded: Final = load_bundle(recorder.root, profile="stateless_v1")
|
|
assert isinstance(loaded, LoadedBundle)
|
|
with running_edge(ReplayEdge(ReplaySource(loaded)), mounts) as edge:
|
|
values: Final = (
|
|
b"{}",
|
|
b'{"value":null}',
|
|
b'{"value":""}',
|
|
b'{"value":false}',
|
|
b'{"value":0}',
|
|
b'{"value":[]}',
|
|
b'{"value":{}}',
|
|
b'{"value":0.123456789012345678901}',
|
|
b'{"value":0.123456789012345678902}',
|
|
b'{"value":1e400}',
|
|
b'{"value":1}',
|
|
b'{"value":1e0}',
|
|
b'{"value":-0}',
|
|
b'{"value":1e9999999999999999999}',
|
|
)
|
|
for rejected in (
|
|
call_edge(edge, "POST", CHAT_PATH, body=value, headers={"content-type": "application/json"})
|
|
for value in values
|
|
if value != body
|
|
):
|
|
assert rejected.status_code == REPLAY_MISS_STATUS
|
|
assert b"body" in rejected.body
|
|
assert (
|
|
call_edge(
|
|
edge, "POST", CHAT_PATH, body=body, headers={"content-type": "application/json"}
|
|
).status_code
|
|
== 200
|
|
)
|
|
assert len(provider.hits) == 1
|
|
|
|
@pytest.mark.parametrize(
|
|
"path,body,headers",
|
|
[
|
|
(UPLOAD_PATH, b"{}", {"content-type": "application/json"}),
|
|
(CHAT_PATH + "?part=%FF", b"{}", {"content-type": "application/json"}),
|
|
(CHAT_PATH + "?part=%FE", b"{}", {"content-type": "application/json"}),
|
|
(CHAT_PATH, b"opaque", {"content-type": "application/octet-stream"}),
|
|
(CHAT_PATH, b"--boundary", {"content-type": "multipart/form-data; boundary=boundary"}),
|
|
(CHAT_PATH, b'{"x":1,"x":2}', {"content-type": "application/json"}),
|
|
(CHAT_PATH, b"{}", {"content-type": "application/json", "x-custom-behavior": "synthetic-private-value"}),
|
|
],
|
|
)
|
|
def test_ineligible_capture_never_calls_provider(
|
|
self, tmp_path: Path, path: str, body: bytes, headers: dict[str, str]
|
|
) -> None:
|
|
recorder: Final = prepare_bundle(tmp_path / "strict", profile="stateless_v1")
|
|
assert isinstance(recorder, BundleRecorder)
|
|
with fake_provider() as provider:
|
|
with running_edge(RecordEdge(recorder, threading.Lock()), {"openai": provider_url(provider)}) as edge:
|
|
result: Final = call_edge(edge, "POST", path, body=body, headers=headers)
|
|
assert result.status_code == REPLAY_MISS_STATUS
|
|
assert b"eligibility error" in result.body
|
|
assert b"synthetic-private-value" not in result.body
|
|
assert provider.hits == []
|
|
assert this_tests_files(recorder.root) == []
|
|
|
|
def test_destination_is_part_of_actual_http_identity(self, tmp_path: Path) -> None:
|
|
recorder: Final = prepare_bundle(tmp_path / "strict", profile="stateless_v1")
|
|
assert isinstance(recorder, BundleRecorder)
|
|
with fake_provider() as provider:
|
|
with running_edge(RecordEdge(recorder, threading.Lock()), {"openai": provider_url(provider)}) as edge:
|
|
assert (
|
|
call_edge(
|
|
edge, "POST", CHAT_PATH, body=b"{}", headers={"content-type": "application/json"}
|
|
).status_code
|
|
== 200
|
|
)
|
|
loaded: Final = load_bundle(recorder.root, profile="stateless_v1")
|
|
assert isinstance(loaded, LoadedBundle)
|
|
with running_edge(ReplayEdge(ReplaySource(loaded)), {"openai": provider_url(provider) + "/other"}) as edge:
|
|
result: Final = call_edge(
|
|
edge, "POST", CHAT_PATH, body=b"{}", headers={"content-type": "application/json"}
|
|
)
|
|
assert result.status_code == REPLAY_MISS_STATUS
|
|
assert b"upstream" in result.body
|
|
assert len(provider.hits) == 1
|
|
|
|
def test_credentials_are_not_identity_and_fresh_process_replays(self, tmp_path: Path) -> None:
|
|
recorder: Final = prepare_bundle(tmp_path / "strict", profile="stateless_v1")
|
|
assert isinstance(recorder, BundleRecorder)
|
|
headers: Final = {
|
|
"content-type": "application/json",
|
|
"authorization": "bEaReR synthetic-token",
|
|
"x-api-key": "synthetic-api-key",
|
|
"cookie": "synthetic-cookie",
|
|
}
|
|
path: Final = CHAT_PATH + "?api_key=synthetic-query-secret&part=one&part=two"
|
|
body: Final = b'{"model":"synthetic","messages":[]}'
|
|
with fake_provider(echo_request=False) as provider:
|
|
mounts: Final = {"openai": provider_url(provider)}
|
|
with running_edge(RecordEdge(recorder, threading.Lock()), mounts) as edge:
|
|
captured: Final = call_edge(edge, "POST", path, body=body, headers=headers)
|
|
assert captured.status_code == 200
|
|
seen_headers, seen_body = provider.requests[0]
|
|
assert {k.lower(): v for k, v in seen_headers.items()}.items() >= headers.items()
|
|
assert seen_body == body
|
|
assert provider.hits == ["POST " + path.removeprefix("/openai")]
|
|
artifacts: Final = "".join(file.read_text() for file in recorder.root.rglob("*.json"))
|
|
for secret in ("synthetic-token", "synthetic-api-key", "synthetic-cookie", "synthetic-query-secret"):
|
|
assert secret not in artifacts
|
|
child: Final = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-c",
|
|
"""
|
|
import json, sys
|
|
from pathlib import Path
|
|
from fixture_bundle import LoadedBundle, load_bundle
|
|
from provider_edge import ProviderRequestObservation, observed_provider_edge, replay_leftover_error
|
|
from test_provider_edge import call_edge
|
|
from fixture_profile import MatchProfile
|
|
from fixture_mode import current_test_key
|
|
loaded = load_bundle(Path(sys.argv[1]), profile="stateless_v1")
|
|
assert isinstance(loaded, LoadedBundle)
|
|
with observed_provider_edge(ProviderRequestObservation("synthetic"), mode_raw="replay", bundle_dir=Path(sys.argv[1]), bind_host="127.0.0.1", advertise_host="127.0.0.1", mounts={"openai": sys.argv[2]}) as edge:
|
|
response = call_edge(edge, "POST", sys.argv[3], body=sys.argv[4].encode(), headers=json.loads(sys.argv[5]))
|
|
assert response.status_code == 200
|
|
print(response.body.decode())
|
|
assert replay_leftover_error(mode_raw="replay", bundle_dir=Path(sys.argv[1]), test_key=current_test_key()) is None
|
|
""",
|
|
str(recorder.root),
|
|
provider_url(provider),
|
|
path.replace("synthetic-query-secret", "new-query-credential"),
|
|
body.decode(),
|
|
json.dumps({**headers, "authorization": "Bearer another-credential", "x-api-key": "another-key"}),
|
|
],
|
|
env={
|
|
**os.environ,
|
|
"PYTHONPATH": os.pathsep.join(
|
|
str(Path(__file__).resolve().parents[1] / tree) for tree in ("e2e", "e2e_harness")
|
|
),
|
|
"E2E_REPLAY_MATCH_PROFILE": "stateless_v1",
|
|
},
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30,
|
|
)
|
|
assert child.returncode == 0, child.stderr
|
|
assert child.stdout.strip().encode() == captured.body
|
|
assert len(provider.hits) == 1
|
|
|
|
@pytest.mark.parametrize("profile,other", [("legacy", "stateless_v1"), ("stateless_v1", "legacy")])
|
|
def test_profiles_cannot_load_each_others_bundles(
|
|
self, tmp_path: Path, profile: MatchProfile, other: MatchProfile
|
|
) -> None:
|
|
from fixture_bundle import UnreadableBundle
|
|
|
|
recorder: Final = prepare_bundle(tmp_path / profile, profile=profile)
|
|
assert isinstance(recorder, BundleRecorder)
|
|
mismatch: Final = load_bundle(recorder.root, profile=other)
|
|
assert isinstance(mismatch, UnreadableBundle)
|
|
assert "profile mismatch" in mismatch.reason
|
|
assert "re-record" in mismatch.reason
|
|
|
|
@pytest.mark.parametrize("abort_after", [None, 2])
|
|
def test_strict_stream_preserves_chunks_and_truncation(self, tmp_path: Path, abort_after: int | None) -> None:
|
|
recorder: Final = prepare_bundle(tmp_path / "strict", profile="stateless_v1")
|
|
assert isinstance(recorder, BundleRecorder)
|
|
with chunked_provider(abort_after=abort_after) as provider:
|
|
mounts: Final = {"anthropic": provider_url(provider)}
|
|
with running_edge(RecordEdge(recorder, threading.Lock()), mounts) as edge:
|
|
_, captured, captured_ending = raw_stream_post(edge.port, "/anthropic/v1/messages", STREAM_BODY)
|
|
loaded: Final = load_bundle(recorder.root, profile="stateless_v1")
|
|
assert isinstance(loaded, LoadedBundle)
|
|
source: Final = ReplaySource(loaded)
|
|
with running_edge(ReplayEdge(source), mounts) as edge:
|
|
_, replayed, ending = raw_stream_post(edge.port, "/anthropic/v1/messages", STREAM_BODY)
|
|
assert captured == replayed == list(SSE_CHUNKS[:abort_after])
|
|
assert ending == captured_ending
|
|
assert (ending == "terminated") == (abort_after is None)
|
|
assert source.leftover_error(current_test_key()) is None
|
|
assert len(provider.hits) == 1
|
|
|
|
def test_auth_scheme_survives_missing_credentials(self, tmp_path: Path) -> None:
|
|
recorder: Final = prepare_bundle(tmp_path / "strict", profile="stateless_v1")
|
|
assert isinstance(recorder, BundleRecorder)
|
|
headers: Final = {"content-type": "application/json", "authorization": "Bearer"}
|
|
with fake_provider() as provider:
|
|
mounts: Final = {"openai": provider_url(provider)}
|
|
with running_edge(RecordEdge(recorder, threading.Lock()), mounts) as edge:
|
|
assert call_edge(edge, "POST", CHAT_PATH, body=b"{}", headers=headers).status_code == 200
|
|
loaded: Final = load_bundle(recorder.root, profile="stateless_v1")
|
|
assert isinstance(loaded, LoadedBundle)
|
|
with running_edge(ReplayEdge(ReplaySource(loaded)), mounts) as edge:
|
|
for result in (
|
|
call_edge(edge, "POST", CHAT_PATH, body=b"{}", headers={**headers, "authorization": scheme})
|
|
for scheme in ("Basic", "Digest")
|
|
):
|
|
assert result.status_code == REPLAY_MISS_STATUS
|
|
assert b"auth" in result.body
|
|
assert (
|
|
call_edge(
|
|
edge,
|
|
"POST",
|
|
CHAT_PATH,
|
|
body=b"{}",
|
|
headers={**headers, "authorization": "bEaReR synthetic-token"},
|
|
).status_code
|
|
== 200
|
|
)
|
|
assert len(provider.hits) == 1
|