mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-16 23:41:43 +00:00
329 lines
17 KiB
Python
329 lines
17 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
from pathlib import Path
|
|
from typing import Final
|
|
|
|
import pytest
|
|
from fixture_bundle import BundleRecorder, LoadedBundle, load_bundle, prepare_bundle
|
|
from fixture_mode import current_test_key
|
|
from fixture_profile import MatchProfile
|
|
from provider_edge import REPLAY_MISS_STATUS, RecordEdge, ReplayEdge, ReplaySource
|
|
from test_provider_edge import (
|
|
CHAT_PATH,
|
|
SSE_CHUNKS,
|
|
STREAM_BODY,
|
|
UPLOAD_PATH,
|
|
call_edge,
|
|
chunked_provider,
|
|
fake_provider,
|
|
json_object,
|
|
provider_url,
|
|
raw_stream_post,
|
|
running_edge,
|
|
this_tests_files,
|
|
)
|
|
|
|
|
|
class TestStrictIdentity:
|
|
@pytest.mark.parametrize("path", [CHAT_PATH, "/anthropic/v1/messages"])
|
|
def test_roundtrip_rejects_semantic_changes(self, tmp_path: Path, path: str) -> None:
|
|
recorder: Final = prepare_bundle(tmp_path / "strict", profile="stateless_v1")
|
|
assert isinstance(recorder, BundleRecorder)
|
|
original: Final = (
|
|
b'{"model":"synthetic","messages":[{"role":"user",'
|
|
b'"content":"2031-04-05 00000000-0000-0000-0000-000000000001"}],"options":[1,2]}'
|
|
)
|
|
headers: Final = {
|
|
"content-type": "application/json",
|
|
"accept": "application/json",
|
|
"anthropic-version": "2023-06-01",
|
|
"anthropic-beta": "feature-a",
|
|
"openai-beta": "feature-b",
|
|
"authorization": "Bearer synthetic-secret-one",
|
|
}
|
|
query: Final = "?part=one&part=two&blank="
|
|
with fake_provider() as provider:
|
|
mounts: Final = {"openai": provider_url(provider), "anthropic": provider_url(provider)}
|
|
with running_edge(RecordEdge(recorder, threading.Lock()), mounts) as edge:
|
|
captured: Final = call_edge(edge, "POST", path + query, body=original, headers=headers)
|
|
assert captured.status_code == 200
|
|
assert json_object(captured.body)["echo"] == original.decode()
|
|
loaded: Final = load_bundle(recorder.root, profile="stateless_v1")
|
|
assert isinstance(loaded, LoadedBundle)
|
|
assert loaded.manifest.match_profile == "stateless_v1"
|
|
source: Final = ReplaySource(loaded)
|
|
with running_edge(ReplayEdge(source), mounts) as edge:
|
|
cases: Final = (
|
|
(original.replace(b"2031-04-05", b"2032-06-07"), headers, query, "body"),
|
|
(original.replace(b"000000000001", b"000000000002"), headers, query, "body"),
|
|
(original.replace(b"[1,2]", b"[2,1]"), headers, query, "body"),
|
|
(original.replace(b"synthetic", b"other"), headers, query, "body"),
|
|
(original, headers, "?part=three&part=two&blank=", "query"),
|
|
(original, headers, "?part=two&part=one&blank=", "query"),
|
|
*(
|
|
(original, {k: v for k, v in headers.items() if k != name}, query, "headers")
|
|
for name in ("accept", "anthropic-version", "anthropic-beta", "openai-beta")
|
|
),
|
|
*(
|
|
(original, {**headers, name: value}, query, "headers")
|
|
for name in ("accept", "anthropic-version", "anthropic-beta", "openai-beta")
|
|
for value in ("different", "")
|
|
),
|
|
(original, {**headers, "authorization": "Basic synthetic-secret-two"}, query, "auth"),
|
|
(original, {k: v for k, v in headers.items() if k != "authorization"}, query, "auth"),
|
|
)
|
|
for rejected, reason in (
|
|
(call_edge(edge, "POST", path + changed_query, body=body, headers=changed_headers), reason)
|
|
for body, changed_headers, changed_query, reason in cases
|
|
):
|
|
assert rejected.status_code == REPLAY_MISS_STATUS
|
|
assert reason in rejected.body.decode()
|
|
assert b"synthetic-secret" not in rejected.body
|
|
reordered: Final = json.dumps(dict(reversed(list(json_object(original).items())))).encode()
|
|
accepted: Final = call_edge(
|
|
edge, "POST", path + query, body=reordered, headers={k.upper(): v for k, v in headers.items()}
|
|
)
|
|
assert accepted.status_code == 200
|
|
assert accepted.body == captured.body
|
|
assert source.leftover_error(current_test_key()) is None
|
|
assert len(provider.hits) == 1
|
|
assert "synthetic-secret" not in "".join(file.read_text() for file in recorder.root.rglob("*.json"))
|
|
|
|
@pytest.mark.parametrize(
|
|
"body",
|
|
[
|
|
b'{"value":null}',
|
|
b'{"value":""}',
|
|
b'{"value":false}',
|
|
b'{"value":0}',
|
|
b'{"value":[]}',
|
|
b'{"value":{}}',
|
|
b'{"value":0.123456789012345678901}',
|
|
b'{"value":0.123456789012345678902}',
|
|
b'{"value":1e400}',
|
|
b'{"value":1}',
|
|
b'{"value":1e0}',
|
|
b'{"value":-0}',
|
|
b'{"value":1e9999999999999999999}',
|
|
],
|
|
)
|
|
def test_json_values_remain_distinct(self, tmp_path: Path, body: bytes) -> None:
|
|
recorder: Final = prepare_bundle(tmp_path / "strict", profile="stateless_v1")
|
|
assert isinstance(recorder, BundleRecorder)
|
|
with fake_provider() as provider:
|
|
mounts: Final = {"openai": provider_url(provider)}
|
|
with running_edge(RecordEdge(recorder, threading.Lock()), mounts) as edge:
|
|
assert (
|
|
call_edge(
|
|
edge, "POST", CHAT_PATH, body=body, headers={"content-type": "application/json"}
|
|
).status_code
|
|
== 200
|
|
)
|
|
loaded: Final = load_bundle(recorder.root, profile="stateless_v1")
|
|
assert isinstance(loaded, LoadedBundle)
|
|
with running_edge(ReplayEdge(ReplaySource(loaded)), mounts) as edge:
|
|
values: Final = (
|
|
b"{}",
|
|
b'{"value":null}',
|
|
b'{"value":""}',
|
|
b'{"value":false}',
|
|
b'{"value":0}',
|
|
b'{"value":[]}',
|
|
b'{"value":{}}',
|
|
b'{"value":0.123456789012345678901}',
|
|
b'{"value":0.123456789012345678902}',
|
|
b'{"value":1e400}',
|
|
b'{"value":1}',
|
|
b'{"value":1e0}',
|
|
b'{"value":-0}',
|
|
b'{"value":1e9999999999999999999}',
|
|
)
|
|
for rejected in (
|
|
call_edge(edge, "POST", CHAT_PATH, body=value, headers={"content-type": "application/json"})
|
|
for value in values
|
|
if value != body
|
|
):
|
|
assert rejected.status_code == REPLAY_MISS_STATUS
|
|
assert b"body" in rejected.body
|
|
assert (
|
|
call_edge(
|
|
edge, "POST", CHAT_PATH, body=body, headers={"content-type": "application/json"}
|
|
).status_code
|
|
== 200
|
|
)
|
|
assert len(provider.hits) == 1
|
|
|
|
@pytest.mark.parametrize(
|
|
"path,body,headers",
|
|
[
|
|
(UPLOAD_PATH, b"{}", {"content-type": "application/json"}),
|
|
(CHAT_PATH + "?part=%FF", b"{}", {"content-type": "application/json"}),
|
|
(CHAT_PATH + "?part=%FE", b"{}", {"content-type": "application/json"}),
|
|
(CHAT_PATH, b"opaque", {"content-type": "application/octet-stream"}),
|
|
(CHAT_PATH, b"--boundary", {"content-type": "multipart/form-data; boundary=boundary"}),
|
|
(CHAT_PATH, b'{"x":1,"x":2}', {"content-type": "application/json"}),
|
|
(CHAT_PATH, b"{}", {"content-type": "application/json", "x-custom-behavior": "synthetic-private-value"}),
|
|
],
|
|
)
|
|
def test_ineligible_capture_never_calls_provider(
|
|
self, tmp_path: Path, path: str, body: bytes, headers: dict[str, str]
|
|
) -> None:
|
|
recorder: Final = prepare_bundle(tmp_path / "strict", profile="stateless_v1")
|
|
assert isinstance(recorder, BundleRecorder)
|
|
with fake_provider() as provider:
|
|
with running_edge(RecordEdge(recorder, threading.Lock()), {"openai": provider_url(provider)}) as edge:
|
|
result: Final = call_edge(edge, "POST", path, body=body, headers=headers)
|
|
assert result.status_code == REPLAY_MISS_STATUS
|
|
assert b"eligibility error" in result.body
|
|
assert b"synthetic-private-value" not in result.body
|
|
assert provider.hits == []
|
|
assert this_tests_files(recorder.root) == []
|
|
|
|
def test_destination_is_part_of_actual_http_identity(self, tmp_path: Path) -> None:
|
|
recorder: Final = prepare_bundle(tmp_path / "strict", profile="stateless_v1")
|
|
assert isinstance(recorder, BundleRecorder)
|
|
with fake_provider() as provider:
|
|
with running_edge(RecordEdge(recorder, threading.Lock()), {"openai": provider_url(provider)}) as edge:
|
|
assert (
|
|
call_edge(
|
|
edge, "POST", CHAT_PATH, body=b"{}", headers={"content-type": "application/json"}
|
|
).status_code
|
|
== 200
|
|
)
|
|
loaded: Final = load_bundle(recorder.root, profile="stateless_v1")
|
|
assert isinstance(loaded, LoadedBundle)
|
|
with running_edge(ReplayEdge(ReplaySource(loaded)), {"openai": provider_url(provider) + "/other"}) as edge:
|
|
result: Final = call_edge(
|
|
edge, "POST", CHAT_PATH, body=b"{}", headers={"content-type": "application/json"}
|
|
)
|
|
assert result.status_code == REPLAY_MISS_STATUS
|
|
assert b"upstream" in result.body
|
|
assert len(provider.hits) == 1
|
|
|
|
def test_credentials_are_not_identity_and_fresh_process_replays(self, tmp_path: Path) -> None:
|
|
recorder: Final = prepare_bundle(tmp_path / "strict", profile="stateless_v1")
|
|
assert isinstance(recorder, BundleRecorder)
|
|
headers: Final = {
|
|
"content-type": "application/json",
|
|
"authorization": "bEaReR synthetic-token",
|
|
"x-api-key": "synthetic-api-key",
|
|
"cookie": "synthetic-cookie",
|
|
}
|
|
path: Final = CHAT_PATH + "?api_key=synthetic-query-secret&part=one&part=two"
|
|
body: Final = b'{"model":"synthetic","messages":[]}'
|
|
with fake_provider(echo_request=False) as provider:
|
|
mounts: Final = {"openai": provider_url(provider)}
|
|
with running_edge(RecordEdge(recorder, threading.Lock()), mounts) as edge:
|
|
captured: Final = call_edge(edge, "POST", path, body=body, headers=headers)
|
|
assert captured.status_code == 200
|
|
seen_headers, seen_body = provider.requests[0]
|
|
assert {k.lower(): v for k, v in seen_headers.items()}.items() >= headers.items()
|
|
assert seen_body == body
|
|
assert provider.hits == ["POST " + path.removeprefix("/openai")]
|
|
artifacts: Final = "".join(file.read_text() for file in recorder.root.rglob("*.json"))
|
|
for secret in ("synthetic-token", "synthetic-api-key", "synthetic-cookie", "synthetic-query-secret"):
|
|
assert secret not in artifacts
|
|
child: Final = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-c",
|
|
"""
|
|
import json, sys
|
|
from pathlib import Path
|
|
from fixture_bundle import LoadedBundle, load_bundle
|
|
from provider_edge import ProviderRequestObservation, observed_provider_edge, replay_leftover_error
|
|
from test_provider_edge import call_edge
|
|
from fixture_profile import MatchProfile
|
|
from fixture_mode import current_test_key
|
|
loaded = load_bundle(Path(sys.argv[1]), profile="stateless_v1")
|
|
assert isinstance(loaded, LoadedBundle)
|
|
with observed_provider_edge(ProviderRequestObservation("synthetic"), mode_raw="replay", bundle_dir=Path(sys.argv[1]), bind_host="127.0.0.1", advertise_host="127.0.0.1", mounts={"openai": sys.argv[2]}) as edge:
|
|
response = call_edge(edge, "POST", sys.argv[3], body=sys.argv[4].encode(), headers=json.loads(sys.argv[5]))
|
|
assert response.status_code == 200
|
|
print(response.body.decode())
|
|
assert replay_leftover_error(mode_raw="replay", bundle_dir=Path(sys.argv[1]), test_key=current_test_key()) is None
|
|
""",
|
|
str(recorder.root),
|
|
provider_url(provider),
|
|
path.replace("synthetic-query-secret", "new-query-credential"),
|
|
body.decode(),
|
|
json.dumps({**headers, "authorization": "Bearer another-credential", "x-api-key": "another-key"}),
|
|
],
|
|
env={
|
|
**os.environ,
|
|
"PYTHONPATH": str(Path(__file__).resolve().parents[1] / "e2e"),
|
|
"E2E_REPLAY_MATCH_PROFILE": "stateless_v1",
|
|
},
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30,
|
|
)
|
|
assert child.returncode == 0, child.stderr
|
|
assert child.stdout.strip().encode() == captured.body
|
|
assert len(provider.hits) == 1
|
|
|
|
@pytest.mark.parametrize("profile,other", [("legacy", "stateless_v1"), ("stateless_v1", "legacy")])
|
|
def test_profiles_cannot_load_each_others_bundles(
|
|
self, tmp_path: Path, profile: MatchProfile, other: MatchProfile
|
|
) -> None:
|
|
from fixture_bundle import UnreadableBundle
|
|
|
|
recorder: Final = prepare_bundle(tmp_path / profile, profile=profile)
|
|
assert isinstance(recorder, BundleRecorder)
|
|
mismatch: Final = load_bundle(recorder.root, profile=other)
|
|
assert isinstance(mismatch, UnreadableBundle)
|
|
assert "profile mismatch" in mismatch.reason
|
|
assert "re-record" in mismatch.reason
|
|
|
|
@pytest.mark.parametrize("abort_after", [None, 2])
|
|
def test_strict_stream_preserves_chunks_and_truncation(self, tmp_path: Path, abort_after: int | None) -> None:
|
|
recorder: Final = prepare_bundle(tmp_path / "strict", profile="stateless_v1")
|
|
assert isinstance(recorder, BundleRecorder)
|
|
with chunked_provider(abort_after=abort_after) as provider:
|
|
mounts: Final = {"anthropic": provider_url(provider)}
|
|
with running_edge(RecordEdge(recorder, threading.Lock()), mounts) as edge:
|
|
_, captured, captured_ending = raw_stream_post(edge.port, "/anthropic/v1/messages", STREAM_BODY)
|
|
loaded: Final = load_bundle(recorder.root, profile="stateless_v1")
|
|
assert isinstance(loaded, LoadedBundle)
|
|
source: Final = ReplaySource(loaded)
|
|
with running_edge(ReplayEdge(source), mounts) as edge:
|
|
_, replayed, ending = raw_stream_post(edge.port, "/anthropic/v1/messages", STREAM_BODY)
|
|
assert captured == replayed == list(SSE_CHUNKS[:abort_after])
|
|
assert ending == captured_ending
|
|
assert (ending == "terminated") == (abort_after is None)
|
|
assert source.leftover_error(current_test_key()) is None
|
|
assert len(provider.hits) == 1
|
|
|
|
def test_auth_scheme_survives_missing_credentials(self, tmp_path: Path) -> None:
|
|
recorder: Final = prepare_bundle(tmp_path / "strict", profile="stateless_v1")
|
|
assert isinstance(recorder, BundleRecorder)
|
|
headers: Final = {"content-type": "application/json", "authorization": "Bearer"}
|
|
with fake_provider() as provider:
|
|
mounts: Final = {"openai": provider_url(provider)}
|
|
with running_edge(RecordEdge(recorder, threading.Lock()), mounts) as edge:
|
|
assert call_edge(edge, "POST", CHAT_PATH, body=b"{}", headers=headers).status_code == 200
|
|
loaded: Final = load_bundle(recorder.root, profile="stateless_v1")
|
|
assert isinstance(loaded, LoadedBundle)
|
|
with running_edge(ReplayEdge(ReplaySource(loaded)), mounts) as edge:
|
|
for result in (
|
|
call_edge(edge, "POST", CHAT_PATH, body=b"{}", headers={**headers, "authorization": scheme})
|
|
for scheme in ("Basic", "Digest")
|
|
):
|
|
assert result.status_code == REPLAY_MISS_STATUS
|
|
assert b"auth" in result.body
|
|
assert (
|
|
call_edge(
|
|
edge,
|
|
"POST",
|
|
CHAT_PATH,
|
|
body=b"{}",
|
|
headers={**headers, "authorization": "bEaReR synthetic-token"},
|
|
).status_code
|
|
== 200
|
|
)
|
|
assert len(provider.hits) == 1
|