mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-10 03:28:53 +00:00
85585b63b0
Snapshot corpus for the translation v2 differential gate: 19 OpenAI-format
request cases x {anthropic, bedrock converse, bedrock invoke} through the
real v1 transform_request path (get_optional_params, validate_environment
included), recorded provider responses through transform_response, SSE and
event-stream replays through CustomStreamWrapper plus the real decoders, and
both /v1/messages surfaces (anthropic<->openai adapter round-trip and the
native AnthropicMessagesConfig quirks: max_tokens pop, reasoning_effort
rewrite, advisor-block stripping). 89 canonical-JSON snapshots, deterministic
via frozen uuid/fastuuid/time, regenerated with --snapshot-update or
SNAPSHOT_UPDATE=1. Wired into CI as a job in test-unit-llm-providers.yml and
a make test-characterization target
55 lines
1.8 KiB
Python
55 lines
1.8 KiB
Python
"""Snapshot machinery shared by all characterization tests.
|
|
|
|
A snapshot is canonical JSON: pretty-printed, sorted keys, trailing newline.
|
|
v2's differential gate re-runs the same corpus through v2 and asserts the
|
|
emitted JSON is byte-identical to these files.
|
|
"""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
PKG_ROOT = Path(__file__).parent
|
|
CASES_DIR = PKG_ROOT / "cases"
|
|
FIXTURES_DIR = PKG_ROOT / "fixtures"
|
|
SNAPSHOTS_DIR = PKG_ROOT / "snapshots"
|
|
|
|
|
|
def jsonable(obj: Any) -> Any:
|
|
"""Best-effort conversion of v1 output objects into plain JSON data."""
|
|
if hasattr(obj, "model_dump"):
|
|
return jsonable(obj.model_dump())
|
|
if isinstance(obj, dict):
|
|
return {str(k): jsonable(v) for k, v in obj.items()}
|
|
if isinstance(obj, (list, tuple)):
|
|
return [jsonable(v) for v in obj]
|
|
if isinstance(obj, (str, int, float, bool)) or obj is None:
|
|
return obj
|
|
return repr(obj)
|
|
|
|
|
|
def canonical_json(obj: Any) -> str:
|
|
return json.dumps(jsonable(obj), indent=2, sort_keys=True) + "\n"
|
|
|
|
|
|
def load_json(path: Path) -> Any:
|
|
with open(path) as f:
|
|
return json.load(f)
|
|
|
|
|
|
def assert_snapshot(rel_path: str, obj: Any, update: bool) -> None:
|
|
"""Compare ``obj`` against ``snapshots/<rel_path>``; rewrite when updating."""
|
|
path = SNAPSHOTS_DIR / rel_path
|
|
rendered = canonical_json(obj)
|
|
if update:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(rendered)
|
|
return
|
|
assert path.exists(), (
|
|
f"Missing snapshot {path}. Run with --snapshot-update (or SNAPSHOT_UPDATE=1) "
|
|
"to record it, then commit the file."
|
|
)
|
|
assert rendered == path.read_text(), (
|
|
f"Characterization drift for {rel_path}. If this change is intentional, "
|
|
"regenerate with --snapshot-update and ship the snapshot diff as its own PR."
|
|
)
|