mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-15 23:31:29 +00:00
test: preserve strict replay numeric spelling
This commit is contained in:
Yuneng Jiang
parent
6604c78120
commit
b9c194076a
3 changed files with 29 additions and 12 deletions
|
|
@ -237,8 +237,8 @@ Before you push
|
|||
|
||||
Set `E2E_REPLAY_MATCH_PROFILE=stateless_v1` for both recording and replay to bind OpenAI `/v1/chat/completions` and Anthropic `/v1/messages` requests to their upstream destination, ordered query pairs, semantic headers and literal JSON content. The default remains `legacy`. Strict bundles use format 5 and cannot load as legacy bundles; select the matching profile or re-record with `E2E_FIXTURE_MODE=record`. Missing profile metadata never enrolls a legacy bundle in strict matching
|
||||
|
||||
Strict matching preserves dates, UUIDs, hashes, model names, tool arguments, array order and omitted/null/empty/false/zero values. JSON object key order and header name casing may change. The strict body uses tagged JSON values so number precision and JSON types survive persistence, including numbers larger than a floating-point value. Invalid UTF-8 query values fail eligibility. Duplicate JSON keys, unsupported endpoints, non-JSON bodies and unknown semantic headers fail eligibility before contacting a provider
|
||||
Strict matching preserves dates, UUIDs, hashes, model names, tool arguments, array order and omitted/null/empty/false/zero values. JSON object key order and header name casing may change. The strict body uses tagged JSON values so number precision and JSON types survive persistence, including exact numeric spelling and numbers larger than a floating-point value. Invalid UTF-8 query values fail eligibility. Duplicate JSON keys, unsupported endpoints, non-JSON bodies and unknown semantic headers fail eligibility before contacting a provider
|
||||
|
||||
The semantic header set is `content-type`, `accept`, `anthropic-version`, `anthropic-beta` and `openai-beta`, including missing versus present values. Authorization records presence and scheme; `x-api-key` records presence only. Credential values and cookies are excluded. Credential query values are redacted while their position and field name remain in the identity. Never use real customer inputs in fixture qualification
|
||||
The semantic header set is `content-type`, `accept`, `anthropic-version`, `anthropic-beta` and `openai-beta`, including missing versus present values. Authorization records presence and the case-insensitive scheme; `x-api-key` records presence only. Credential values and cookies are excluded. Credential query values are redacted while their position and field name remain in the identity. Never use real customer inputs in fixture qualification
|
||||
|
||||
Excluded transport and telemetry headers are `host`, `content-length`, `connection`, `accept-encoding`, `user-agent`, `traceparent`, `tracestate`, `x-request-id`, `x-client-request-id` and `x-stainless-*`. Inbound transfer-encoding is unsupported; send JSON with content-length framing. The destination represents host identity and the relay carries original body bytes. Replay does not verify credentials, SDK timeout/retry behavior, transport performance, model availability or stateful remote IDs. Live relay uses original request bytes and header values, never the stored identity
|
||||
|
|
|
|||
|
|
@ -4,14 +4,20 @@ import json
|
|||
import os
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from decimal import Decimal, DecimalException
|
||||
from typing import Final, Literal
|
||||
from urllib.parse import parse_qsl, urlsplit
|
||||
|
||||
from pydantic import BaseModel, JsonValue, TypeAdapter
|
||||
|
||||
type MatchProfile = Literal["legacy", "stateless_v1"]
|
||||
type ExactJson = dict[str, ExactJson] | list[ExactJson] | str | bool | Decimal | int | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NumberToken:
|
||||
literal: str
|
||||
|
||||
|
||||
type ExactJson = dict[str, ExactJson] | list[ExactJson] | str | bool | NumberToken | None
|
||||
|
||||
SEMANTIC_HEADERS: Final = frozenset({"content-type", "accept", "anthropic-version", "anthropic-beta", "openai-beta"})
|
||||
AUTH_HEADERS: Final = frozenset({"authorization", "x-api-key"})
|
||||
|
|
@ -93,8 +99,8 @@ def _exact_value(value: ExactJson) -> JsonValue:
|
|||
return {"array": [_exact_value(item) for item in value]}
|
||||
case bool():
|
||||
return {"boolean": value}
|
||||
case int() | Decimal():
|
||||
return {"number": str(value)}
|
||||
case NumberToken(literal=literal):
|
||||
return {"number": literal}
|
||||
case str():
|
||||
return {"string": value}
|
||||
case None:
|
||||
|
|
@ -140,13 +146,17 @@ def strict_identity(
|
|||
parsed: Final = (
|
||||
JSON_VALUE.validate_python(
|
||||
json.loads(
|
||||
body, object_pairs_hook=_unique_object, parse_constant=_invalid_constant, parse_float=Decimal
|
||||
body,
|
||||
object_pairs_hook=_unique_object,
|
||||
parse_constant=_invalid_constant,
|
||||
parse_float=NumberToken,
|
||||
parse_int=NumberToken,
|
||||
)
|
||||
)
|
||||
if body
|
||||
else None
|
||||
)
|
||||
except (ValueError, UnicodeError, DecimalException):
|
||||
except (ValueError, UnicodeError):
|
||||
return IneligibleRequest("invalid JSON or duplicate JSON object keys")
|
||||
if body and not isinstance(parsed, dict):
|
||||
return IneligibleRequest("stateless inference requires a JSON object")
|
||||
|
|
@ -160,7 +170,7 @@ def strict_identity(
|
|||
query=tuple((key, "<credential>" if key.lower() in CREDENTIAL_QUERY else value) for key, value in query_pairs),
|
||||
headers={key: value for key, value in lowered.items() if key in SEMANTIC_HEADERS},
|
||||
auth={
|
||||
key: (value.partition(" ")[0] if key == "authorization" else "present")
|
||||
key: (value.partition(" ")[0].lower() if key == "authorization" else "present")
|
||||
for key, value in lowered.items()
|
||||
if key in AUTH_HEADERS
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1434,6 +1434,10 @@ class TestStrictIdentity:
|
|||
b'{"value":0.123456789012345678901}',
|
||||
b'{"value":0.123456789012345678902}',
|
||||
b'{"value":1e400}',
|
||||
b'{"value":1}',
|
||||
b'{"value":1e0}',
|
||||
b'{"value":-0}',
|
||||
b'{"value":1e9999999999999999999}',
|
||||
],
|
||||
)
|
||||
def test_json_values_remain_distinct(self, tmp_path: Path, body: bytes) -> None:
|
||||
|
|
@ -1462,6 +1466,10 @@ class TestStrictIdentity:
|
|||
b'{"value":0.123456789012345678901}',
|
||||
b'{"value":0.123456789012345678902}',
|
||||
b'{"value":1e400}',
|
||||
b'{"value":1}',
|
||||
b'{"value":1e0}',
|
||||
b'{"value":-0}',
|
||||
b'{"value":1e9999999999999999999}',
|
||||
)
|
||||
for rejected in (
|
||||
call_edge(edge, "POST", CHAT_PATH, body=value, headers={"content-type": "application/json"})
|
||||
|
|
@ -1487,7 +1495,6 @@ class TestStrictIdentity:
|
|||
(CHAT_PATH, b"opaque", {"content-type": "application/octet-stream"}),
|
||||
(CHAT_PATH, b"--boundary", {"content-type": "multipart/form-data; boundary=boundary"}),
|
||||
(CHAT_PATH, b'{"x":1,"x":2}', {"content-type": "application/json"}),
|
||||
(CHAT_PATH, b'{"x":1e9999999999999999999}', {"content-type": "application/json"}),
|
||||
(CHAT_PATH, b"{}", {"content-type": "application/json", "x-custom-behavior": "synthetic-private-value"}),
|
||||
],
|
||||
)
|
||||
|
|
@ -1531,7 +1538,7 @@ class TestStrictIdentity:
|
|||
assert isinstance(recorder, BundleRecorder)
|
||||
headers: Final = {
|
||||
"content-type": "application/json",
|
||||
"authorization": "Bearer synthetic-token",
|
||||
"authorization": "bEaReR synthetic-token",
|
||||
"x-api-key": "synthetic-api-key",
|
||||
"cookie": "synthetic-cookie",
|
||||
}
|
||||
|
|
@ -1643,7 +1650,7 @@ assert replay_leftover_error(mode_raw="replay", bundle_dir=Path(sys.argv[1]), te
|
|||
"POST",
|
||||
CHAT_PATH,
|
||||
body=b"{}",
|
||||
headers={**headers, "authorization": "Bearer synthetic-token"},
|
||||
headers={**headers, "authorization": "bEaReR synthetic-token"},
|
||||
).status_code
|
||||
== 200
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue