fix(proxy): preserve large JSON integers without regex scanning

This commit is contained in:
moe-berri 2026-10-24 21:19:31 -07:00
parent 72f1ac4a27
commit f3a198e09f
2 changed files with 91 additions and 3 deletions

View file

@ -7,6 +7,7 @@ from typing import Annotated, Any, Final, Literal, Union, get_args, get_origin
import orjson
from fastapi import Request, UploadFile, status
from pydantic import TypeAdapter
from pydantic_core import from_json
from starlette._utils import get_route_path
from starlette.requests import HTTPConnection
from typing_extensions import NotRequired, ReadOnly, Required, assert_never
@ -162,10 +163,20 @@ def coerce_numeric_form_fields(
}
def _decode_json_body(body: bytes) -> object:
# Preserve strict UTF-8, surrogate and non-finite-number validation without retaining rounded integers.
orjson.loads(body)
try:
return _DECODED_JSON.validate_python(from_json(body, cache_strings=False))
except ValueError:
# The native decoder has a lower nesting limit than orjson.
return _DECODED_JSON.validate_python(json.loads(body))
def _parse_binary_body(body: bytes) -> dict:
"""JSON sent under a binary content type still parses; real binary (protobuf) carries no params -> {}."""
try:
parsed: Final = _DECODED_JSON.validate_python(orjson.loads(body))
parsed: Final = _decode_json_body(body)
if isinstance(parsed, dict):
return parsed
except orjson.JSONDecodeError:
@ -273,7 +284,7 @@ async def read_request_body(request: Request | None) -> dict:
parsed_body = {}
else:
try:
parsed_body = dict(_JSON_REQUEST_BODY.validate_python(orjson.loads(body)))
parsed_body = dict(_JSON_REQUEST_BODY.validate_python(_decode_json_body(body)))
except orjson.JSONDecodeError as e:
# The surrogate-repair fallback below runs two full-body re.sub
# passes, which block the event loop on multi-MB malformed bodies.
@ -421,7 +432,7 @@ def rewrite_request_model(
)
safe_set_request_parsed_body(request=request, parsed_body=body)
request._json = body # pyright: ignore[reportPrivateUsage] # Starlette JSON cache
request._body = orjson.dumps(body) # pyright: ignore[reportPrivateUsage] # Starlette body cache
request._body = _JSON_REQUEST_BODY.dump_json(body) # pyright: ignore[reportPrivateUsage] # Starlette body cache
def safe_get_request_headers(request: Request | None) -> dict:

View file

@ -1,6 +1,7 @@
import asyncio, gzip, importlib, os
import io
import json
import tracemalloc
from collections.abc import Awaitable, Callable, Mapping
from typing import Final, Literal, get_type_hints
from unittest.mock import AsyncMock, MagicMock, patch
@ -29,6 +30,7 @@ from litellm.proxy.common_utils.http_parsing_utils import (
numeric_form_fields,
populate_request_with_path_params,
read_raw_json_body,
rewrite_request_model,
)
from fastapi import Request as Request_http_parsing
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
@ -141,6 +143,81 @@ async def test_protobuf_body_is_not_parsed_as_json(content_type):
assert await request.body() == body # body is still readable by the endpoint
@pytest.mark.asyncio
@pytest.mark.parametrize("content_type", ("application/json", "application/x-protobuf"))
async def test_request_body_preserves_integer_precision_and_other_json_types(content_type: str):
body: Final = (
b'{"model":"integer-test","schema":{"maximum":12345678901234567890123,'
b'"minimum":-9223372036854775809,"enum":[9007199254740993,18446744073709551615,'
b'18446744073709551616,-9223372036854775808]},'
b'"float":1.2345678901234568e22,"decimal":12345678901234567890123.0,'
b'"text":"12345678901234567890123 \\ud83d\\ude00 \\\"escaped\\\"",'
b'"flags":[true,false,null]}'
)
request: Final = _starlette_request(body, content_type)
parsed: Final = await read_request_body(request)
assert parsed == json.loads(body)
assert isinstance(parsed["schema"]["maximum"], int)
assert isinstance(parsed["schema"]["minimum"], int)
assert all(isinstance(value, int) for value in parsed["schema"]["enum"])
assert isinstance(parsed["float"], float)
assert isinstance(parsed["decimal"], float)
assert await read_request_body(request) == parsed
@pytest.mark.asyncio
@pytest.mark.parametrize(
"body",
(b'{"n":1e400}', b'{"n":NaN}', b'{"text":"\\ud800"}', b'{"text":"\xff"}'),
)
async def test_binary_body_keeps_strict_json_validation(body: bytes):
assert await read_request_body(_starlette_request(body, "application/x-protobuf")) == {}
@pytest.mark.asyncio
@pytest.mark.parametrize("content_type", ("application/json", "application/x-protobuf"))
async def test_deep_json_keeps_large_integers_above_the_repair_size_limit(
content_type: str, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setattr(http_parsing_utils, "MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB", 0.001)
body: Final = b'{"nested":' + b"[" * 800 + b"12345678901234567890123" + b"]" * 800 + b"}"
assert await read_request_body(_starlette_request(body, content_type)) == json.loads(body)
@pytest.mark.asyncio
async def test_escape_heavy_json_uses_memory_proportional_to_body_size():
payload: Final = {"text": "1234567890123456789" + '"' * 2_000_000}
body: Final = json.dumps(payload).encode()
request: Final = _starlette_request(body, "application/json")
tracemalloc.start()
try:
parsed: Final = await read_request_body(request)
_, peak = tracemalloc.get_traced_memory()
finally:
tracemalloc.stop()
assert parsed == payload
assert peak < 16 * len(body)
@pytest.mark.asyncio
@pytest.mark.parametrize("cached", (False, True))
async def test_model_alias_rewrite_preserves_large_integers_in_every_body_reader(cached: bool):
payload: Final = {"model": "alias", "schema": {"maximum": 12345678901234567890123}}
request: Final = _starlette_request(json.dumps(payload).encode(), "application/json")
request_data: Final = await read_request_body(request) if cached else dict(payload)
rewrite_request_model(request_data, request, "integer-test")
expected: Final = {**payload, "model": "integer-test"}
assert request_data == expected
assert await read_request_body(request) == expected
assert await request.json() == expected
assert json.loads(await request.body()) == expected
@pytest.mark.asyncio
async def test_gzipped_json_trace_body_survives_auth_pre_read():
body = gzip.compress(b'{"resourceSpans": []}')