mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(proxy): preserve large JSON integers without regex scanning
This commit is contained in:
parent
72f1ac4a27
commit
f3a198e09f
2 changed files with 91 additions and 3 deletions
|
|
@ -7,6 +7,7 @@ from typing import Annotated, Any, Final, Literal, Union, get_args, get_origin
|
|||
import orjson
|
||||
from fastapi import Request, UploadFile, status
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic_core import from_json
|
||||
from starlette._utils import get_route_path
|
||||
from starlette.requests import HTTPConnection
|
||||
from typing_extensions import NotRequired, ReadOnly, Required, assert_never
|
||||
|
|
@ -162,10 +163,20 @@ def coerce_numeric_form_fields(
|
|||
}
|
||||
|
||||
|
||||
def _decode_json_body(body: bytes) -> object:
|
||||
# Preserve strict UTF-8, surrogate and non-finite-number validation without retaining rounded integers.
|
||||
orjson.loads(body)
|
||||
try:
|
||||
return _DECODED_JSON.validate_python(from_json(body, cache_strings=False))
|
||||
except ValueError:
|
||||
# The native decoder has a lower nesting limit than orjson.
|
||||
return _DECODED_JSON.validate_python(json.loads(body))
|
||||
|
||||
|
||||
def _parse_binary_body(body: bytes) -> dict:
|
||||
"""JSON sent under a binary content type still parses; real binary (protobuf) carries no params -> {}."""
|
||||
try:
|
||||
parsed: Final = _DECODED_JSON.validate_python(orjson.loads(body))
|
||||
parsed: Final = _decode_json_body(body)
|
||||
if isinstance(parsed, dict):
|
||||
return parsed
|
||||
except orjson.JSONDecodeError:
|
||||
|
|
@ -273,7 +284,7 @@ async def read_request_body(request: Request | None) -> dict:
|
|||
parsed_body = {}
|
||||
else:
|
||||
try:
|
||||
parsed_body = dict(_JSON_REQUEST_BODY.validate_python(orjson.loads(body)))
|
||||
parsed_body = dict(_JSON_REQUEST_BODY.validate_python(_decode_json_body(body)))
|
||||
except orjson.JSONDecodeError as e:
|
||||
# The surrogate-repair fallback below runs two full-body re.sub
|
||||
# passes, which block the event loop on multi-MB malformed bodies.
|
||||
|
|
@ -421,7 +432,7 @@ def rewrite_request_model(
|
|||
)
|
||||
safe_set_request_parsed_body(request=request, parsed_body=body)
|
||||
request._json = body # pyright: ignore[reportPrivateUsage] # Starlette JSON cache
|
||||
request._body = orjson.dumps(body) # pyright: ignore[reportPrivateUsage] # Starlette body cache
|
||||
request._body = _JSON_REQUEST_BODY.dump_json(body) # pyright: ignore[reportPrivateUsage] # Starlette body cache
|
||||
|
||||
|
||||
def safe_get_request_headers(request: Request | None) -> dict:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import asyncio, gzip, importlib, os
|
||||
import io
|
||||
import json
|
||||
import tracemalloc
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from typing import Final, Literal, get_type_hints
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
|
@ -29,6 +30,7 @@ from litellm.proxy.common_utils.http_parsing_utils import (
|
|||
numeric_form_fields,
|
||||
populate_request_with_path_params,
|
||||
read_raw_json_body,
|
||||
rewrite_request_model,
|
||||
)
|
||||
from fastapi import Request as Request_http_parsing
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
|
@ -141,6 +143,81 @@ async def test_protobuf_body_is_not_parsed_as_json(content_type):
|
|||
assert await request.body() == body # body is still readable by the endpoint
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("content_type", ("application/json", "application/x-protobuf"))
|
||||
async def test_request_body_preserves_integer_precision_and_other_json_types(content_type: str):
|
||||
body: Final = (
|
||||
b'{"model":"integer-test","schema":{"maximum":12345678901234567890123,'
|
||||
b'"minimum":-9223372036854775809,"enum":[9007199254740993,18446744073709551615,'
|
||||
b'18446744073709551616,-9223372036854775808]},'
|
||||
b'"float":1.2345678901234568e22,"decimal":12345678901234567890123.0,'
|
||||
b'"text":"12345678901234567890123 \\ud83d\\ude00 \\\"escaped\\\"",'
|
||||
b'"flags":[true,false,null]}'
|
||||
)
|
||||
request: Final = _starlette_request(body, content_type)
|
||||
parsed: Final = await read_request_body(request)
|
||||
|
||||
assert parsed == json.loads(body)
|
||||
assert isinstance(parsed["schema"]["maximum"], int)
|
||||
assert isinstance(parsed["schema"]["minimum"], int)
|
||||
assert all(isinstance(value, int) for value in parsed["schema"]["enum"])
|
||||
assert isinstance(parsed["float"], float)
|
||||
assert isinstance(parsed["decimal"], float)
|
||||
assert await read_request_body(request) == parsed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"body",
|
||||
(b'{"n":1e400}', b'{"n":NaN}', b'{"text":"\\ud800"}', b'{"text":"\xff"}'),
|
||||
)
|
||||
async def test_binary_body_keeps_strict_json_validation(body: bytes):
|
||||
assert await read_request_body(_starlette_request(body, "application/x-protobuf")) == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("content_type", ("application/json", "application/x-protobuf"))
|
||||
async def test_deep_json_keeps_large_integers_above_the_repair_size_limit(
|
||||
content_type: str, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(http_parsing_utils, "MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB", 0.001)
|
||||
body: Final = b'{"nested":' + b"[" * 800 + b"12345678901234567890123" + b"]" * 800 + b"}"
|
||||
|
||||
assert await read_request_body(_starlette_request(body, content_type)) == json.loads(body)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_escape_heavy_json_uses_memory_proportional_to_body_size():
|
||||
payload: Final = {"text": "1234567890123456789" + '"' * 2_000_000}
|
||||
body: Final = json.dumps(payload).encode()
|
||||
request: Final = _starlette_request(body, "application/json")
|
||||
|
||||
tracemalloc.start()
|
||||
try:
|
||||
parsed: Final = await read_request_body(request)
|
||||
_, peak = tracemalloc.get_traced_memory()
|
||||
finally:
|
||||
tracemalloc.stop()
|
||||
|
||||
assert parsed == payload
|
||||
assert peak < 16 * len(body)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("cached", (False, True))
|
||||
async def test_model_alias_rewrite_preserves_large_integers_in_every_body_reader(cached: bool):
|
||||
payload: Final = {"model": "alias", "schema": {"maximum": 12345678901234567890123}}
|
||||
request: Final = _starlette_request(json.dumps(payload).encode(), "application/json")
|
||||
request_data: Final = await read_request_body(request) if cached else dict(payload)
|
||||
rewrite_request_model(request_data, request, "integer-test")
|
||||
expected: Final = {**payload, "model": "integer-test"}
|
||||
|
||||
assert request_data == expected
|
||||
assert await read_request_body(request) == expected
|
||||
assert await request.json() == expected
|
||||
assert json.loads(await request.body()) == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gzipped_json_trace_body_survives_auth_pre_read():
|
||||
body = gzip.compress(b'{"resourceSpans": []}')
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue