diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index a38769ac41b..2c6b756c702 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -7,6 +7,7 @@ from typing import Annotated, Any, Final, Literal, Union, get_args, get_origin import orjson from fastapi import Request, UploadFile, status from pydantic import TypeAdapter +from pydantic_core import from_json from starlette._utils import get_route_path from starlette.requests import HTTPConnection from typing_extensions import NotRequired, ReadOnly, Required, assert_never @@ -162,10 +163,20 @@ def coerce_numeric_form_fields( } +def _decode_json_body(body: bytes) -> object: + # Preserve strict UTF-8, surrogate and non-finite-number validation without retaining rounded integers. + orjson.loads(body) + try: + return _DECODED_JSON.validate_python(from_json(body, cache_strings=False)) + except ValueError: + # The native decoder has a lower nesting limit than orjson. + return _DECODED_JSON.validate_python(json.loads(body)) + + def _parse_binary_body(body: bytes) -> dict: """JSON sent under a binary content type still parses; real binary (protobuf) carries no params -> {}.""" try: - parsed: Final = _DECODED_JSON.validate_python(orjson.loads(body)) + parsed: Final = _decode_json_body(body) if isinstance(parsed, dict): return parsed except orjson.JSONDecodeError: @@ -273,7 +284,7 @@ async def read_request_body(request: Request | None) -> dict: parsed_body = {} else: try: - parsed_body = dict(_JSON_REQUEST_BODY.validate_python(orjson.loads(body))) + parsed_body = dict(_JSON_REQUEST_BODY.validate_python(_decode_json_body(body))) except orjson.JSONDecodeError as e: # The surrogate-repair fallback below runs two full-body re.sub # passes, which block the event loop on multi-MB malformed bodies. @@ -421,7 +432,7 @@ def rewrite_request_model( ) safe_set_request_parsed_body(request=request, parsed_body=body) request._json = body # pyright: ignore[reportPrivateUsage] # Starlette JSON cache - request._body = orjson.dumps(body) # pyright: ignore[reportPrivateUsage] # Starlette body cache + request._body = _JSON_REQUEST_BODY.dump_json(body) # pyright: ignore[reportPrivateUsage] # Starlette body cache def safe_get_request_headers(request: Request | None) -> dict: diff --git a/tests/unit/proxy/common_utils/test_http_parsing_utils.py b/tests/unit/proxy/common_utils/test_http_parsing_utils.py index cd5820fa2fd..06386070fe5 100644 --- a/tests/unit/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/unit/proxy/common_utils/test_http_parsing_utils.py @@ -1,6 +1,7 @@ import asyncio, gzip, importlib, os import io import json +import tracemalloc from collections.abc import Awaitable, Callable, Mapping from typing import Final, Literal, get_type_hints from unittest.mock import AsyncMock, MagicMock, patch @@ -29,6 +30,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( numeric_form_fields, populate_request_with_path_params, read_raw_json_body, + rewrite_request_model, ) from fastapi import Request as Request_http_parsing from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER @@ -141,6 +143,81 @@ async def test_protobuf_body_is_not_parsed_as_json(content_type): assert await request.body() == body # body is still readable by the endpoint +@pytest.mark.asyncio +@pytest.mark.parametrize("content_type", ("application/json", "application/x-protobuf")) +async def test_request_body_preserves_integer_precision_and_other_json_types(content_type: str): + body: Final = ( + b'{"model":"integer-test","schema":{"maximum":12345678901234567890123,' + b'"minimum":-9223372036854775809,"enum":[9007199254740993,18446744073709551615,' + b'18446744073709551616,-9223372036854775808]},' + b'"float":1.2345678901234568e22,"decimal":12345678901234567890123.0,' + b'"text":"12345678901234567890123 \\ud83d\\ude00 \\\"escaped\\\"",' + b'"flags":[true,false,null]}' + ) + request: Final = _starlette_request(body, content_type) + parsed: Final = await read_request_body(request) + + assert parsed == json.loads(body) + assert isinstance(parsed["schema"]["maximum"], int) + assert isinstance(parsed["schema"]["minimum"], int) + assert all(isinstance(value, int) for value in parsed["schema"]["enum"]) + assert isinstance(parsed["float"], float) + assert isinstance(parsed["decimal"], float) + assert await read_request_body(request) == parsed + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "body", + (b'{"n":1e400}', b'{"n":NaN}', b'{"text":"\\ud800"}', b'{"text":"\xff"}'), +) +async def test_binary_body_keeps_strict_json_validation(body: bytes): + assert await read_request_body(_starlette_request(body, "application/x-protobuf")) == {} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("content_type", ("application/json", "application/x-protobuf")) +async def test_deep_json_keeps_large_integers_above_the_repair_size_limit( + content_type: str, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(http_parsing_utils, "MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB", 0.001) + body: Final = b'{"nested":' + b"[" * 800 + b"12345678901234567890123" + b"]" * 800 + b"}" + + assert await read_request_body(_starlette_request(body, content_type)) == json.loads(body) + + +@pytest.mark.asyncio +async def test_escape_heavy_json_uses_memory_proportional_to_body_size(): + payload: Final = {"text": "1234567890123456789" + '"' * 2_000_000} + body: Final = json.dumps(payload).encode() + request: Final = _starlette_request(body, "application/json") + + tracemalloc.start() + try: + parsed: Final = await read_request_body(request) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert parsed == payload + assert peak < 16 * len(body) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cached", (False, True)) +async def test_model_alias_rewrite_preserves_large_integers_in_every_body_reader(cached: bool): + payload: Final = {"model": "alias", "schema": {"maximum": 12345678901234567890123}} + request: Final = _starlette_request(json.dumps(payload).encode(), "application/json") + request_data: Final = await read_request_body(request) if cached else dict(payload) + rewrite_request_model(request_data, request, "integer-test") + expected: Final = {**payload, "model": "integer-test"} + + assert request_data == expected + assert await read_request_body(request) == expected + assert await request.json() == expected + assert json.loads(await request.body()) == expected + + @pytest.mark.asyncio async def test_gzipped_json_trace_body_survives_auth_pre_read(): body = gzip.compress(b'{"resourceSpans": []}')