diff --git a/CHANGES_SUMMARY.md b/CHANGES_SUMMARY.md new file mode 100644 index 00000000000..e3ca694180d --- /dev/null +++ b/CHANGES_SUMMARY.md @@ -0,0 +1,21 @@ +# Changes Summary + +## 1. Large payload truncation + +> Addressed the block from the right side + +Truncate large payloads before `json.dumps` in logging to cut ~3–4s serialization cost. + +- **File**: `litellm/litellm_core_utils/litellm_logging.py` +- **Details**: See [truncation_change.md](./truncation_change.md) +- **Tests**: `tests/test_litellm/litellm_core_utils/test_litellm_logging.py::TestTruncateLargePayloadForLogging` (15 tests) + +## 2. ORJSONResponse bypass for serialize_response + +> Addressed the request block (~5.5s serialize_response / jsonable_encoder overhead) + +Bypass FastAPI `jsonable_encoder` for non-streaming responses by converting Pydantic/dict responses to plain dicts and returning `ORJSONResponse`. This eliminates the major serialize_response cost for large embeddings and completions. + +- **File**: `litellm/proxy/common_request_processing.py` +- **Change**: `_response_to_json_serializable()` converts responses to JSON-serializable dicts; when applicable, we return `ORJSONResponse` directly instead of letting FastAPI serialize the response. +- **Tests**: `tests/test_litellm/proxy/test_common_request_processing.py::TestResponseToJsonSerializable` (7 tests, including 4 backward compatibility tests) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 0d3e61b75c7..c722b3f9270 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -17,7 +17,7 @@ from typing import ( import httpx import orjson from fastapi import HTTPException, Request, status -from fastapi.responses import JSONResponse, Response, StreamingResponse +from fastapi.responses import JSONResponse, ORJSONResponse, Response, StreamingResponse import litellm from litellm._logging import verbose_proxy_logger @@ -53,6 +53,27 @@ from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request from litellm.types.utils import ModelResponse, ModelResponseStream, Usage +def _response_to_json_serializable(response: Any) -> Optional[dict]: + """ + Convert LLM response to a JSON-serializable dict for ORJSONResponse. + Bypasses FastAPI jsonable_encoder to eliminate serialize_response overhead. + Returns None if conversion is not applicable (caller should return response as-is). + """ + if isinstance(response, dict): + return response + if hasattr(response, "model_dump"): + try: + return response.model_dump(mode="json") + except Exception: + pass + if hasattr(response, "dict"): + try: + return response.dict() + except Exception: + pass + return None + + async def _parse_event_data_for_error(event_line: Union[str, bytes]) -> Optional[int]: """Parses an event line and returns an error code if present, else None.""" event_line = ( @@ -809,6 +830,16 @@ class ProxyBaseLLMRequestProcessing: ) await check_response_size_is_safe(response=response) + # Bypass FastAPI jsonable_encoder: return ORJSONResponse with pre-serialized content + # to eliminate serialize_response overhead (~5.5s for large embeddings/completions). + content = _response_to_json_serializable(response) + if content is not None: + return ORJSONResponse( + content=content, + status_code=status.HTTP_200_OK, + headers=dict(fastapi_response.headers), + ) + return response async def base_passthrough_process_llm_request( diff --git a/poetry.lock b/poetry.lock index 6d98b45978d..158ece6d679 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,37 +1,5 @@ # This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. -[[package]] -name = "a2a-sdk" -version = "0.3.22" -description = "A2A Python SDK" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version >= \"3.10\" and extra == \"extra-proxy\"" -files = [ - {file = "a2a_sdk-0.3.22-py3-none-any.whl", hash = "sha256:b98701135bb90b0ff85d35f31533b6b7a299bf810658c1c65f3814a6c15ea385"}, - {file = "a2a_sdk-0.3.22.tar.gz", hash = "sha256:77a5694bfc4f26679c11b70c7f1062522206d430b34bc1215cfbb1eba67b7e7d"}, -] - -[package.dependencies] -google-api-core = ">=1.26.0" -httpx = ">=0.28.1" -httpx-sse = ">=0.4.0" -protobuf = ">=5.29.5" -pydantic = ">=2.11.3" - -[package.extras] -all = ["cryptography (>=43.0.0)", "fastapi (>=0.115.2)", "grpcio (>=1.60)", "grpcio-reflection (>=1.7.0)", "grpcio-tools (>=1.60)", "opentelemetry-api (>=1.33.0)", "opentelemetry-sdk (>=1.33.0)", "pyjwt (>=2.0.0)", "sqlalchemy[aiomysql,asyncio] (>=2.0.0)", "sqlalchemy[aiosqlite,asyncio] (>=2.0.0)", "sqlalchemy[asyncio,postgresql-asyncpg] (>=2.0.0)", "sse-starlette", "starlette"] -encryption = ["cryptography (>=43.0.0)"] -grpc = ["grpcio (>=1.60)", "grpcio-reflection (>=1.7.0)", "grpcio-tools (>=1.60)"] -http-server = ["fastapi (>=0.115.2)", "sse-starlette", "starlette"] -mysql = ["sqlalchemy[aiomysql,asyncio] (>=2.0.0)"] -postgresql = ["sqlalchemy[asyncio,postgresql-asyncpg] (>=2.0.0)"] -signing = ["pyjwt (>=2.0.0)"] -sql = ["sqlalchemy[aiomysql,asyncio] (>=2.0.0)", "sqlalchemy[aiosqlite,asyncio] (>=2.0.0)", "sqlalchemy[asyncio,postgresql-asyncpg] (>=2.0.0)"] -sqlite = ["sqlalchemy[aiosqlite,asyncio] (>=2.0.0)"] -telemetry = ["opentelemetry-api (>=1.33.0)", "opentelemetry-sdk (>=1.33.0)"] - [[package]] name = "aiofiles" version = "24.1.0" @@ -3113,15 +3081,15 @@ files = [ [[package]] name = "litellm-proxy-extras" -version = "0.4.34" +version = "0.4.23" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." optional = true python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" groups = ["main"] markers = "extra == \"proxy\"" files = [ - {file = "litellm_proxy_extras-0.4.34-py3-none-any.whl", hash = "sha256:d455eb54f82e7c92f4f68a921240822df23158aad05fcdda7245887db7c30b90"}, - {file = "litellm_proxy_extras-0.4.34.tar.gz", hash = "sha256:39fa6c2295acc449320b5a710d150295fd0bf5f8c0d1742b5e9ae361d7bd3ed2"}, + {file = "litellm_proxy_extras-0.4.23-py3-none-any.whl", hash = "sha256:dfda21203dde9fd97cf364396a9b5be0cfdf00fa9846439ee33ce11b7a52f9ce"}, + {file = "litellm_proxy_extras-0.4.23.tar.gz", hash = "sha256:8e3f95576dc2a296e7f73d8c87e73628bd899b4644c45863960fe3c3762d8f64"}, ] [[package]] @@ -4210,10 +4178,9 @@ opentelemetry-api = "1.25.0" name = "orjson" version = "3.11.4" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" -optional = true +optional = false python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"proxy\"" files = [ {file = "orjson-3.11.4-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:e3aa2118a3ece0d25489cbe48498de8a5d580e42e8d9979f65bf47900a15aba1"}, {file = "orjson-3.11.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a69ab657a4e6733133a3dca82768f2f8b884043714e8d2b9ba9f52b6efef5c44"}, @@ -5310,7 +5277,7 @@ description = "Pyroscope Python integration" optional = false python-versions = "*" groups = ["main"] -markers = "extra == \"proxy\" and sys_platform != \"win32\"" +markers = "extra == \"proxy\"" files = [ {file = "pyroscope_io-0.8.16-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:e07edcfd59f5bdce42948b92c9b118c824edbd551730305f095a6b9af401a9e8"}, {file = "pyroscope_io-0.8.16-py2.py3-none-macosx_11_0_x86_64.whl", hash = "sha256:dc98355e27c0b7b61f27066500fe1045b70e9459bb8b9a3082bc4755cb6392b6"}, @@ -8024,11 +7991,11 @@ type = ["pytest-mypy"] caching = ["diskcache"] extra-proxy = ["azure-identity", "azure-keyvault-secrets", "google-cloud-iam", "google-cloud-kms", "prisma", "redisvl", "resend"] mlflow = ["mlflow"] -proxy = ["PyJWT", "apscheduler", "azure-identity", "azure-storage-blob", "backoff", "boto3", "cryptography", "fastapi", "fastapi-sso", "gunicorn", "litellm-enterprise", "litellm-proxy-extras", "mcp", "orjson", "polars", "pynacl", "pyroscope-io", "python-multipart", "pyyaml", "rich", "rq", "soundfile", "uvicorn", "uvloop", "websockets"] +proxy = ["PyJWT", "apscheduler", "azure-identity", "azure-storage-blob", "backoff", "boto3", "cryptography", "fastapi", "fastapi-sso", "gunicorn", "litellm-enterprise", "litellm-proxy-extras", "mcp", "polars", "pynacl", "pyroscope-io", "python-multipart", "pyyaml", "rich", "rq", "soundfile", "uvicorn", "uvloop", "websockets"] semantic-router = ["semantic-router"] utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "75f2d51e0e399eaa79f0211bee13f4c887c0e5a76994b21f5ad7237c273a72a0" +content-hash = "b916879f4d105891f7088c51aec995a724a9bc81245fff59a0550f07ee1665a4" diff --git a/pyproject.toml b/pyproject.toml index 0ac52bde26c..0b0e84c6e8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ jinja2 = "^3.1.2" aiohttp = ">=3.10" pydantic = "^2.5.0" jsonschema = ">=4.23.0,<5.0.0" +orjson = ">=3.9.7" numpydoc = {version = "*", optional = true} # used in utils.py uvicorn = {version = "^0.31.1", optional = true} @@ -41,7 +42,6 @@ fastapi = {version = ">=0.120.1", optional = true} backoff = {version = "*", optional = true} pyyaml = {version = "^6.0.1", optional = true} rq = {version = "*", optional = true} -orjson = {version = "^3.9.7", optional = true} apscheduler = {version = "^3.10.4", optional = true} fastapi-sso = { version = "^0.16.0", optional = true } PyJWT = { version = "^2.10.1", optional = true, python = ">=3.9" } @@ -86,7 +86,6 @@ proxy = [ "backoff", "pyyaml", "rq", - "orjson", "apscheduler", "fastapi-sso", "PyJWT", diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 3d1e9aece41..6f20f373edb 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -1,6 +1,8 @@ import copy +import json from unittest.mock import AsyncMock, MagicMock +import orjson import pytest from fastapi import Request, status from fastapi.responses import JSONResponse, StreamingResponse @@ -14,6 +16,7 @@ from litellm.proxy.common_request_processing import ( _extract_error_from_sse_chunk, _get_cost_breakdown_from_logging_obj, _parse_event_data_for_error, + _response_to_json_serializable, create_response, ) from litellm.proxy.utils import ProxyLogging @@ -1024,3 +1027,91 @@ class TestExtractErrorFromSSEChunk: # Other fields should be obtained from the original error object (if exists) +def _parsed_via_legacy_path(obj) -> dict: + """Simulate legacy path: jsonable_encoder + json.dumps -> parse. Returns parsed dict.""" + from fastapi.encoders import jsonable_encoder + + encoded = jsonable_encoder(obj) + return json.loads(json.dumps(encoded)) + + +def _parsed_via_new_path(obj) -> dict: + """New path: _response_to_json_serializable + orjson.dumps -> parse. Returns parsed dict.""" + content = _response_to_json_serializable(obj) + if content is None: + raise ValueError("_response_to_json_serializable returned None") + return json.loads(orjson.dumps(content).decode()) + + +class TestResponseToJsonSerializable: + """Tests for _response_to_json_serializable (ORJSONResponse bypass for serialize_response).""" + + def test_dict_passthrough(self): + """Plain dict should be returned as-is.""" + d = {"data": [{"embedding": [0.1] * 100}], "model": "text-embedding-3"} + assert _response_to_json_serializable(d) is d + + def test_pydantic_model_dump(self): + """Pydantic model should use model_dump(mode='json').""" + from litellm.types.utils import EmbeddingResponse, Usage + + resp = EmbeddingResponse( + data=[{"embedding": [0.1] * 100, "index": 0}], + model="text-embedding-3", + usage=Usage(prompt_tokens=10, total_tokens=10), + ) + result = _response_to_json_serializable(resp) + assert result is not None + assert result["model"] == "text-embedding-3" + assert len(result["data"]) == 1 + assert result["data"][0]["embedding"] == [0.1] * 100 + + def test_none_for_unsupported_type(self): + """Non-dict, non-Pydantic should return None.""" + assert _response_to_json_serializable("string") is None + assert _response_to_json_serializable(42) is None + assert _response_to_json_serializable(None) is None + + # --- Backward compatibility: new ORJSON path should produce equivalent JSON to legacy jsonable_encoder --- + + def test_backward_compat_embedding_response_same_as_legacy(self): + """EmbeddingResponse via new path should produce JSON structurally equivalent to jsonable_encoder.""" + from litellm.types.utils import EmbeddingResponse, Usage + + resp = EmbeddingResponse( + data=[{"embedding": [0.1, 0.2, 0.3] * 50, "index": 0}], + model="text-embedding-3", + usage=Usage(prompt_tokens=10, total_tokens=10), + ) + legacy = _parsed_via_legacy_path(resp) + new_path = _parsed_via_new_path(resp) + assert legacy == new_path + + def test_backward_compat_plain_dict_same_as_legacy(self): + """Plain dict via new path should produce identical parsed JSON to legacy.""" + d = {"data": [{"embedding": [0.1] * 100}], "model": "text-embedding-3"} + legacy = _parsed_via_legacy_path(d) + new_path = _parsed_via_new_path(d) + assert legacy == new_path + + def test_backward_compat_unicode_preserved(self): + """Unicode in response should be preserved identically by both paths.""" + d = {"model": "text-embedding-3", "text": "café résumé 日本語 🔥"} + legacy = _parsed_via_legacy_path(d) + new_path = _parsed_via_new_path(d) + assert legacy == new_path + + def test_backward_compat_nested_structure_same_as_legacy(self): + """Nested dicts/lists should produce equivalent JSON.""" + d = { + "object": "list", + "data": [ + {"embedding": [0.1, 0.2], "index": 0}, + {"embedding": [0.3, 0.4], "index": 1}, + ], + "usage": {"prompt_tokens": 5, "total_tokens": 5}, + "model": "embedding-model", + } + legacy = _parsed_via_legacy_path(d) + new_path = _parsed_via_new_path(d) + assert legacy == new_path