From c9c6a5edc971c55cb5cc26cc5911faba6ee816aa Mon Sep 17 00:00:00 2001 From: Varun Chawla <34209028+veeceey@users.noreply.github.com> Date: Sat, 7 Feb 2026 23:02:29 -0800 Subject: [PATCH] Fix: Spend logs pickle error with Pydantic models and redaction (#20685) * docs: add callback registration optimization to v1.81.9 release notes (#20681) * docs: add callback registration optimization to v1.81.9 release notes * Update v1.81.9.md --------- Co-authored-by: Alexsander Hamir * Fix spend logs pickle error with Pydantic models Replace copy.deepcopy() with Pydantic-safe serialization to avoid "cannot pickle '_thread.RLock' object" errors when request/response redaction is enabled. Changes: - Add _convert_to_json_serializable_dict() helper that uses model_dump() for Pydantic models instead of pickle - Replace copy.deepcopy() calls in request and response redaction paths with the new helper function - Recursively handles nested dicts, lists, and Pydantic models Root cause: Pydantic v2 BaseModel instances contain internal _thread.RLock objects for thread-safety. When copy.deepcopy() attempts to pickle these objects, it fails because threading primitives cannot be pickled. Fixes #20647 * chore: remove unused copy import Remove unused copy import that was causing lint failure. The copy.deepcopy() calls were replaced with _convert_to_json_serializable_dict() helper function in the previous commit, making the copy module no longer needed. --------- Co-authored-by: ryan-crabbe <128659760+ryan-crabbe@users.noreply.github.com> Co-authored-by: Alexsander Hamir --- docs/my-website/release_notes/v1.81.9.md | 7 ++++ .../spend_tracking/spend_tracking_utils.py | 39 ++++++++++++++++--- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/docs/my-website/release_notes/v1.81.9.md b/docs/my-website/release_notes/v1.81.9.md index 08b70e029e2..c34d3056cae 100644 --- a/docs/my-website/release_notes/v1.81.9.md +++ b/docs/my-website/release_notes/v1.81.9.md @@ -48,6 +48,13 @@ pip install litellm==1.81.9 - **UI Team Soft Budget Alerts** - [Set soft budgets on teams and receive email alerts when spending crosses the threshold — without blocking requests](../../docs/proxy/ui_team_soft_budget_alerts) - **Performance Optimizations** - Multiple performance improvements including ~40% Prometheus CPU reduction, LRU caching, and optimized logging paths - **LiteLLM Observatory** - [Automated 24-hour load tests](../../blog/litellm-observatory) +- **30% Faster Request Processing for Callback-Heavy Deployments** - [Performance improvement for callback heavy deployments][PR #20354](https://github.com/BerriAI/litellm/pull/20354) + +--- + +## 30% Faster Request Processing for Callback-Heavy Deployments + + If you use logging callbacks like Langfuse, Datadog, or Prometheus, every request was paying an unnecessary cost: three loops that re-sorted your callbacks on every single request, even though the callback list hadn't changed. The more callbacks you had configured, the more time was wasted. We moved this work to happen once at startup instead of on every request. For deployments with the default callback set, this is a ~30% speedup in request setup. For deployments with many callbacks configured, the improvement is even larger. --- diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index bd148ecb481..cb8b9ec0395 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -1,4 +1,3 @@ -import copy import hashlib import json import secrets @@ -642,6 +641,34 @@ def _sanitize_request_body_for_spend_logs_payload( return {k: _sanitize_value(v) for k, v in request_body.items()} +def _convert_to_json_serializable_dict(obj: Any) -> Any: + """ + Convert object to JSON-serializable dict, handling Pydantic models safely. + + This avoids pickle-based deepcopy which fails on Pydantic v2 models + containing _thread.RLock objects. + + Args: + obj: Object to convert (dict, list, Pydantic model, or primitive) + + Returns: + JSON-serializable version of the object + """ + if isinstance(obj, BaseModel): + # Use Pydantic's model_dump() instead of pickle + return obj.model_dump() + elif isinstance(obj, dict): + return {k: _convert_to_json_serializable_dict(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [_convert_to_json_serializable_dict(item) for item in obj] + elif hasattr(obj, "__dict__"): + # Handle objects with __dict__ attribute + return _convert_to_json_serializable_dict(obj.__dict__) + else: + # Primitives (str, int, float, bool, None) pass through + return obj + + def _get_proxy_server_request_for_spend_logs_payload( metadata: dict, litellm_params: dict, @@ -649,7 +676,7 @@ def _get_proxy_server_request_for_spend_logs_payload( ) -> str: """ Only store if _should_store_prompts_and_responses_in_spend_logs() is True - + If turn_off_message_logging is enabled, redact messages in the request body. """ if _should_store_prompts_and_responses_in_spend_logs(): @@ -674,9 +701,9 @@ def _get_proxy_server_request_for_spend_logs_payload( ), } - # If redaction is enabled, deep copy request body before redacting + # If redaction is enabled, convert to serializable dict before redacting if should_redact_message_logging(model_call_details=model_call_details): - _request_body = copy.deepcopy(_request_body) + _request_body = _convert_to_json_serializable_dict(_request_body) perform_redaction(model_call_details=_request_body, result=None) _request_body = _sanitize_request_body_for_spend_logs_payload(_request_body) @@ -736,9 +763,9 @@ def _get_response_for_spend_logs_payload( ), } - # If redaction is enabled, deep copy response before redacting + # If redaction is enabled, convert to serializable dict before redacting if should_redact_message_logging(model_call_details=model_call_details): - response_obj = copy.deepcopy(response_obj) + response_obj = _convert_to_json_serializable_dict(response_obj) response_obj = perform_redaction(model_call_details={}, result=response_obj) sanitized_wrapper = _sanitize_request_body_for_spend_logs_payload(