diff --git a/docs/my-website/release_notes/v1.81.9.md b/docs/my-website/release_notes/v1.81.9.md index 08b70e029e2..c34d3056cae 100644 --- a/docs/my-website/release_notes/v1.81.9.md +++ b/docs/my-website/release_notes/v1.81.9.md @@ -48,6 +48,13 @@ pip install litellm==1.81.9 - **UI Team Soft Budget Alerts** - [Set soft budgets on teams and receive email alerts when spending crosses the threshold — without blocking requests](../../docs/proxy/ui_team_soft_budget_alerts) - **Performance Optimizations** - Multiple performance improvements including ~40% Prometheus CPU reduction, LRU caching, and optimized logging paths - **LiteLLM Observatory** - [Automated 24-hour load tests](../../blog/litellm-observatory) +- **30% Faster Request Processing for Callback-Heavy Deployments** - [Performance improvement for callback heavy deployments][PR #20354](https://github.com/BerriAI/litellm/pull/20354) + +--- + +## 30% Faster Request Processing for Callback-Heavy Deployments + + If you use logging callbacks like Langfuse, Datadog, or Prometheus, every request was paying an unnecessary cost: three loops that re-sorted your callbacks on every single request, even though the callback list hadn't changed. The more callbacks you had configured, the more time was wasted. We moved this work to happen once at startup instead of on every request. For deployments with the default callback set, this is a ~30% speedup in request setup. For deployments with many callbacks configured, the improvement is even larger. --- diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index bd148ecb481..cb8b9ec0395 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -1,4 +1,3 @@ -import copy import hashlib import json import secrets @@ -642,6 +641,34 @@ def _sanitize_request_body_for_spend_logs_payload( return {k: _sanitize_value(v) for k, v in request_body.items()} +def _convert_to_json_serializable_dict(obj: Any) -> Any: + """ + Convert object to JSON-serializable dict, handling Pydantic models safely. + + This avoids pickle-based deepcopy which fails on Pydantic v2 models + containing _thread.RLock objects. + + Args: + obj: Object to convert (dict, list, Pydantic model, or primitive) + + Returns: + JSON-serializable version of the object + """ + if isinstance(obj, BaseModel): + # Use Pydantic's model_dump() instead of pickle + return obj.model_dump() + elif isinstance(obj, dict): + return {k: _convert_to_json_serializable_dict(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [_convert_to_json_serializable_dict(item) for item in obj] + elif hasattr(obj, "__dict__"): + # Handle objects with __dict__ attribute + return _convert_to_json_serializable_dict(obj.__dict__) + else: + # Primitives (str, int, float, bool, None) pass through + return obj + + def _get_proxy_server_request_for_spend_logs_payload( metadata: dict, litellm_params: dict, @@ -649,7 +676,7 @@ def _get_proxy_server_request_for_spend_logs_payload( ) -> str: """ Only store if _should_store_prompts_and_responses_in_spend_logs() is True - + If turn_off_message_logging is enabled, redact messages in the request body. """ if _should_store_prompts_and_responses_in_spend_logs(): @@ -674,9 +701,9 @@ def _get_proxy_server_request_for_spend_logs_payload( ), } - # If redaction is enabled, deep copy request body before redacting + # If redaction is enabled, convert to serializable dict before redacting if should_redact_message_logging(model_call_details=model_call_details): - _request_body = copy.deepcopy(_request_body) + _request_body = _convert_to_json_serializable_dict(_request_body) perform_redaction(model_call_details=_request_body, result=None) _request_body = _sanitize_request_body_for_spend_logs_payload(_request_body) @@ -736,9 +763,9 @@ def _get_response_for_spend_logs_payload( ), } - # If redaction is enabled, deep copy response before redacting + # If redaction is enabled, convert to serializable dict before redacting if should_redact_message_logging(model_call_details=model_call_details): - response_obj = copy.deepcopy(response_obj) + response_obj = _convert_to_json_serializable_dict(response_obj) response_obj = perform_redaction(model_call_details={}, result=response_obj) sanitized_wrapper = _sanitize_request_body_for_spend_logs_payload(