From c892e7cce6a23b1768373f136ad79ed185198d5e Mon Sep 17 00:00:00 2001 From: Varun Sripad Date: Wed, 28 Jan 2026 16:45:41 -0600 Subject: [PATCH] fix(litellm_logging.py): prevent double logging of success/failure events (Fixes #19929) --- litellm/litellm_core_utils/litellm_logging.py | 19 ++++++ .../test_prometheus_double_count.py | 65 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 tests/test_litellm/test_prometheus_double_count.py diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 0d9245a686a..2c7a5198163 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1492,6 +1492,25 @@ class Logging(LiteLLMLoggingBaseClass): stream: bool = False, ) -> bool: try: + # Check for double logging + # If we are checking for success, check if EITHER async or sync success has explicitly been logged + if "success" in event_type: + if ( + self.model_call_details.get("has_logged_async_success", False) + is True + or self.model_call_details.get("has_logged_sync_success", False) + is True + ): + return False + elif "failure" in event_type: + if ( + self.model_call_details.get("has_logged_async_failure", False) + is True + or self.model_call_details.get("has_logged_sync_failure", False) + is True + ): + return False + if self.model_call_details.get(f"has_logged_{event_type}", False) is True: return False diff --git a/tests/test_litellm/test_prometheus_double_count.py b/tests/test_litellm/test_prometheus_double_count.py new file mode 100644 index 00000000000..38acd5d066f --- /dev/null +++ b/tests/test_litellm/test_prometheus_double_count.py @@ -0,0 +1,65 @@ +import pytest +import time +from unittest.mock import MagicMock +from litellm.litellm_core_utils.litellm_logging import Logging + +def test_should_run_logging_double_count_reproduction(): + """ + Reproduction for Issue #19929. + + Verifies that 'should_run_logging' currently returns True for both 'async_success' + and 'sync_success' for the same request, leading to double counting. + """ + logging_obj = Logging( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hello"}], + stream=False, + call_type="completion", + start_time=time.time(), + litellm_call_id="test_id", + function_id="test_func_id" + ) + logging_obj.model_call_details = {} + + # 1. First call: async_success + should_run_async = logging_obj.should_run_logging(event_type="async_success", stream=False) + assert should_run_async is True + + # Simulate the handler running and setting the flag + logging_obj.model_call_details["has_logged_async_success"] = True + + # 2. Second call: sync_success (triggered by fallback logic) + should_run_sync = logging_obj.should_run_logging(event_type="sync_success", stream=False) + + # BUG: This currently returns True, causing double logging + # assert should_run_sync is True + + # Updated to assert correct behavior (Fix for #19929) + assert should_run_sync is False + +def test_should_run_logging_double_count_fix_expectation(): + """ + This test currently FAILS if the bug exists. + It represents the Desired Behavior. + """ + logging_obj = Logging( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hello"}], + stream=False, + call_type="completion", + start_time=time.time(), + litellm_call_id="test_id", + function_id="test_func_id" + ) + logging_obj.model_call_details = {} + + # 1. First call: async_success + should_run_async = logging_obj.should_run_logging(event_type="async_success", stream=False) + assert should_run_async is True + logging_obj.model_call_details["has_logged_async_success"] = True + + # 2. Second call: sync_success + should_run_sync = logging_obj.should_run_logging(event_type="sync_success", stream=False) + + # DESIRED: Should be False to prevent double counting + assert should_run_sync is False