From 2cfba944aeac3f68876c71eb41690907062b852e Mon Sep 17 00:00:00 2001 From: MOHIT PRASANTH KALAHASTHI <22170424+mohitprasanth@users.noreply.github.com> Date: Thu, 6 Aug 2026 22:23:44 +0530 Subject: [PATCH 1/6] Test that no-log skips observability callbacks but not spend tracking --- .../test_no_log_spend_tracking.py | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 tests/test_litellm/litellm_core_utils/test_no_log_spend_tracking.py diff --git a/tests/test_litellm/litellm_core_utils/test_no_log_spend_tracking.py b/tests/test_litellm/litellm_core_utils/test_no_log_spend_tracking.py new file mode 100644 index 00000000000..2193d48416f --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_no_log_spend_tracking.py @@ -0,0 +1,128 @@ +"""`no-log` must not disable spend tracking. + +`no-log` means "do not send this request to logging integrations". It is not a +way for a caller to opt out of being billed, and `Logging.should_run_callback` +has always intended to exempt the proxy's cost-tracking callback from it. The +exemption used to be a case-sensitive substring test for `"_PROXY_"` in the +callback's class name, which never matched the cost callback's actual class +(`_ProxyDBLogger`), so spend tracking was silently skipped for every `no-log` +request. Callbacks now declare the behaviour explicitly via `runs_on_no_log`. +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../../..")) + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger + + +class _ObservabilityLogger(CustomLogger): + """A stand-in for langfuse/datadog/etc. -- must be skipped on no-log.""" + + +class _MeteringLogger(CustomLogger): + """A stand-in for a billing callback -- must still run on no-log.""" + + runs_on_no_log = True + + +def _logging_obj() -> LiteLLMLogging: + return LiteLLMLogging( + model="gpt-4o", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="acompletion", + start_time=None, + litellm_call_id="test-call-id", + function_id="test-function-id", + ) + + +def test_cost_callback_still_runs_on_no_log(): + """The regression: _ProxyDBLogger must survive a no-log request.""" + assert _ProxyDBLogger.runs_on_no_log is True + assert ( + _logging_obj().should_run_callback( + callback=_ProxyDBLogger(), + litellm_params={"no-log": True}, + event_hook="async_log_success_event", + ) + is True + ) + + +def test_observability_callback_is_skipped_on_no_log(): + """no-log still does what it says for logging integrations.""" + assert _ObservabilityLogger.runs_on_no_log is False + assert ( + _logging_obj().should_run_callback( + callback=_ObservabilityLogger(), + litellm_params={"no-log": True}, + event_hook="async_log_success_event", + ) + is False + ) + + +def test_runs_on_no_log_is_opt_in_for_any_callback(): + """Any callback can declare itself as infrastructure, not just the proxy's.""" + assert ( + _logging_obj().should_run_callback( + callback=_MeteringLogger(), + litellm_params={"no-log": True}, + event_hook="async_log_success_event", + ) + is True + ) + + +@pytest.mark.parametrize("litellm_params", [{}, {"no-log": False}]) +def test_everything_runs_when_no_log_is_absent_or_false(litellm_params): + for callback in (_ObservabilityLogger(), _ProxyDBLogger()): + assert ( + _logging_obj().should_run_callback( + callback=callback, + litellm_params=litellm_params, + event_hook="async_log_success_event", + ) + is True + ) + + +def test_legacy_underscore_proxy_naming_still_exempt(): + """Callbacks named _PROXY_* keep working without setting the new flag.""" + + class _PROXY_LegacyHandler(CustomLogger): + pass + + assert _PROXY_LegacyHandler.runs_on_no_log is False + assert ( + _logging_obj().should_run_callback( + callback=_PROXY_LegacyHandler(), + litellm_params={"no-log": True}, + event_hook="async_log_success_event", + ) + is True + ) + + +def test_global_disable_no_log_param_overrides_everything(): + original = litellm.global_disable_no_log_param + litellm.global_disable_no_log_param = True + try: + assert ( + _logging_obj().should_run_callback( + callback=_ObservabilityLogger(), + litellm_params={"no-log": True}, + event_hook="async_log_success_event", + ) + is True + ) + finally: + litellm.global_disable_no_log_param = original From 129220bd6b82fc989077aadd5e74522e1e7f28a9 Mon Sep 17 00:00:00 2001 From: MOHIT PRASANTH KALAHASTHI <22170424+mohitprasanth@users.noreply.github.com> Date: Thu, 6 Aug 2026 22:24:13 +0530 Subject: [PATCH 2/6] Add CustomLogger.runs_on_no_log so callbacks can opt out of the no-log skip --- litellm/integrations/custom_logger.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index a0c78674ac8..f86dadab178 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -59,6 +59,12 @@ _BASE64_INLINE_PATTERN: Final = re.compile( class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callback#callback-class + + # Set True on callbacks that must still run for `no-log` requests -- + # metering, billing and budget enforcement, as opposed to observability. + # `no-log` means "do not send this request to logging integrations"; it is + # not a way for a caller to opt out of being billed. + runs_on_no_log: bool = False # Class variables or attributes def __init__( self, From 18a4031dfc2b5dbf45ee833a08ae260bc517b8d9 Mon Sep 17 00:00:00 2001 From: MOHIT PRASANTH KALAHASTHI <22170424+mohitprasanth@users.noreply.github.com> Date: Thu, 6 Aug 2026 22:24:15 +0530 Subject: [PATCH 3/6] Mark _ProxyDBLogger runs_on_no_log: spend tracking is accounting, not logging --- litellm/proxy/hooks/proxy_track_cost_callback.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 3346f9d7e3b..51161fd7e54 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -50,6 +50,10 @@ _UNATTRIBUTED_TRACKABLE_CALL_TYPES: Final[frozenset[str]] = frozenset( class _ProxyDBLogger(CustomLogger): + # Spend tracking is accounting, not observability: it must still run when a + # caller sends `no-log`, or that caller silently stops being billed. + runs_on_no_log: bool = True + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): await self._PROXY_track_cost_callback(kwargs, response_obj, start_time, end_time) From 30caa9e4e1450b904c1ecdf6ba22bdc501452557 Mon Sep 17 00:00:00 2001 From: MOHIT PRASANTH KALAHASTHI <22170424+mohitprasanth@users.noreply.github.com> Date: Thu, 6 Aug 2026 22:24:18 +0530 Subject: [PATCH 4/6] Honour runs_on_no_log in should_run_callback instead of sniffing class names --- litellm/litellm_core_utils/litellm_logging.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 9475441e214..af0faf9200d 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1669,7 +1669,21 @@ class Logging(LiteLLMLoggingBaseClass): if litellm_params.get("no-log", False) is True: # proxy cost tracking cal backs should run - if not (isinstance(callback, CustomLogger) and "_PROXY_" in callback.__class__.__name__): + if not ( + + isinstance(callback, CustomLogger) + + and ( + + callback.runs_on_no_log + + # Retained for callbacks predating `runs_on_no_log`. + + or "_PROXY_" in callback.__class__.__name__ + + ) + + ): verbose_logger.debug("no-log request, skipping logging for %s event", event_hook) return False From 1ece379519eab5b6951a8e3d86343f5e5ccf4774 Mon Sep 17 00:00:00 2001 From: MOHIT PRASANTH KALAHASTHI <22170424+mohitprasanth@users.noreply.github.com> Date: Fri, 7 Aug 2026 08:44:44 +0530 Subject: [PATCH 5/6] test: align no-log spend test with repo conventions Drops the sys.path manipulation (neighbouring core_utils tests import directly), builds the Logging object the same way the existing test_litellm_logging_no_log_param does (real start_time), and moves the shared construction into a fixture. Also covers the string-callback case explicitly: callbacks referenced by name are not CustomLogger instances, so they stay skipped on no-log -- that is the path the pre-existing test exercises, and it must not change. --- .../test_no_log_spend_tracking.py | 120 +++++++----------- 1 file changed, 43 insertions(+), 77 deletions(-) diff --git a/tests/test_litellm/litellm_core_utils/test_no_log_spend_tracking.py b/tests/test_litellm/litellm_core_utils/test_no_log_spend_tracking.py index 2193d48416f..316049ed714 100644 --- a/tests/test_litellm/litellm_core_utils/test_no_log_spend_tracking.py +++ b/tests/test_litellm/litellm_core_utils/test_no_log_spend_tracking.py @@ -2,127 +2,93 @@ `no-log` means "do not send this request to logging integrations". It is not a way for a caller to opt out of being billed, and `Logging.should_run_callback` -has always intended to exempt the proxy's cost-tracking callback from it. The -exemption used to be a case-sensitive substring test for `"_PROXY_"` in the +has always intended to exempt the proxy's cost-tracking callback from it. + +The exemption used to be a case-sensitive substring test for `"_PROXY_"` in the callback's class name, which never matched the cost callback's actual class (`_ProxyDBLogger`), so spend tracking was silently skipped for every `no-log` request. Callbacks now declare the behaviour explicitly via `runs_on_no_log`. """ -import os -import sys +from datetime import datetime import pytest -sys.path.insert(0, os.path.abspath("../../..")) - import litellm from litellm.integrations.custom_logger import CustomLogger -from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger class _ObservabilityLogger(CustomLogger): - """A stand-in for langfuse/datadog/etc. -- must be skipped on no-log.""" + """Stand-in for langfuse/datadog/etc -- must be skipped on no-log.""" class _MeteringLogger(CustomLogger): - """A stand-in for a billing callback -- must still run on no-log.""" + """Stand-in for a billing callback -- must still run on no-log.""" runs_on_no_log = True -def _logging_obj() -> LiteLLMLogging: - return LiteLLMLogging( - model="gpt-4o", +@pytest.fixture +def logging_obj() -> Logging: + return Logging( + model="gpt-3.5-turbo", messages=[{"role": "user", "content": "hi"}], stream=False, call_type="acompletion", - start_time=None, - litellm_call_id="test-call-id", - function_id="test-function-id", + start_time=datetime.now(), + litellm_call_id="my-unique-call-id", + function_id="1234", ) -def test_cost_callback_still_runs_on_no_log(): - """The regression: _ProxyDBLogger must survive a no-log request.""" +def _should_run(logging_obj: Logging, callback, litellm_params: dict) -> bool: + return logging_obj.should_run_callback( + callback=callback, + litellm_params=litellm_params, + event_hook="success_handler", + ) + + +def test_cost_callback_still_runs_on_no_log(logging_obj): + """The regression: `_ProxyDBLogger` must survive a no-log request.""" assert _ProxyDBLogger.runs_on_no_log is True - assert ( - _logging_obj().should_run_callback( - callback=_ProxyDBLogger(), - litellm_params={"no-log": True}, - event_hook="async_log_success_event", - ) - is True - ) + assert _should_run(logging_obj, _ProxyDBLogger(), {"no-log": True}) is True -def test_observability_callback_is_skipped_on_no_log(): +def test_observability_callback_is_skipped_on_no_log(logging_obj): """no-log still does what it says for logging integrations.""" assert _ObservabilityLogger.runs_on_no_log is False - assert ( - _logging_obj().should_run_callback( - callback=_ObservabilityLogger(), - litellm_params={"no-log": True}, - event_hook="async_log_success_event", - ) - is False - ) + assert _should_run(logging_obj, _ObservabilityLogger(), {"no-log": True}) is False -def test_runs_on_no_log_is_opt_in_for_any_callback(): - """Any callback can declare itself as infrastructure, not just the proxy's.""" - assert ( - _logging_obj().should_run_callback( - callback=_MeteringLogger(), - litellm_params={"no-log": True}, - event_hook="async_log_success_event", - ) - is True - ) +def test_string_callbacks_are_still_skipped_on_no_log(logging_obj): + """Callbacks referenced by name are not CustomLogger instances.""" + assert _should_run(logging_obj, "langfuse", {"no-log": True}) is False + + +def test_runs_on_no_log_is_opt_in_for_any_callback(logging_obj): + """Any callback can declare itself infrastructure, not just the proxy's.""" + assert _should_run(logging_obj, _MeteringLogger(), {"no-log": True}) is True @pytest.mark.parametrize("litellm_params", [{}, {"no-log": False}]) -def test_everything_runs_when_no_log_is_absent_or_false(litellm_params): - for callback in (_ObservabilityLogger(), _ProxyDBLogger()): - assert ( - _logging_obj().should_run_callback( - callback=callback, - litellm_params=litellm_params, - event_hook="async_log_success_event", - ) - is True - ) +def test_everything_runs_when_no_log_absent_or_false(logging_obj, litellm_params): + assert _should_run(logging_obj, _ObservabilityLogger(), litellm_params) is True + assert _should_run(logging_obj, _ProxyDBLogger(), litellm_params) is True -def test_legacy_underscore_proxy_naming_still_exempt(): +def test_legacy_underscore_proxy_naming_still_exempt(logging_obj): """Callbacks named _PROXY_* keep working without setting the new flag.""" class _PROXY_LegacyHandler(CustomLogger): pass assert _PROXY_LegacyHandler.runs_on_no_log is False - assert ( - _logging_obj().should_run_callback( - callback=_PROXY_LegacyHandler(), - litellm_params={"no-log": True}, - event_hook="async_log_success_event", - ) - is True - ) + assert _should_run(logging_obj, _PROXY_LegacyHandler(), {"no-log": True}) is True -def test_global_disable_no_log_param_overrides_everything(): - original = litellm.global_disable_no_log_param - litellm.global_disable_no_log_param = True - try: - assert ( - _logging_obj().should_run_callback( - callback=_ObservabilityLogger(), - litellm_params={"no-log": True}, - event_hook="async_log_success_event", - ) - is True - ) - finally: - litellm.global_disable_no_log_param = original +def test_global_disable_no_log_param_overrides_everything(monkeypatch, logging_obj): + monkeypatch.setattr(litellm, "global_disable_no_log_param", True) + assert _should_run(logging_obj, _ObservabilityLogger(), {"no-log": True}) is True From bbb5a44a56293d6da22a54c85cb717c034a794f2 Mon Sep 17 00:00:00 2001 From: MOHIT PRASANTH KALAHASTHI <22170424+mohitprasanth@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:23:35 +0530 Subject: [PATCH 6/6] style: apply ruff format to the changed lines `ruff format --check` flagged custom_logger.py and litellm_logging.py: the inserted block carried a stray blank line after the class statement, and the reworked condition had blank lines between its clauses. Formatting only -- no behaviour change. Verified with the pinned ruff==0.15.3 and the repo's ruff.toml (line-length 120); `ruff check` passes on all three files. --- litellm/integrations/custom_logger.py | 2 +- litellm/litellm_core_utils/litellm_logging.py | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index f86dadab178..2cba74061de 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -59,12 +59,12 @@ _BASE64_INLINE_PATTERN: Final = re.compile( class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callback#callback-class - # Set True on callbacks that must still run for `no-log` requests -- # metering, billing and budget enforcement, as opposed to observability. # `no-log` means "do not send this request to logging integrations"; it is # not a way for a caller to opt out of being billed. runs_on_no_log: bool = False + # Class variables or attributes def __init__( self, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index af0faf9200d..4c5ef4b61f0 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1670,19 +1670,12 @@ class Logging(LiteLLMLoggingBaseClass): # proxy cost tracking cal backs should run if not ( - isinstance(callback, CustomLogger) - and ( - callback.runs_on_no_log - # Retained for callbacks predating `runs_on_no_log`. - or "_PROXY_" in callback.__class__.__name__ - ) - ): verbose_logger.debug("no-log request, skipping logging for %s event", event_hook) return False