mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
test: align no-log spend test with repo conventions
Drops the sys.path manipulation (neighbouring core_utils tests import directly), builds the Logging object the same way the existing test_litellm_logging_no_log_param does (real start_time), and moves the shared construction into a fixture. Also covers the string-callback case explicitly: callbacks referenced by name are not CustomLogger instances, so they stay skipped on no-log -- that is the path the pre-existing test exercises, and it must not change.
This commit is contained in:
parent
30caa9e4e1
commit
1ece379519
1 changed files with 43 additions and 77 deletions
|
|
@ -2,127 +2,93 @@
|
|||
|
||||
`no-log` means "do not send this request to logging integrations". It is not a
|
||||
way for a caller to opt out of being billed, and `Logging.should_run_callback`
|
||||
has always intended to exempt the proxy's cost-tracking callback from it. The
|
||||
exemption used to be a case-sensitive substring test for `"_PROXY_"` in the
|
||||
has always intended to exempt the proxy's cost-tracking callback from it.
|
||||
|
||||
The exemption used to be a case-sensitive substring test for `"_PROXY_"` in the
|
||||
callback's class name, which never matched the cost callback's actual class
|
||||
(`_ProxyDBLogger`), so spend tracking was silently skipped for every `no-log`
|
||||
request. Callbacks now declare the behaviour explicitly via `runs_on_no_log`.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.abspath("../../.."))
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger
|
||||
|
||||
|
||||
class _ObservabilityLogger(CustomLogger):
|
||||
"""A stand-in for langfuse/datadog/etc. -- must be skipped on no-log."""
|
||||
"""Stand-in for langfuse/datadog/etc -- must be skipped on no-log."""
|
||||
|
||||
|
||||
class _MeteringLogger(CustomLogger):
|
||||
"""A stand-in for a billing callback -- must still run on no-log."""
|
||||
"""Stand-in for a billing callback -- must still run on no-log."""
|
||||
|
||||
runs_on_no_log = True
|
||||
|
||||
|
||||
def _logging_obj() -> LiteLLMLogging:
|
||||
return LiteLLMLogging(
|
||||
model="gpt-4o",
|
||||
@pytest.fixture
|
||||
def logging_obj() -> Logging:
|
||||
return Logging(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=False,
|
||||
call_type="acompletion",
|
||||
start_time=None,
|
||||
litellm_call_id="test-call-id",
|
||||
function_id="test-function-id",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="my-unique-call-id",
|
||||
function_id="1234",
|
||||
)
|
||||
|
||||
|
||||
def test_cost_callback_still_runs_on_no_log():
|
||||
"""The regression: _ProxyDBLogger must survive a no-log request."""
|
||||
def _should_run(logging_obj: Logging, callback, litellm_params: dict) -> bool:
|
||||
return logging_obj.should_run_callback(
|
||||
callback=callback,
|
||||
litellm_params=litellm_params,
|
||||
event_hook="success_handler",
|
||||
)
|
||||
|
||||
|
||||
def test_cost_callback_still_runs_on_no_log(logging_obj):
|
||||
"""The regression: `_ProxyDBLogger` must survive a no-log request."""
|
||||
assert _ProxyDBLogger.runs_on_no_log is True
|
||||
assert (
|
||||
_logging_obj().should_run_callback(
|
||||
callback=_ProxyDBLogger(),
|
||||
litellm_params={"no-log": True},
|
||||
event_hook="async_log_success_event",
|
||||
)
|
||||
is True
|
||||
)
|
||||
assert _should_run(logging_obj, _ProxyDBLogger(), {"no-log": True}) is True
|
||||
|
||||
|
||||
def test_observability_callback_is_skipped_on_no_log():
|
||||
def test_observability_callback_is_skipped_on_no_log(logging_obj):
|
||||
"""no-log still does what it says for logging integrations."""
|
||||
assert _ObservabilityLogger.runs_on_no_log is False
|
||||
assert (
|
||||
_logging_obj().should_run_callback(
|
||||
callback=_ObservabilityLogger(),
|
||||
litellm_params={"no-log": True},
|
||||
event_hook="async_log_success_event",
|
||||
)
|
||||
is False
|
||||
)
|
||||
assert _should_run(logging_obj, _ObservabilityLogger(), {"no-log": True}) is False
|
||||
|
||||
|
||||
def test_runs_on_no_log_is_opt_in_for_any_callback():
|
||||
"""Any callback can declare itself as infrastructure, not just the proxy's."""
|
||||
assert (
|
||||
_logging_obj().should_run_callback(
|
||||
callback=_MeteringLogger(),
|
||||
litellm_params={"no-log": True},
|
||||
event_hook="async_log_success_event",
|
||||
)
|
||||
is True
|
||||
)
|
||||
def test_string_callbacks_are_still_skipped_on_no_log(logging_obj):
|
||||
"""Callbacks referenced by name are not CustomLogger instances."""
|
||||
assert _should_run(logging_obj, "langfuse", {"no-log": True}) is False
|
||||
|
||||
|
||||
def test_runs_on_no_log_is_opt_in_for_any_callback(logging_obj):
|
||||
"""Any callback can declare itself infrastructure, not just the proxy's."""
|
||||
assert _should_run(logging_obj, _MeteringLogger(), {"no-log": True}) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("litellm_params", [{}, {"no-log": False}])
|
||||
def test_everything_runs_when_no_log_is_absent_or_false(litellm_params):
|
||||
for callback in (_ObservabilityLogger(), _ProxyDBLogger()):
|
||||
assert (
|
||||
_logging_obj().should_run_callback(
|
||||
callback=callback,
|
||||
litellm_params=litellm_params,
|
||||
event_hook="async_log_success_event",
|
||||
)
|
||||
is True
|
||||
)
|
||||
def test_everything_runs_when_no_log_absent_or_false(logging_obj, litellm_params):
|
||||
assert _should_run(logging_obj, _ObservabilityLogger(), litellm_params) is True
|
||||
assert _should_run(logging_obj, _ProxyDBLogger(), litellm_params) is True
|
||||
|
||||
|
||||
def test_legacy_underscore_proxy_naming_still_exempt():
|
||||
def test_legacy_underscore_proxy_naming_still_exempt(logging_obj):
|
||||
"""Callbacks named _PROXY_* keep working without setting the new flag."""
|
||||
|
||||
class _PROXY_LegacyHandler(CustomLogger):
|
||||
pass
|
||||
|
||||
assert _PROXY_LegacyHandler.runs_on_no_log is False
|
||||
assert (
|
||||
_logging_obj().should_run_callback(
|
||||
callback=_PROXY_LegacyHandler(),
|
||||
litellm_params={"no-log": True},
|
||||
event_hook="async_log_success_event",
|
||||
)
|
||||
is True
|
||||
)
|
||||
assert _should_run(logging_obj, _PROXY_LegacyHandler(), {"no-log": True}) is True
|
||||
|
||||
|
||||
def test_global_disable_no_log_param_overrides_everything():
|
||||
original = litellm.global_disable_no_log_param
|
||||
litellm.global_disable_no_log_param = True
|
||||
try:
|
||||
assert (
|
||||
_logging_obj().should_run_callback(
|
||||
callback=_ObservabilityLogger(),
|
||||
litellm_params={"no-log": True},
|
||||
event_hook="async_log_success_event",
|
||||
)
|
||||
is True
|
||||
)
|
||||
finally:
|
||||
litellm.global_disable_no_log_param = original
|
||||
def test_global_disable_no_log_param_overrides_everything(monkeypatch, logging_obj):
|
||||
monkeypatch.setattr(litellm, "global_disable_no_log_param", True)
|
||||
assert _should_run(logging_obj, _ObservabilityLogger(), {"no-log": True}) is True
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue