From 9955230611f4488c8d5018dd506220c2f81c5e3e Mon Sep 17 00:00:00 2001 From: Peter Nancarrow Date: Wed, 14 Oct 2026 16:50:33 -0500 Subject: [PATCH] feat(logging): expose per-callback redaction for generic API loggers --- .../logging_callback_manager.py | 15 ++- .../test_logging_callback_manager.py | 116 +++++++++++++++++- 2 files changed, 126 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py index 5eaa4b467fe..9f57cc627c7 100644 --- a/litellm/litellm_core_utils/logging_callback_manager.py +++ b/litellm/litellm_core_utils/logging_callback_manager.py @@ -175,11 +175,17 @@ class LoggingCallbackManager: callback_settings: custom_callback_name: callback_type: generic_api + turn_off_message_logging: true endpoint: https://webhook-test.com/30343bc33591bc5e6dc44217ceae3e0a headers: Authorization: Bearer $LITELLM_MASTER_KEY + ramp: + turn_off_message_logging: true """ callback_config: Final = litellm.callback_settings.get(callback) + turn_off_message_logging: Final = ( + isinstance(callback_config, dict) and callback_config.get("turn_off_message_logging") is True + ) # Check if callback is in callback_settings with callback_type: generic_api if isinstance(callback_config, dict) and callback_config.get("callback_type") == "generic_api": @@ -212,6 +218,7 @@ class LoggingCallbackManager: and cached_logger.max_retries == max_retries and cached_logger.retry_delay == retry_delay and cached_logger.timeout == timeout + and cached_logger.turn_off_message_logging == turn_off_message_logging ): return cached_logger @@ -223,6 +230,7 @@ class LoggingCallbackManager: max_retries=max_retries, retry_delay=retry_delay, timeout=timeout, + turn_off_message_logging=turn_off_message_logging, ) _generic_api_logger_cache[callback] = new_logger return new_logger @@ -235,12 +243,15 @@ class LoggingCallbackManager: if is_callback_compatible(callback): # Check if we already have a cached logger for this callback cached_logger = _generic_api_logger_cache.get(callback) - if isinstance(cached_logger, GenericAPILogger): + if ( + isinstance(cached_logger, GenericAPILogger) + and cached_logger.turn_off_message_logging == turn_off_message_logging + ): return cached_logger # Create new GenericAPILogger with callback_name parameter # This will load config from generic_api_compatible_callbacks.json - new_logger = GenericAPILogger(callback_name=callback) + new_logger = GenericAPILogger(callback_name=callback, turn_off_message_logging=turn_off_message_logging) _generic_api_logger_cache[callback] = new_logger return new_logger diff --git a/tests/unit/litellm_core_utils/test_logging_callback_manager.py b/tests/unit/litellm_core_utils/test_logging_callback_manager.py index f8260702068..84b02b68b60 100644 --- a/tests/unit/litellm_core_utils/test_logging_callback_manager.py +++ b/tests/unit/litellm_core_utils/test_logging_callback_manager.py @@ -1,13 +1,21 @@ import os +from copy import deepcopy +from datetime import datetime +from typing import Final from unittest.mock import AsyncMock, patch +import httpx import pytest +import respx +from pydantic import TypeAdapter import litellm from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.generic_api.generic_api_callback import GenericAPILogger from litellm.integrations.langfuse.langfuse_prompt_management import LangfusePromptManagement from litellm.integrations.opentelemetry import OpenTelemetry from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager +from litellm.litellm_core_utils import logging_callback_manager as callback_manager_module @pytest.fixture @@ -305,9 +313,7 @@ async def test_generic_api_callback_settings_retry_config(): } try: - result = LoggingCallbackManager.add_custom_callback_generic_api_str( - callback_name - ) + result = LoggingCallbackManager.add_custom_callback_generic_api_str(callback_name) assert isinstance(result, GenericAPILogger) assert result.endpoint == "https://example.com/api/logs" @@ -342,3 +348,107 @@ def test_duplicate_multiple_loggers_test(): assert langfuse_count == 1, "Should have exactly one LangfusePromptManagement instance" assert otel_count == 1, "Should have exactly one OpenTelemetry instance" + + +@pytest.fixture +def generic_api_redaction_state(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(callback_manager_module, "_generic_api_logger_cache", {}) + monkeypatch.setattr(litellm, "callback_settings", {}) + monkeypatch.setattr(litellm, "turn_off_message_logging", False) + monkeypatch.setattr(litellm, "generic_api_use_v1", False) + monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", None) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("RAMP_API_KEY", "test-ramp-key") + + +async def _assert_generic_api_export(logger: GenericAPILogger, redact: bool) -> None: + payload: Final = { + "messages": [{"role": "user", "content": "private-prompt-canary"}], + "response": "private-response-canary", + "model": "accounting-model", + "prompt_tokens": 20, + "completion_tokens": 5, + "total_tokens": 25, + "response_cost": 0.03, + } + original: Final = deepcopy(payload) + full_logger: Final = CustomLogger() + full_details: Final = {"standard_logging_object": payload} + ramp_details: Final = logger.redact_standard_logging_payload_from_model_call_details(full_details) + now: Final = datetime(2026, 1, 1) + with respx.mock as http: + destination: Final = http.post(logger.endpoint).mock(return_value=httpx.Response(200)) + await logger.async_log_success_event(ramp_details, None, now, now) + await logger.flush_queue() + assert destination.call_count == 1 + request_body: Final = destination.calls.last.request.content + exported: Final = TypeAdapter(list[dict[str, object]]).validate_json(request_body) + + assert len(exported) == 1 + assert exported[0]["model"] == payload["model"] + assert exported[0]["prompt_tokens"] == payload["prompt_tokens"] + assert exported[0]["completion_tokens"] == payload["completion_tokens"] + assert exported[0]["total_tokens"] == payload["total_tokens"] + assert exported[0]["response_cost"] == payload["response_cost"] + assert (b"private-prompt-canary" not in request_body) is redact + assert (b"private-response-canary" not in request_body) is redact + assert full_logger.redact_standard_logging_payload_from_model_call_details(full_details) == full_details + assert payload == original + + +@pytest.mark.asyncio +@pytest.mark.parametrize("callback_name", ["ramp", "accounting_sink"]) +@pytest.mark.parametrize("redaction", [None, False, True]) +async def test_generic_api_redaction_keeps_original_logging_payload( + callback_name: str, + redaction: bool | None, + generic_api_redaction_state: None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings: Final = ( + { + "callback_type": "generic_api", + "endpoint": "https://example.com/logs", + "headers": {"Content-Type": "application/json"}, + "log_format": "json_array", + } + if callback_name == "accounting_sink" + else {} + ) + redaction_setting: Final = {} if redaction is None else {"turn_off_message_logging": redaction} + monkeypatch.setattr(litellm, "callback_settings", {callback_name: {**settings, **redaction_setting}}) + logger: Final = LoggingCallbackManager.add_custom_callback_generic_api_str(callback_name) + assert isinstance(logger, GenericAPILogger) + await _assert_generic_api_export(logger, redact=redaction is True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("callback_name", ["ramp", "accounting_sink"]) +@pytest.mark.parametrize("before,after", [(False, True), (True, False), (True, True)]) +async def test_generic_api_cached_logger_tracks_redaction_changes( + callback_name: str, + before: bool, + after: bool, + generic_api_redaction_state: None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings: Final = ( + { + "callback_type": "generic_api", + "endpoint": "https://example.com/logs", + "headers": {"Content-Type": "application/json"}, + "log_format": "json_array", + } + if callback_name == "accounting_sink" + else {} + ) + monkeypatch.setattr(litellm, "callback_settings", {callback_name: {**settings, "turn_off_message_logging": before}}) + initial: Final = LoggingCallbackManager.add_custom_callback_generic_api_str(callback_name) + assert isinstance(initial, GenericAPILogger) + await _assert_generic_api_export(initial, redact=before) + + monkeypatch.setattr(litellm, "callback_settings", {callback_name: {**settings, "turn_off_message_logging": after}}) + updated: Final = LoggingCallbackManager.add_custom_callback_generic_api_str(callback_name) + assert isinstance(updated, GenericAPILogger) + await _assert_generic_api_export(updated, redact=after) + assert (updated is initial) is (before == after)