test(logging): fake the Langfuse and Lunary vendor clients instead of our loggers

The sync failure tests swapped litellm's own Langfuse and Lunary loggers for MagicMocks. The Lunary test now runs the real LunaryLogger with the SDK's track_event recorded, and the Langfuse test moves next to the Langfuse export fixture, where a real LangFuseLogger exports to a recorded OTLP route.

tests/unit/integrations/langfuse/test_langfuse.py

@@ -2,6 +2,7 @@ import asyncio
 import json
 import threading
 from collections.abc import Iterator, Mapping
+from datetime import datetime
 from pathlib import Path
 from typing import Final
 from uuid import uuid4
@@ -24,6 +25,7 @@ from litellm.integrations.langfuse.langfuse_sdk import (
 )
 from litellm.litellm_core_utils import litellm_logging
 from litellm.litellm_core_utils.litellm_logging import DynamicLoggingCache as LegacyDynamicLoggingCache
+from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
 from litellm.litellm_core_utils.thread_pool_executor import MAX_THREADS, executor
 from litellm.llms.custom_httpx.http_handler import HTTPHandler
@@ -183,6 +185,45 @@ async def test_langfuse_logging_completion(langfuse_export: tuple[respx.Route, s
     _expected_generation(route, "completion.json", trace_id)
 
 
+@pytest.mark.asyncio
+async def test_langfuse_failure_with_redaction_on_logs_the_error_without_the_prompt(
+    langfuse_export: tuple[respx.Route, str], monkeypatch: pytest.MonkeyPatch
+) -> None:
+    route, _ = langfuse_export
+    prompt: Final = "my ssn is 123-45-6789"
+    monkeypatch.setattr(litellm, "turn_off_message_logging", True)
+    monkeypatch.setattr(litellm, "failure_callback", ["langfuse"])
+    logging_obj: Final = LitellmLogging(
+        model="gpt-4o",
+        messages=[{"role": "user", "content": prompt}],
+        stream=False,
+        call_type="completion",
+        start_time=datetime(2025, 1, 1),
+        litellm_call_id="failure-scrub",
+        function_id="failure-scrub",
+    )
+    logging_obj.model_call_details["messages"] = logging_obj.messages
+    error: Final = litellm.BadRequestError(
+        message=f"Invalid message passed in {{'role': 'user', 'content': '{prompt}'}}",
+        model="gpt-4o",
+        llm_provider="openai",
+    )
+
+    logging_obj.failure_handler(error, f"Traceback: {error}")
+    await _flush_langfuse()
+
+    spans: Final = _exported_spans(route)
+    status_messages: Final = [
+        attributes["langfuse.observation.status_message"]
+        for attributes in (TypeAdapter(dict[str, object]).validate_python(span["attributes"]) for span in spans)
+        if "langfuse.observation.status_message" in attributes
+    ]
+    assert status_messages == [
+        "litellm.BadRequestError: Invalid message passed in {'role': 'user', 'content': 'redacted-by-litellm'}"
+    ]
+    assert prompt not in json.dumps(spans)
+
+
 @pytest.mark.asyncio
 async def test_langfuse_logging_completion_with_tags(langfuse_export: tuple[respx.Route, str]) -> None:
     route, trace_id = langfuse_export

tests/unit/litellm_core_utils/test_litellm_logging.py

@@ -39,6 +39,7 @@ from litellm.constants import (
 )
 from litellm.cost_calculator import ocr_batch_cost
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.integrations.lunary import LunaryLogger
 from litellm.litellm_core_utils import litellm_logging
 from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
 from litellm.litellm_core_utils.litellm_logging import (
@@ -12149,34 +12150,30 @@ async def test_failure_with_non_mapping_headers_is_still_logged(monkeypatch, hea
     assert (SCRUB_PROMPT in capture.async_kwargs["traceback_exception"]) is not redact
 
 
-def test_sync_failure_hands_callbacks_and_langfuse_the_scrubbed_error(monkeypatch):
+def test_sync_failure_hands_callbacks_the_scrubbed_error(monkeypatch):
     capture: Final = _FailureCapture()
-    langfuse: Final = MagicMock()
     monkeypatch.setattr(litellm, "turn_off_message_logging", True)
-    monkeypatch.setattr(litellm, "failure_callback", [capture, "langfuse"])
-    monkeypatch.setattr(litellm_logging, "langFuseLogger", langfuse)
+    monkeypatch.setattr(litellm, "failure_callback", [capture])
 
     _prompt_failure_logging_obj().failure_handler(_prompt_echo_error(), f"Traceback: {SCRUB_PROMPT}")
 
     assert capture.sync_kwargs is not None
     assert SCRUB_PROMPT not in json.dumps(capture.sync_kwargs["standard_logging_object"], default=str)
-    status_message: Final = langfuse.log_event_on_langfuse.call_args.kwargs["status_message"]
-    assert "Invalid message passed in" in status_message
-    assert SCRUB_PROMPT not in status_message
 
 
 def test_sync_failure_hands_lunary_the_scrubbed_traceback_when_only_the_traceback_quotes_the_prompt(monkeypatch):
-    lunary: Final = MagicMock()
+    tracked_events: Final[list[dict[str, object]]] = []
+    lunary_logger: Final = LunaryLogger()
+    monkeypatch.setattr(lunary_logger.lunary_client, "track_event", lambda *_, **event: tracked_events.append(event))
     monkeypatch.setattr(litellm, "turn_off_message_logging", True)
     monkeypatch.setattr(litellm, "failure_callback", ["lunary"])
-    monkeypatch.setattr(litellm_logging, "lunaryLogger", lunary)
+    monkeypatch.setattr(litellm_logging, "lunaryLogger", lunary_logger)
     error: Final = litellm.APIConnectionError(message="Connection failed", model="gpt-4o", llm_provider="openai")
 
     _prompt_failure_logging_obj().failure_handler(error, f"ValueError: bad input {SCRUB_PROMPT}\n\n{error}")
 
-    logged_error: Final = lunary.log_event.call_args.kwargs["error"]
-    assert "ValueError: bad input redacted-by-litellm" in logged_error
-    assert SCRUB_PROMPT not in logged_error
+    logged_errors: Final = [event["error"] for event in tracked_events if event.get("error") is not None]
+    assert logged_errors == [{"stack": f"ValueError: bad input redacted-by-litellm\n\n{error}"}]
 
 
 def test_enterprise_alerting_loggers_resolves_real_classes():