diff --git a/tests/unit/integrations/langfuse/test_langfuse.py b/tests/unit/integrations/langfuse/test_langfuse.py index 7a374c1affe..738f2f6ecf7 100644 --- a/tests/unit/integrations/langfuse/test_langfuse.py +++ b/tests/unit/integrations/langfuse/test_langfuse.py @@ -2,6 +2,7 @@ import asyncio import json import threading from collections.abc import Iterator, Mapping +from datetime import datetime from pathlib import Path from typing import Final from uuid import uuid4 @@ -24,6 +25,7 @@ from litellm.integrations.langfuse.langfuse_sdk import ( ) from litellm.litellm_core_utils import litellm_logging from litellm.litellm_core_utils.litellm_logging import DynamicLoggingCache as LegacyDynamicLoggingCache +from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.thread_pool_executor import MAX_THREADS, executor from litellm.llms.custom_httpx.http_handler import HTTPHandler @@ -183,6 +185,45 @@ async def test_langfuse_logging_completion(langfuse_export: tuple[respx.Route, s _expected_generation(route, "completion.json", trace_id) +@pytest.mark.asyncio +async def test_langfuse_failure_with_redaction_on_logs_the_error_without_the_prompt( + langfuse_export: tuple[respx.Route, str], monkeypatch: pytest.MonkeyPatch +) -> None: + route, _ = langfuse_export + prompt: Final = "my ssn is 123-45-6789" + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + monkeypatch.setattr(litellm, "failure_callback", ["langfuse"]) + logging_obj: Final = LitellmLogging( + model="gpt-4o", + messages=[{"role": "user", "content": prompt}], + stream=False, + call_type="completion", + start_time=datetime(2025, 1, 1), + litellm_call_id="failure-scrub", + function_id="failure-scrub", + ) + logging_obj.model_call_details["messages"] = logging_obj.messages + error: Final = litellm.BadRequestError( + message=f"Invalid message passed in {{'role': 'user', 'content': '{prompt}'}}", + model="gpt-4o", + llm_provider="openai", + ) + + logging_obj.failure_handler(error, f"Traceback: {error}") + await _flush_langfuse() + + spans: Final = _exported_spans(route) + status_messages: Final = [ + attributes["langfuse.observation.status_message"] + for attributes in (TypeAdapter(dict[str, object]).validate_python(span["attributes"]) for span in spans) + if "langfuse.observation.status_message" in attributes + ] + assert status_messages == [ + "litellm.BadRequestError: Invalid message passed in {'role': 'user', 'content': 'redacted-by-litellm'}" + ] + assert prompt not in json.dumps(spans) + + @pytest.mark.asyncio async def test_langfuse_logging_completion_with_tags(langfuse_export: tuple[respx.Route, str]) -> None: route, trace_id = langfuse_export diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 62b0ce996b2..cbbf27cde6d 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -39,6 +39,7 @@ from litellm.constants import ( ) from litellm.cost_calculator import ocr_batch_cost from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.lunary import LunaryLogger from litellm.litellm_core_utils import litellm_logging from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging from litellm.litellm_core_utils.litellm_logging import ( @@ -12149,34 +12150,30 @@ async def test_failure_with_non_mapping_headers_is_still_logged(monkeypatch, hea assert (SCRUB_PROMPT in capture.async_kwargs["traceback_exception"]) is not redact -def test_sync_failure_hands_callbacks_and_langfuse_the_scrubbed_error(monkeypatch): +def test_sync_failure_hands_callbacks_the_scrubbed_error(monkeypatch): capture: Final = _FailureCapture() - langfuse: Final = MagicMock() monkeypatch.setattr(litellm, "turn_off_message_logging", True) - monkeypatch.setattr(litellm, "failure_callback", [capture, "langfuse"]) - monkeypatch.setattr(litellm_logging, "langFuseLogger", langfuse) + monkeypatch.setattr(litellm, "failure_callback", [capture]) _prompt_failure_logging_obj().failure_handler(_prompt_echo_error(), f"Traceback: {SCRUB_PROMPT}") assert capture.sync_kwargs is not None assert SCRUB_PROMPT not in json.dumps(capture.sync_kwargs["standard_logging_object"], default=str) - status_message: Final = langfuse.log_event_on_langfuse.call_args.kwargs["status_message"] - assert "Invalid message passed in" in status_message - assert SCRUB_PROMPT not in status_message def test_sync_failure_hands_lunary_the_scrubbed_traceback_when_only_the_traceback_quotes_the_prompt(monkeypatch): - lunary: Final = MagicMock() + tracked_events: Final[list[dict[str, object]]] = [] + lunary_logger: Final = LunaryLogger() + monkeypatch.setattr(lunary_logger.lunary_client, "track_event", lambda *_, **event: tracked_events.append(event)) monkeypatch.setattr(litellm, "turn_off_message_logging", True) monkeypatch.setattr(litellm, "failure_callback", ["lunary"]) - monkeypatch.setattr(litellm_logging, "lunaryLogger", lunary) + monkeypatch.setattr(litellm_logging, "lunaryLogger", lunary_logger) error: Final = litellm.APIConnectionError(message="Connection failed", model="gpt-4o", llm_provider="openai") _prompt_failure_logging_obj().failure_handler(error, f"ValueError: bad input {SCRUB_PROMPT}\n\n{error}") - logged_error: Final = lunary.log_event.call_args.kwargs["error"] - assert "ValueError: bad input redacted-by-litellm" in logged_error - assert SCRUB_PROMPT not in logged_error + logged_errors: Final = [event["error"] for event in tracked_events if event.get("error") is not None] + assert logged_errors == [{"stack": f"ValueError: bad input redacted-by-litellm\n\n{error}"}] def test_enterprise_alerting_loggers_resolves_real_classes():