diff --git a/litellm/integrations/mlflow.py b/litellm/integrations/mlflow.py index 8731e96440f..f58a600ae56 100644 --- a/litellm/integrations/mlflow.py +++ b/litellm/integrations/mlflow.py @@ -1,9 +1,11 @@ import json import threading +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.redact_messages import scrubbed_exception_record if TYPE_CHECKING: from litellm.types.utils import StandardLoggingPayload @@ -74,7 +76,7 @@ class MlflowLogger(CustomLogger): async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): self._handle_failure(kwargs, response_obj, start_time, end_time) - def _handle_failure(self, kwargs, response_obj, start_time, end_time): + def _handle_failure(self, kwargs: Mapping[str, object], response_obj, start_time, end_time): """ Log the failure event as an MLflow span. Note that this method is called *synchronously* unlike the success handler. @@ -88,7 +90,19 @@ class MlflowLogger(CustomLogger): # Record exception info as event if exception := kwargs.get("exception"): - span.add_event(SpanEvent.from_exception(exception)) + scrubbed: Final = scrubbed_exception_record(kwargs) + span.add_event( + SpanEvent.from_exception(exception) + if scrubbed is None + else SpanEvent( + name="exception", + attributes={ + "exception.message": scrubbed[0], + "exception.type": exception.__class__.__name__, + "exception.stacktrace": scrubbed[1], + }, + ) + ) self._extract_and_set_chat_attributes(span, kwargs, response_obj) self._end_span_or_trace( diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index a9bd93e6420..fecd40dfc9b 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -36,6 +36,7 @@ from litellm.integrations.otel.model.semconv import LiteLLM, Metric from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls from litellm.integrations.otel.routing import routing_decision_attributes from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params +from litellm.litellm_core_utils.redact_messages import scrubbed_exception_record from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.secret_redaction import redact_string from litellm.litellm_core_utils.service_tier_utils import ( @@ -2271,9 +2272,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): # Get the exception object if available exception: Final = kwargs.get("exception") + scrubbed: Final = scrubbed_exception_record(kwargs) - # Record the exception using OTEL's standard method - if exception is not None: + if exception is not None and scrubbed is not None: + span.record_exception( + exception, attributes={"exception.message": scrubbed[0], "exception.stacktrace": scrubbed[1]} + ) + elif exception is not None: span.record_exception(exception) # Get StandardLoggingPayload for structured error information diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index ce57a9bd4a5..57571b217f8 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -143,10 +143,15 @@ from litellm.litellm_core_utils.logging_utils import ( from litellm.litellm_core_utils.model_param_helper import ModelParamHelper from litellm.litellm_core_utils.ptu_pricing import is_spilled_over_ptu_request from litellm.litellm_core_utils.redact_messages import ( + failure_request_texts, redact_message_input_output_from_custom_logger, redact_message_input_output_from_logging, redacted_model_call_details_for_custom_logger, redacted_response_obj_for_custom_logger, + scrub_request_text, + scrubbed_error_fields, + scrubbed_exception_text, + should_redact_failure_logging, should_redact_message_logging, ) from litellm.litellm_core_utils.served_output_texts import ( @@ -4552,8 +4557,9 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["log_event_type"] = "failed_api_call" self.model_call_details["exception"] = exception + traceback_text: Final = redact_string(traceback_exception) if isinstance(traceback_exception, str) else None self.model_call_details["traceback_exception"] = ( - redact_string(traceback_exception) if isinstance(traceback_exception, str) else traceback_exception + traceback_exception if traceback_text is None else traceback_text ) self.model_call_details["end_time"] = end_time self.model_call_details.setdefault("original_response", None) @@ -4571,9 +4577,15 @@ class Logging(LiteLLMLoggingBaseClass): metadata: Final = self.model_call_details["litellm_params"].get("metadata", {}) or {} metadata.update(exception.headers) + # Building the payload redacts the messages in place, so read the request text first + redact: Final = should_redact_failure_logging(self.model_call_details) + request_texts: Final[frozenset[str]] = ( + failure_request_texts(self.model_call_details, getattr(self, "messages", None)) if redact else frozenset() + ) + ## STANDARDIZED LOGGING PAYLOAD - self.model_call_details["standard_logging_object"] = get_standard_logging_object_payload( + payload: Final = get_standard_logging_object_payload( kwargs=self.model_call_details, init_response_obj={}, start_time=start_time, @@ -4584,6 +4596,11 @@ class Logging(LiteLLMLoggingBaseClass): original_exception=exception, standard_built_in_tools_params=self.standard_built_in_tools_params, ) + self.model_call_details["standard_logging_object"] = ( + {**payload, **scrubbed_error_fields(payload, request_texts)} if redact and payload is not None else payload + ) + if redact and traceback_text is not None: + self.model_call_details["traceback_exception"] = scrub_request_text(traceback_text, request_texts) return start_time, end_time async def special_failure_handlers(self, exception: Exception): @@ -4679,6 +4696,9 @@ class Logging(LiteLLMLoggingBaseClass): ) ) self.mark_logging_complete(event_type="sync_failure") + scrubbed_exception: Final = scrubbed_exception_text(self.model_call_details) + logged_error: Final = str(exception) if scrubbed_exception is None else scrubbed_exception[0] + logged_traceback: Final = traceback_exception if scrubbed_exception is None else scrubbed_exception[1] for callback in callbacks: try: should_run = self.should_run_callback( @@ -4704,7 +4724,7 @@ class Logging(LiteLLMLoggingBaseClass): user_id=self._model_call_details_view.get("user", "default"), model=model, input=input, - error=traceback_exception, + error=logged_traceback, run_id=self.litellm_call_id, start_time=start_time, end_time=end_time, @@ -4771,7 +4791,7 @@ class Logging(LiteLLMLoggingBaseClass): end_time=end_time, response_obj=None, user_id=kwargs.get("user", None), - status_message=str(exception), + status_message=logged_error, level="ERROR", kwargs=kwargs, ) @@ -4790,7 +4810,7 @@ class Logging(LiteLLMLoggingBaseClass): response_obj=None, user_id=self.model_call_details.get("user", None), print_verbose=print_verbose, - status_message=str(exception), + status_message=logged_error, level="ERROR", kwargs=self.model_call_details, ) diff --git a/tests/unit/integrations/langfuse/test_langfuse.py b/tests/unit/integrations/langfuse/test_langfuse.py index 7a374c1affe..a1a5f7141ed 100644 --- a/tests/unit/integrations/langfuse/test_langfuse.py +++ b/tests/unit/integrations/langfuse/test_langfuse.py @@ -2,6 +2,7 @@ import asyncio import json import threading from collections.abc import Iterator, Mapping +from datetime import datetime from pathlib import Path from typing import Final from uuid import uuid4 @@ -24,6 +25,7 @@ from litellm.integrations.langfuse.langfuse_sdk import ( ) from litellm.litellm_core_utils import litellm_logging from litellm.litellm_core_utils.litellm_logging import DynamicLoggingCache as LegacyDynamicLoggingCache +from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.thread_pool_executor import MAX_THREADS, executor from litellm.llms.custom_httpx.http_handler import HTTPHandler @@ -183,6 +185,45 @@ async def test_langfuse_logging_completion(langfuse_export: tuple[respx.Route, s _expected_generation(route, "completion.json", trace_id) +@pytest.mark.asyncio +async def test_langfuse_failure_with_redaction_on_logs_the_error_without_the_prompt( + langfuse_export: tuple[respx.Route, str], monkeypatch: pytest.MonkeyPatch +) -> None: + route: Final = langfuse_export[0] + prompt: Final = "my ssn is 123-45-6789" + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + monkeypatch.setattr(litellm, "failure_callback", ["langfuse"]) + logging_obj: Final = LitellmLogging( + model="gpt-4o", + messages=[{"role": "user", "content": prompt}], + stream=False, + call_type="completion", + start_time=datetime(2025, 1, 1), + litellm_call_id="failure-scrub", + function_id="failure-scrub", + ) + logging_obj.model_call_details["messages"] = logging_obj.messages + error: Final = litellm.BadRequestError( + message=f"Invalid message passed in {{'role': 'user', 'content': '{prompt}'}}", + model="gpt-4o", + llm_provider="openai", + ) + + logging_obj.failure_handler(error, f"Traceback: {error}") + await _flush_langfuse() + + spans: Final = _exported_spans(route) + status_messages: Final = [ + attributes["langfuse.observation.status_message"] + for attributes in (TypeAdapter(dict[str, object]).validate_python(span["attributes"]) for span in spans) + if "langfuse.observation.status_message" in attributes + ] + assert status_messages == [ + "litellm.BadRequestError: Invalid message passed in {'role': 'user', 'content': 'redacted-by-litellm'}" + ] + assert prompt not in json.dumps(spans) + + @pytest.mark.asyncio async def test_langfuse_logging_completion_with_tags(langfuse_export: tuple[respx.Route, str]) -> None: route, trace_id = langfuse_export diff --git a/tests/unit/integrations/test_mlflow.py b/tests/unit/integrations/test_mlflow.py index 058d0ae885d..0d86ef445a8 100644 --- a/tests/unit/integrations/test_mlflow.py +++ b/tests/unit/integrations/test_mlflow.py @@ -1,7 +1,9 @@ import asyncio import json import threading +from collections.abc import Mapping from datetime import datetime +from typing import Final from unittest.mock import MagicMock, patch # Adds the grandparent directory to sys.path to allow importing project modules @@ -268,3 +270,95 @@ def test_mlflow_end_span_or_trace_works_with_mlflow_2x_client(): span=child_span, outputs="out", end_time_ns=1, status="OK" ) assert client.ended_spans == [("req-2", "span-2")] + + +class _ExceptionSpanEvent: + def __init__(self, name: str, attributes: Mapping[str, str]) -> None: + self.name: Final = name + self.attributes: Final = attributes + + @classmethod + def from_exception(cls, exception: BaseException) -> "_ExceptionSpanEvent": + return cls( + "exception", + { + "exception.message": str(exception), + "exception.type": type(exception).__name__, + "exception.stacktrace": "raw", + }, + ) + + +@pytest.mark.parametrize("scrubbed", [True, False], ids=["scrubbed", "not_scrubbed"]) +def test_mlflow_failure_event_records_the_error_text_the_payload_logged(scrubbed: bool) -> None: + secret: Final = "my ssn is 123-45-6789" + modules: Final = _mock_mlflow_modules() + modules["mlflow"].get_current_active_span.return_value = None + modules["mlflow.entities"].SpanEvent = _ExceptionSpanEvent + with patch.dict("sys.modules", modules): + from litellm.integrations.mlflow import MlflowLogger + + mlflow_logger: Final = MlflowLogger() + span: Final = MagicMock(parent_id=None, request_id="req-1") + mlflow_logger._client = MagicMock(**{"start_trace.return_value": span}) + error_message: Final = "bad input: redacted-by-litellm" if scrubbed else f"bad input: {secret}" + + mlflow_logger._handle_failure( + kwargs={ + "litellm_call_id": "x", + "messages": [{"role": "user", "content": "hi"}], + "exception": ValueError(f"bad input: {secret}"), + "standard_logging_object": { + "request_tags": [], + "error_information": {"error_message": error_message, "traceback": "frames"}, + }, + }, + response_obj=None, + start_time=datetime(2025, 1, 1), + end_time=datetime(2025, 1, 1), + ) + + event: Final = span.add_event.call_args.args[0] + mlflow_logger._client.end_trace.assert_called_once() + assert event.attributes["exception.type"] == "ValueError" + assert event.attributes["exception.message"] == error_message + assert event.attributes["exception.stacktrace"] == ("frames" if scrubbed else "raw") + + +def _raise_provider_error(message: str) -> None: + raise ValueError(message) + + +def test_mlflow_failure_event_records_the_exceptions_own_frames_when_no_traceback_was_logged() -> None: + secret: Final = "my ssn is 123-45-6789" + modules: Final = _mock_mlflow_modules() + modules["mlflow"].get_current_active_span.return_value = None + modules["mlflow.entities"].SpanEvent = _ExceptionSpanEvent + with pytest.raises(ValueError, match="bad input") as raised: + _raise_provider_error(f"bad input: {secret}") + with patch.dict("sys.modules", modules): + from litellm.integrations.mlflow import MlflowLogger + + mlflow_logger: Final = MlflowLogger() + span: Final = MagicMock(parent_id=None, request_id="req-1") + mlflow_logger._client = MagicMock(**{"start_trace.return_value": span}) + + mlflow_logger._handle_failure( + kwargs={ + "litellm_call_id": "x", + "messages": [{"role": "user", "content": "hi"}], + "exception": raised.value, + "standard_logging_object": { + "request_tags": [], + "error_information": {"error_message": "bad input: redacted-by-litellm", "traceback": ""}, + }, + }, + response_obj=None, + start_time=datetime(2025, 1, 1), + end_time=datetime(2025, 1, 1), + ) + + stacktrace: Final = span.add_event.call_args.args[0].attributes["exception.stacktrace"] + assert secret not in stacktrace + assert "in _raise_provider_error" in stacktrace + assert stacktrace.endswith("ValueError: bad input: redacted-by-litellm\n") diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index 5f749e951b1..2e8be029e9d 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -7636,3 +7636,76 @@ class TestOpentelemetryUnitTests: # Assert: error.message should be set from error_str using ErrorAttributes constant mock_span.set_attribute.assert_called_with(ErrorAttributes.ERROR_MESSAGE, "Fallback error message") + + +def test_record_exception_on_span_records_the_scrubbed_error_text_with_the_same_event_shape() -> None: + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + from litellm.integrations.opentelemetry import OpenTelemetry + + exporter: Final = InMemorySpanExporter() + provider: Final = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + secret: Final = "my ssn is 123-45-6789" + exception: Final = ValueError(f"bad input: {secret}") + + with provider.get_tracer(__name__).start_as_current_span("failure") as span: + OpenTelemetry()._record_exception_on_span( + span=span, + kwargs={ + "exception": exception, + "standard_logging_object": { + "error_information": { + "error_class": "ValueError", + "error_message": "bad input: redacted-by-litellm", + "traceback": "frames", + }, + }, + }, + ) + + (event,) = exporter.get_finished_spans()[0].events + assert secret not in str(dict(event.attributes)) + assert event.attributes["exception.message"] == "bad input: redacted-by-litellm" + assert event.attributes["exception.stacktrace"] == "frames" + assert event.attributes["exception.type"] == "ValueError" + assert event.attributes["exception.escaped"] == "False" + + +def _raise_provider_error(message: str) -> None: + raise ValueError(message) + + +def test_record_exception_on_span_records_the_exceptions_own_frames_when_no_traceback_was_logged() -> None: + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + from litellm.integrations.opentelemetry import OpenTelemetry + + exporter: Final = InMemorySpanExporter() + provider: Final = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + secret: Final = "my ssn is 123-45-6789" + with pytest.raises(ValueError, match="bad input") as raised: + _raise_provider_error(f"bad input: {secret}") + + with provider.get_tracer(__name__).start_as_current_span("failure") as span: + OpenTelemetry()._record_exception_on_span( + span=span, + kwargs={ + "exception": raised.value, + "standard_logging_object": { + "error_information": {"error_message": "bad input: redacted-by-litellm", "traceback": ""} + }, + }, + ) + + (event,) = exporter.get_finished_spans()[0].events + stacktrace: Final = str(event.attributes["exception.stacktrace"]) + assert secret not in str(dict(event.attributes)) + assert stacktrace.startswith("Traceback (most recent call last):\n") + assert "in _raise_provider_error" in stacktrace + assert stacktrace.endswith("ValueError: bad input: redacted-by-litellm\n") diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index e8ea0c04f6f..e0c574830d9 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -39,6 +39,7 @@ from litellm.constants import ( ) from litellm.cost_calculator import ocr_batch_cost from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.lunary import LunaryLogger from litellm.litellm_core_utils import litellm_logging from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging from litellm.litellm_core_utils.litellm_logging import ( @@ -12049,6 +12050,152 @@ def test_uncached_price_keeps_an_earlier_price_failure(pr44187_priced_model: str assert logging_obj.model_call_details["response_cost_failure_debug_information"] == earlier_failure +SCRUB_PROMPT: Final = "my ssn is 123-45-6789" + + +class _FailureCapture(CustomLogger): + def __init__(self, **kwargs: bool) -> None: + super().__init__(**kwargs) + self.sync_kwargs: dict[str, object] | None = None + self.async_kwargs: dict[str, object] | None = None + + def log_failure_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + self.sync_kwargs = kwargs + + async def async_log_failure_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + self.async_kwargs = kwargs + + +def _prompt_failure_logging_obj(litellm_params: Mapping[str, object] | None = None) -> LitellmLogging: + logging_obj: Final = LitellmLogging( + model="gpt-4o", + messages=[{"role": "user", "content": SCRUB_PROMPT}], + stream=False, + call_type="completion", + start_time=datetime.datetime(2025, 1, 1), + litellm_call_id="failure-scrub", + function_id="failure-scrub", + ) + logging_obj.model_call_details["messages"] = logging_obj.messages + if litellm_params is not None: + logging_obj.model_call_details["litellm_params"] = dict(litellm_params) + return logging_obj + + +def _prompt_echo_error() -> litellm.BadRequestError: + return litellm.BadRequestError( + message=f"Invalid message passed in {{'role': 'user', 'content': '{SCRUB_PROMPT}'}}", + model="gpt-4o", + llm_provider="openai", + ) + + +async def _async_failure_payload( + monkeypatch: pytest.MonkeyPatch, *, redact: bool, error: Exception, capture: _FailureCapture | None = None +) -> dict[str, object]: + logger: Final = capture or _FailureCapture() + monkeypatch.setattr(litellm, "turn_off_message_logging", redact) + monkeypatch.setattr(litellm, "_async_failure_callback", [logger]) + await _prompt_failure_logging_obj().async_failure_handler(error, f"Traceback: {error}") + assert logger.async_kwargs is not None + return logger.async_kwargs + + +@pytest.mark.asyncio +async def test_failure_with_redaction_on_scrubs_the_prompt_out_of_the_error_and_keeps_the_rest( + monkeypatch: pytest.MonkeyPatch, +) -> None: + kwargs: Final = await _async_failure_payload(monkeypatch, redact=True, error=_prompt_echo_error()) + + payload: Final = kwargs["standard_logging_object"] + assert SCRUB_PROMPT not in json.dumps(payload, default=str) + assert SCRUB_PROMPT not in kwargs["traceback_exception"] + assert "Invalid message passed in" in payload["error_str"] + assert "'content': 'redacted-by-litellm'" in payload["error_information"]["error_message"] + assert payload["error_information"]["error_class"] == "BadRequestError" + assert payload["error_information"]["error_code"] == "400" + + +@pytest.mark.asyncio +async def test_failure_with_redaction_on_keeps_error_text_that_does_not_quote_the_prompt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + error: Final = litellm.AuthenticationError( + message="Incorrect API key provided", model="gpt-4o", llm_provider="openai" + ) + + redacted: Final = (await _async_failure_payload(monkeypatch, redact=True, error=error))["standard_logging_object"] + raw: Final = (await _async_failure_payload(monkeypatch, redact=False, error=error))["standard_logging_object"] + + assert redacted["error_str"] == raw["error_str"] + assert redacted["error_information"] == raw["error_information"] + + +@pytest.mark.asyncio +async def test_failure_with_redaction_off_logs_the_error_as_raised(monkeypatch: pytest.MonkeyPatch) -> None: + kwargs: Final = await _async_failure_payload(monkeypatch, redact=False, error=_prompt_echo_error()) + + assert SCRUB_PROMPT in kwargs["standard_logging_object"]["error_information"]["error_message"] + assert SCRUB_PROMPT in kwargs["traceback_exception"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("headers", [None, "x", ["a"]], ids=["none", "str", "list"]) +@pytest.mark.parametrize("redact", [True, False], ids=["redaction_on", "redaction_off"]) +async def test_failure_with_non_mapping_headers_is_still_logged( + monkeypatch: pytest.MonkeyPatch, headers: str | list[str] | None, redact: bool +) -> None: + capture: Final = _FailureCapture() + monkeypatch.setattr(litellm, "turn_off_message_logging", redact) + monkeypatch.setattr(litellm, "_async_failure_callback", [capture]) + logging_obj: Final = _prompt_failure_logging_obj({"metadata": {"headers": headers}}) + + await logging_obj.async_failure_handler(_prompt_echo_error(), f"Traceback: {SCRUB_PROMPT}") + + assert capture.async_kwargs is not None + assert (SCRUB_PROMPT in capture.async_kwargs["traceback_exception"]) is not redact + + +def test_sync_failure_hands_callbacks_the_scrubbed_error(monkeypatch: pytest.MonkeyPatch) -> None: + capture: Final = _FailureCapture() + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + monkeypatch.setattr(litellm, "failure_callback", [capture]) + + _prompt_failure_logging_obj().failure_handler(_prompt_echo_error(), f"Traceback: {SCRUB_PROMPT}") + + assert capture.sync_kwargs is not None + assert SCRUB_PROMPT not in json.dumps(capture.sync_kwargs["standard_logging_object"], default=str) + + +def test_sync_failure_hands_lunary_the_scrubbed_traceback_when_only_the_traceback_quotes_the_prompt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + tracked_events: Final[list[dict[str, object]]] = [] + lunary_logger: Final = LunaryLogger() + monkeypatch.setattr(lunary_logger.lunary_client, "track_event", lambda *_, **event: tracked_events.append(event)) + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + monkeypatch.setattr(litellm, "failure_callback", ["lunary"]) + monkeypatch.setattr(litellm_logging, "lunaryLogger", lunary_logger) + error: Final = litellm.APIConnectionError(message="Connection failed", model="gpt-4o", llm_provider="openai") + + _prompt_failure_logging_obj().failure_handler(error, f"ValueError: bad input {SCRUB_PROMPT}\n\n{error}") + + logged_errors: Final = [event["error"] for event in tracked_events if event.get("error") is not None] + assert logged_errors == [{"stack": f"ValueError: bad input redacted-by-litellm\n\n{error}"}] + + def test_enterprise_alerting_loggers_resolves_real_classes(): from litellm_enterprise.enterprise_callbacks.pagerduty.pagerduty import PagerDutyAlerting from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import ResendEmailLogger