mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge 0a813aecee into 7aa7948a47
This commit is contained in:
commit
2d123acb64
23 changed files with 3156 additions and 66 deletions
|
|
@ -9,7 +9,11 @@ from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional
|
|||
from pydantic import BaseModel
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER, EMPTY_MAPPING
|
||||
from litellm.constants import (
|
||||
DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER,
|
||||
EMPTY_MAPPING,
|
||||
REDACTED_BY_LITELLM,
|
||||
)
|
||||
from litellm.types.integrations.argilla import ArgillaItem
|
||||
from litellm.types.integrations.custom_logger import AgenticLoopPlan
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionRequest
|
||||
|
|
@ -898,13 +902,15 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
|
|||
def redacts_messages_itself(self) -> bool:
|
||||
return False
|
||||
|
||||
def redact_standard_logging_payload_from_model_call_details(self, model_call_details: dict) -> dict:
|
||||
def redact_standard_logging_payload_from_model_call_details(self, model_call_details: dict) -> dict[str, object]:
|
||||
"""
|
||||
Redacts or excludes fields from StandardLoggingPayload before callbacks receive it.
|
||||
|
||||
This method handles two features:
|
||||
1. turn_off_message_logging: When True, redacts messages and responses (unless the callback
|
||||
redacts them itself, see `redacts_messages_itself`)
|
||||
redacts them itself, see `redacts_messages_itself`), and redacts `error_str`,
|
||||
`error_information`'s message/traceback and `traceback_exception` independent of
|
||||
`redacts_messages_itself`
|
||||
2. standard_logging_payload_excluded_fields: Removes specified fields entirely
|
||||
|
||||
Return a modified copy of the provided logging payload.
|
||||
|
|
@ -914,6 +920,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
|
|||
import litellm
|
||||
from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS
|
||||
from litellm.litellm_core_utils.redact_messages import (
|
||||
redacted_error_fields,
|
||||
redacted_litellm_params,
|
||||
redacted_standard_logging_payload,
|
||||
)
|
||||
|
|
@ -931,25 +938,41 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
|
|||
if turn_off_message_logging and isinstance(params, Mapping)
|
||||
else EMPTY_MAPPING
|
||||
)
|
||||
redacted_failure_fields: Final[Mapping[str, object]] = (
|
||||
MappingProxyType({"traceback_exception": REDACTED_BY_LITELLM})
|
||||
if turn_off_message_logging
|
||||
and "traceback_exception" in model_call_details
|
||||
and model_call_details["traceback_exception"]
|
||||
else EMPTY_MAPPING
|
||||
)
|
||||
standard_logging_object: Final = model_call_details.get("standard_logging_object")
|
||||
if standard_logging_object is None:
|
||||
return {**model_call_details, **redacted_params}
|
||||
return {**model_call_details, **redacted_params, **redacted_failure_fields}
|
||||
|
||||
# Make a copy of just the standard_logging_object to avoid modifying the original
|
||||
standard_logging_object_copy: Final = {
|
||||
standard_logging_object_copy: Final[dict[str, object]] = { # mutable-ok: typed for the redaction helpers
|
||||
key: value
|
||||
for key, value in standard_logging_object.items()
|
||||
if key not in (excluded_fields or ()) and not (turn_off_message_logging and key in CLASSIFIER_AUDIT_FIELDS)
|
||||
}
|
||||
|
||||
redacted_standard_logging_object: Final = (
|
||||
{
|
||||
**(
|
||||
standard_logging_object_copy
|
||||
if self.redacts_messages_itself()
|
||||
else redacted_standard_logging_payload(standard_logging_object_copy)
|
||||
),
|
||||
**redacted_error_fields(standard_logging_object_copy),
|
||||
}
|
||||
if turn_off_message_logging
|
||||
else standard_logging_object_copy
|
||||
)
|
||||
return {
|
||||
**model_call_details,
|
||||
**redacted_params,
|
||||
"standard_logging_object": (
|
||||
redacted_standard_logging_payload(standard_logging_object_copy)
|
||||
if turn_off_message_logging and not self.redacts_messages_itself()
|
||||
else standard_logging_object_copy
|
||||
),
|
||||
**redacted_failure_fields,
|
||||
"standard_logging_object": redacted_standard_logging_object,
|
||||
}
|
||||
|
||||
async def get_proxy_server_request_from_cold_storage_with_object_key(
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import datetime
|
|||
import os
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import datetime as datetimeObj
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
|
|
@ -287,7 +287,7 @@ class DataDogLogger(
|
|||
|
||||
async def async_post_call_failure_hook(
|
||||
self,
|
||||
request_data: dict,
|
||||
request_data: Mapping[str, object],
|
||||
original_exception: Exception,
|
||||
user_api_key_dict: "UserAPIKeyAuth",
|
||||
traceback_str: str | None = None,
|
||||
|
|
@ -303,12 +303,21 @@ class DataDogLogger(
|
|||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
StandardLoggingPayloadSetup,
|
||||
)
|
||||
from litellm.litellm_core_utils.redact_messages import (
|
||||
redact_error_information,
|
||||
should_redact_failed_request,
|
||||
)
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
|
||||
error_information: Final = StandardLoggingPayloadSetup.get_error_information(
|
||||
_error_information_raw: Final = StandardLoggingPayloadSetup.get_error_information(
|
||||
original_exception=original_exception,
|
||||
traceback_str=traceback_str,
|
||||
)
|
||||
error_information: Final = (
|
||||
redact_error_information(_error_information_raw)
|
||||
if should_redact_failed_request(request_data)
|
||||
else _error_information_raw
|
||||
)
|
||||
_code: Final = error_information.get("error_code") or ""
|
||||
status_code: int | None = None
|
||||
if _code and str(_code).strip().isdigit():
|
||||
|
|
|
|||
|
|
@ -1,9 +1,12 @@
|
|||
import json
|
||||
import threading
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import REDACTED_BY_LITELLM
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_failure_logging
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.utils import StandardLoggingPayload
|
||||
|
|
@ -74,7 +77,7 @@ class MlflowLogger(CustomLogger):
|
|||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
self._handle_failure(kwargs, response_obj, start_time, end_time)
|
||||
|
||||
def _handle_failure(self, kwargs, response_obj, start_time, end_time):
|
||||
def _handle_failure(self, kwargs: Mapping, response_obj, start_time, end_time):
|
||||
"""
|
||||
Log the failure event as an MLflow span.
|
||||
Note that this method is called *synchronously* unlike the success handler.
|
||||
|
|
@ -88,7 +91,19 @@ class MlflowLogger(CustomLogger):
|
|||
|
||||
# Record exception info as event
|
||||
if exception := kwargs.get("exception"):
|
||||
span.add_event(SpanEvent.from_exception(exception))
|
||||
exception_event: Final = (
|
||||
SpanEvent(
|
||||
name="exception",
|
||||
attributes={
|
||||
"exception.type": exception.__class__.__name__,
|
||||
"exception.message": REDACTED_BY_LITELLM,
|
||||
"exception.stacktrace": REDACTED_BY_LITELLM,
|
||||
},
|
||||
)
|
||||
if should_redact_failure_logging(kwargs)
|
||||
else SpanEvent.from_exception(exception)
|
||||
)
|
||||
span.add_event(exception_event)
|
||||
|
||||
self._extract_and_set_chat_attributes(span, kwargs, response_obj)
|
||||
self._end_span_or_trace(
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import KUBERNETES_POD_ROUTING_KEY
|
||||
from litellm.constants import KUBERNETES_POD_ROUTING_KEY, REDACTED_BY_LITELLM
|
||||
from litellm.integrations._types.open_inference import (
|
||||
OpenInferenceSpanKindValues,
|
||||
)
|
||||
|
|
@ -422,6 +422,23 @@ class OpenTelemetryConfig:
|
|||
)
|
||||
|
||||
|
||||
def _server_span_failure_redact(span: "Span") -> bool:
|
||||
"""Redaction decision for re-stamping a SERVER span the failure hook may
|
||||
have already marked: an existing error message/stack-trace attribute keeps
|
||||
its request-aware value, so a restamp with the global-only probe can't leak
|
||||
an opt-in's raw text or clobber a valid opt-out."""
|
||||
from litellm.integrations._types.open_inference import ErrorAttributes
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
|
||||
|
||||
attributes: Final[Mapping[str, object]] = getattr(span, "attributes", None) or {}
|
||||
if ErrorAttributes.ERROR_MESSAGE in attributes or ErrorAttributes.ERROR_STACK_TRACE in attributes:
|
||||
return (
|
||||
attributes.get(ErrorAttributes.ERROR_MESSAGE) == REDACTED_BY_LITELLM
|
||||
or attributes.get(ErrorAttributes.ERROR_STACK_TRACE) == REDACTED_BY_LITELLM
|
||||
)
|
||||
return should_redact_message_logging({})
|
||||
|
||||
|
||||
class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -976,17 +993,26 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
StandardLoggingPayloadSetup,
|
||||
)
|
||||
from litellm.litellm_core_utils.redact_messages import (
|
||||
redact_error_information,
|
||||
should_redact_failed_request,
|
||||
)
|
||||
|
||||
error_information: Final = StandardLoggingPayloadSetup.get_error_information(
|
||||
redact: Final = should_redact_failed_request(request_data)
|
||||
_error_information_raw: Final = StandardLoggingPayloadSetup.get_error_information(
|
||||
original_exception=original_exception,
|
||||
traceback_str=traceback_str,
|
||||
)
|
||||
error_information: Final = (
|
||||
redact_error_information(_error_information_raw) if redact else _error_information_raw
|
||||
)
|
||||
server_error_message: Final = self._record_exception_on_span(
|
||||
span=parent_otel_span,
|
||||
kwargs={
|
||||
"exception": original_exception,
|
||||
"standard_logging_object": {"error_information": error_information},
|
||||
},
|
||||
redact_content=redact,
|
||||
)
|
||||
parent_otel_span.set_status(Status(StatusCode.ERROR, server_error_message))
|
||||
|
||||
|
|
@ -1010,7 +1036,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
self.safe_set_attribute(
|
||||
span=exception_logging_span,
|
||||
key="exception",
|
||||
value=str(original_exception),
|
||||
value=REDACTED_BY_LITELLM if redact else str(original_exception),
|
||||
)
|
||||
self._set_team_attributes_on_span(
|
||||
span=exception_logging_span,
|
||||
|
|
@ -2254,7 +2280,9 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
):
|
||||
parent_otel_span.end(end_time=self._to_ns(end_time))
|
||||
|
||||
def _record_exception_on_span(self, span: Span, kwargs: _FailureKwargsView) -> str | None:
|
||||
def _record_exception_on_span(
|
||||
self, span: Span, kwargs: _FailureKwargsView, redact_content: bool | None = None
|
||||
) -> str | None:
|
||||
"""
|
||||
Record exception information on the span using OTEL standard methods.
|
||||
|
||||
|
|
@ -2263,18 +2291,32 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
2. Sets structured error attributes from StandardLoggingPayloadErrorInformation
|
||||
|
||||
Returns the error message it recorded, if any, for the span status description.
|
||||
``redact_content`` overrides the message-redaction decision; when unset it is
|
||||
resolved from kwargs via ``should_redact_message_logging``.
|
||||
"""
|
||||
try:
|
||||
from litellm.integrations._types.open_inference import (
|
||||
ErrorAttributes,
|
||||
)
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_failure_logging
|
||||
|
||||
redact: Final = should_redact_failure_logging(kwargs) if redact_content is None else redact_content
|
||||
|
||||
# Get the exception object if available
|
||||
exception: Final = kwargs.get("exception")
|
||||
|
||||
# Record the exception using OTEL's standard method
|
||||
if exception is not None:
|
||||
span.record_exception(exception)
|
||||
if redact:
|
||||
span.record_exception(
|
||||
exception,
|
||||
attributes={
|
||||
"exception.message": REDACTED_BY_LITELLM,
|
||||
"exception.stacktrace": REDACTED_BY_LITELLM,
|
||||
},
|
||||
)
|
||||
else:
|
||||
span.record_exception(exception)
|
||||
|
||||
# Get StandardLoggingPayload for structured error information
|
||||
standard_logging_payload: Final = kwargs.get("standard_logging_object")
|
||||
|
|
@ -3638,12 +3680,19 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
StandardLoggingPayloadSetup,
|
||||
)
|
||||
from litellm.litellm_core_utils.redact_messages import redact_error_information
|
||||
|
||||
redact: Final = _server_span_failure_redact(span)
|
||||
error_information: Final = StandardLoggingPayloadSetup.get_error_information(original_exception=exception)
|
||||
error_information["error_code"] = str(status_code)
|
||||
return self._record_exception_on_span(
|
||||
span=span,
|
||||
kwargs={"standard_logging_object": {"error_information": error_information}},
|
||||
kwargs={
|
||||
"standard_logging_object": {
|
||||
"error_information": redact_error_information(error_information) if redact else error_information
|
||||
}
|
||||
},
|
||||
redact_content=redact,
|
||||
)
|
||||
|
||||
def set_preprocessing_duration_attribute(self, span: Span | None, container: object) -> None:
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ from typing_extensions import TypedDict, Unpack
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import REDACTED_BY_LITELLM
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.otel.emitter import SpanEmitter, stamp_error
|
||||
from litellm.integrations.otel.mappers import resolve_mappers
|
||||
|
|
@ -49,7 +50,7 @@ from litellm.integrations.otel.model.payloads import (
|
|||
is_mcp_list_tools,
|
||||
is_mcp_tool_call,
|
||||
)
|
||||
from litellm.integrations.otel.model.semconv import Error
|
||||
from litellm.integrations.otel.model.semconv import Error, LiteLLMError
|
||||
from litellm.integrations.otel.model.spans import SpanRole, span_role_for_service
|
||||
from litellm.integrations.otel.model.utils import to_ns
|
||||
from litellm.integrations.otel.plumbing.context import (
|
||||
|
|
@ -84,6 +85,10 @@ from litellm.integrations.otel.plumbing.providers import (
|
|||
resolve_meter_provider,
|
||||
)
|
||||
from litellm.integrations.otel.plumbing.routing import TenantTracerCache
|
||||
from litellm.litellm_core_utils.redact_messages import (
|
||||
should_redact_failed_request,
|
||||
should_redact_message_logging,
|
||||
)
|
||||
from litellm.types.utils import captures_span_content
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -99,6 +104,21 @@ if TYPE_CHECKING:
|
|||
)
|
||||
|
||||
_published_v2_provider: ApiTracerProvider | None = None
|
||||
_GLOBAL_REDACTION_PROBE: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
|
||||
def _span_failure_redact(span: "Span") -> bool:
|
||||
"""Redaction decision for re-stamping a span the failure hook may have
|
||||
already marked: an existing ``error.message``/stack-trace attribute keeps
|
||||
its request-aware value, so a later restamp with the global probe can't
|
||||
leak an opt-in's raw text or clobber a valid opt-out."""
|
||||
attributes: Final[Mapping[str, object]] = getattr(span, "attributes", None) or {}
|
||||
if Error.MESSAGE in attributes or LiteLLMError.STACK_TRACE in attributes:
|
||||
return (
|
||||
attributes.get(Error.MESSAGE) == REDACTED_BY_LITELLM
|
||||
or attributes.get(LiteLLMError.STACK_TRACE) == REDACTED_BY_LITELLM
|
||||
)
|
||||
return should_redact_message_logging(_GLOBAL_REDACTION_PROBE)
|
||||
|
||||
|
||||
def _span_error_from_exception(
|
||||
|
|
@ -106,6 +126,7 @@ def _span_error_from_exception(
|
|||
*,
|
||||
status_code: int | None = None,
|
||||
traceback_str: str | None = None,
|
||||
redact_content: bool = False,
|
||||
) -> SpanError:
|
||||
"""A ``SpanError`` for a proxy-level failure that never produced a
|
||||
``StandardLoggingPayload`` (auth / validation / malformed-body rejections),
|
||||
|
|
@ -120,9 +141,13 @@ def _span_error_from_exception(
|
|||
)
|
||||
return SpanError(
|
||||
error_type=info.get("error_class") or info.get("error_code") or None,
|
||||
message=info.get("error_message") or None,
|
||||
message=REDACTED_BY_LITELLM
|
||||
if redact_content and info.get("error_message")
|
||||
else (info.get("error_message") or None),
|
||||
code=str(status_code) if status_code is not None else (info.get("error_code") or None),
|
||||
stack_trace=info.get("traceback") or None,
|
||||
stack_trace=REDACTED_BY_LITELLM
|
||||
if redact_content and info.get("traceback")
|
||||
else (info.get("traceback") or None),
|
||||
llm_provider=info.get("llm_provider") or None,
|
||||
)
|
||||
|
||||
|
|
@ -778,7 +803,7 @@ class OpenTelemetryV2(CustomLogger):
|
|||
pass
|
||||
|
||||
@contextmanager
|
||||
def start_phase_span(self, name: str) -> "Iterator[Span]":
|
||||
def start_phase_span(self, name: str, *, redact_content: bool = False) -> "Iterator[Span]":
|
||||
"""A live INTERNAL span the service calls inside the block nest under.
|
||||
|
||||
Parents like a service span: ambient first, and from the post-response phase
|
||||
|
|
@ -787,8 +812,14 @@ class OpenTelemetryV2(CustomLogger):
|
|||
"""
|
||||
parent_context, links = resolve_service_span_context()
|
||||
span: Final = self._emitter.start_span(SpanRole.SERVICE, name, parent_context=parent_context, links=links)
|
||||
redact: Final = redact_content or should_redact_message_logging(_GLOBAL_REDACTION_PROBE)
|
||||
with (
|
||||
use_span(span, end_on_exit=True),
|
||||
use_span(
|
||||
span,
|
||||
end_on_exit=True,
|
||||
record_exception=not redact,
|
||||
set_status_on_exception=not redact,
|
||||
),
|
||||
active_phase(span),
|
||||
post_response_root(span) if links else nullcontext(),
|
||||
):
|
||||
|
|
@ -796,7 +827,12 @@ class OpenTelemetryV2(CustomLogger):
|
|||
yield span
|
||||
except Exception as exc:
|
||||
if is_recordable_span(span):
|
||||
stamp_error(span, _span_error_from_exception(exc), record_event=False, set_status=False)
|
||||
stamp_error(
|
||||
span,
|
||||
_span_error_from_exception(exc, redact_content=redact),
|
||||
record_event=redact,
|
||||
set_status=redact,
|
||||
)
|
||||
raise
|
||||
|
||||
def add_phase_event(self, name: str, attributes: Mapping[str, str | int] | None = None) -> None:
|
||||
|
|
@ -848,13 +884,17 @@ class OpenTelemetryV2(CustomLogger):
|
|||
already_stamped: Final = Error.TYPE in (getattr(span, "attributes", None) or ())
|
||||
stamp_error(
|
||||
span,
|
||||
_span_error_from_exception(exception, status_code=status_code),
|
||||
_span_error_from_exception(
|
||||
exception,
|
||||
status_code=status_code,
|
||||
redact_content=_span_failure_redact(span),
|
||||
),
|
||||
record_event=not already_stamped,
|
||||
)
|
||||
|
||||
async def async_post_call_failure_hook(
|
||||
self,
|
||||
request_data: dict,
|
||||
request_data: Mapping[str, object],
|
||||
original_exception: Exception,
|
||||
user_api_key_dict: "UserAPIKeyAuth",
|
||||
traceback_str: "str | None" = None,
|
||||
|
|
@ -874,7 +914,14 @@ class OpenTelemetryV2(CustomLogger):
|
|||
span: Final = mcp_message_transport_span() or request_root_span() or user_api_key_dict.parent_otel_span
|
||||
if span is None or not is_recordable_span(span):
|
||||
return
|
||||
stamp_error(span, _span_error_from_exception(original_exception, traceback_str=traceback_str))
|
||||
stamp_error(
|
||||
span,
|
||||
_span_error_from_exception(
|
||||
original_exception,
|
||||
traceback_str=traceback_str,
|
||||
redact_content=should_redact_failed_request(request_data),
|
||||
),
|
||||
)
|
||||
return
|
||||
|
||||
def emit_guardrail_span(self, entry: "StandardLoggingGuardrailInformation") -> None:
|
||||
|
|
@ -1063,12 +1110,12 @@ def fan_out_provider() -> ApiTracerProvider:
|
|||
|
||||
|
||||
@contextmanager
|
||||
def phase_span(name: str) -> "Iterator[Span | None]":
|
||||
def phase_span(name: str, *, redact_content: bool = False) -> "Iterator[Span | None]":
|
||||
logger: Final = _registered_v2_logger()
|
||||
if logger is None:
|
||||
yield None
|
||||
return
|
||||
with logger.start_phase_span(name) as span:
|
||||
with logger.start_phase_span(name, redact_content=redact_content) as span:
|
||||
yield span
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ PhaseAttributes: TypeAlias = Mapping[str, str | int | float | bool]
|
|||
@cache
|
||||
def _otel_runtime() -> (
|
||||
tuple[
|
||||
Callable[[str], AbstractContextManager[Span | None]],
|
||||
Callable[..., AbstractContextManager[Span | None]],
|
||||
Callable[..., None],
|
||||
Callable[[str, PhaseEventAttributes | None], None],
|
||||
Callable[[PhaseAttributes], None],
|
||||
|
|
@ -45,7 +45,7 @@ def _otel_runtime() -> (
|
|||
|
||||
|
||||
@contextmanager
|
||||
def phase_span(name: str) -> Generator[Span | None]:
|
||||
def phase_span(name: str, *, redact_content: bool = False) -> Generator[Span | None]:
|
||||
"""Run a request phase inside a live active span so its DB/service calls nest.
|
||||
|
||||
Yields ``None`` (a plain no-op) when the OTel SDK is unavailable or V2 is not
|
||||
|
|
@ -55,7 +55,7 @@ def phase_span(name: str) -> Generator[Span | None]:
|
|||
if runtime is None:
|
||||
yield None
|
||||
return
|
||||
with runtime[0](name) as span:
|
||||
with runtime[0](name, redact_content=redact_content) as span:
|
||||
yield span
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -122,6 +122,7 @@ from litellm.litellm_core_utils.logging_utils import (
|
|||
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
|
||||
from litellm.litellm_core_utils.ptu_pricing import is_spilled_over_ptu_request
|
||||
from litellm.litellm_core_utils.redact_messages import (
|
||||
redact_failure_from_logging,
|
||||
redact_message_input_output_from_custom_logger,
|
||||
redact_message_input_output_from_logging,
|
||||
redacted_model_call_details_for_custom_logger,
|
||||
|
|
@ -4305,14 +4306,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
global_callbacks=litellm.failure_callback,
|
||||
)
|
||||
|
||||
result: object = None # result sent to all loggers, init this to None incase it's not created
|
||||
|
||||
result = _REDACTED_RESULT.validate_python(
|
||||
redact_message_input_output_from_logging(
|
||||
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
|
||||
result=result,
|
||||
)
|
||||
)
|
||||
redact_failure_from_logging(self.model_call_details)
|
||||
result: Final = None
|
||||
self.mark_logging_complete(event_type="sync_failure")
|
||||
for callback in callbacks:
|
||||
try:
|
||||
|
|
@ -4386,7 +4381,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
response_obj=result,
|
||||
kwargs=self.model_call_details,
|
||||
kwargs=callback.redact_standard_logging_payload_from_model_call_details(
|
||||
model_call_details=self.model_call_details
|
||||
),
|
||||
)
|
||||
if callback == "langfuse":
|
||||
global langFuseLogger
|
||||
|
|
@ -4505,8 +4502,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
global_callbacks=litellm._async_failure_callback,
|
||||
)
|
||||
|
||||
result: Final = None # result sent to all loggers, init this to None incase it's not created
|
||||
|
||||
redact_failure_from_logging(self.model_call_details)
|
||||
result: Final = None
|
||||
self.mark_logging_complete(event_type="async_failure")
|
||||
for callback in callbacks:
|
||||
try:
|
||||
|
|
@ -4520,7 +4517,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
continue
|
||||
if isinstance(callback, CustomLogger): # custom logger class
|
||||
await callback.async_log_failure_event(
|
||||
kwargs=self.model_call_details,
|
||||
kwargs=callback.redact_standard_logging_payload_from_model_call_details(
|
||||
model_call_details=self.model_call_details
|
||||
),
|
||||
response_obj=result,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
|
|
|
|||
|
|
@ -12,7 +12,13 @@ import copy
|
|||
import inspect
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Final,
|
||||
cast, # noqa: TID251 # logging payloads arrive as untyped dicts; values are re-typed after an isinstance check
|
||||
)
|
||||
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
|
||||
|
|
@ -23,6 +29,9 @@ from litellm.litellm_core_utils.classifier_logging import without_classifier_aud
|
|||
from litellm.litellm_core_utils.core_helpers import (
|
||||
get_metadata_variable_name_from_kwargs,
|
||||
)
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
|
||||
iter_client_callback_metadata_dicts,
|
||||
)
|
||||
from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY
|
||||
from litellm.llms.base_llm.guardrail_translation.utils import stream_item_field, stream_item_items
|
||||
from litellm.llms.vertex_ai.common_utils import (
|
||||
|
|
@ -31,7 +40,10 @@ from litellm.llms.vertex_ai.common_utils import (
|
|||
)
|
||||
from litellm.secret_managers.main import str_to_bool
|
||||
from litellm.types.router import BaselineRouteStamp
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
from litellm.types.utils import (
|
||||
StandardCallbackDynamicParams,
|
||||
StandardLoggingPayloadErrorInformation,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
|
|
@ -371,6 +383,114 @@ def redacted_standard_logging_payload(payload: Mapping[str, object]) -> Mapping[
|
|||
return _redact_standard_logging_object(payload)
|
||||
|
||||
|
||||
def _is_non_empty_str(value: object) -> bool:
|
||||
return isinstance(value, str) and bool(value)
|
||||
|
||||
|
||||
def _str_keyed_mapping(value: object) -> Mapping[str, object]:
|
||||
return (
|
||||
cast("Mapping[str, object]", value) # cast-ok: logging payload and request body mappings are str-keyed
|
||||
if isinstance(value, Mapping)
|
||||
else MappingProxyType({})
|
||||
)
|
||||
|
||||
|
||||
def redact_error_information(
|
||||
error_information: StandardLoggingPayloadErrorInformation,
|
||||
) -> StandardLoggingPayloadErrorInformation:
|
||||
"""
|
||||
Return a copy of ``StandardLoggingPayloadErrorInformation`` with the fields that can
|
||||
quote the prompt (``error_message``, ``traceback``) replaced by ``REDACTED_BY_LITELLM``
|
||||
when they are non-empty strings. Every other field is carried over unchanged.
|
||||
"""
|
||||
redacted: Final = error_information.copy()
|
||||
if _is_non_empty_str(redacted.get("error_message")):
|
||||
redacted["error_message"] = REDACTED_BY_LITELLM
|
||||
if _is_non_empty_str(redacted.get("traceback")):
|
||||
redacted["traceback"] = REDACTED_BY_LITELLM
|
||||
return redacted
|
||||
|
||||
|
||||
def redacted_error_fields(standard_logging_object: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""
|
||||
The ``error_str`` and ``error_information`` entries of a ``StandardLoggingPayload`` with
|
||||
the text that can quote the prompt replaced by ``REDACTED_BY_LITELLM``. Entries that are
|
||||
absent or empty are left out, so the result merges over a copy of the payload.
|
||||
"""
|
||||
fields: Final[dict[str, object]] = {} # mutable-ok: filled here, merged into the caller's payload copy
|
||||
if standard_logging_object.get("error_str"):
|
||||
fields["error_str"] = REDACTED_BY_LITELLM
|
||||
error_information: Final = standard_logging_object.get("error_information")
|
||||
if isinstance(error_information, Mapping):
|
||||
fields["error_information"] = redact_error_information(
|
||||
cast( # cast-ok: same TypedDict shape as the input mapping
|
||||
StandardLoggingPayloadErrorInformation, error_information
|
||||
)
|
||||
)
|
||||
return fields
|
||||
|
||||
|
||||
def _request_turn_off_message_logging(request_data: object) -> object:
|
||||
"""``turn_off_message_logging`` resolves like ``initialize_standard_callback_dynamic_params``:
|
||||
the top-level value when present, else the first client-metadata slot carrying it.
|
||||
A body that is not a JSON object carries no value."""
|
||||
body: Final = _str_keyed_mapping(request_data)
|
||||
return (
|
||||
body["turn_off_message_logging"]
|
||||
if "turn_off_message_logging" in body
|
||||
else next(
|
||||
(
|
||||
slot["turn_off_message_logging"]
|
||||
for _, slot in iter_client_callback_metadata_dicts(dict(body))
|
||||
if "turn_off_message_logging" in slot
|
||||
),
|
||||
None,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def request_opts_into_message_redaction(headers: Mapping[str, str], request_data: object) -> bool:
|
||||
"""Opt-in signals only: usable at auth time before the key's
|
||||
``allow_client_message_redaction_opt_out`` permission is known, so the disable
|
||||
header and the global flag are deliberately not consulted."""
|
||||
return (
|
||||
any(bool(headers.get(header)) for header in MESSAGE_REDACTION_ENABLE_HEADERS)
|
||||
or _request_turn_off_message_logging(request_data) is True
|
||||
)
|
||||
|
||||
|
||||
def should_redact_failed_request(request_data: Mapping[str, object]) -> bool:
|
||||
"""
|
||||
Message-redaction decision for proxy ``async_post_call_failure_hook`` callbacks, which
|
||||
only receive ``request_data`` (the Logging object is popped before hooks run).
|
||||
``litellm_metadata`` is included only when present so
|
||||
``get_metadata_variable_name_from_kwargs`` resolves ``metadata`` for chat routes.
|
||||
"""
|
||||
dynamic_param: Final = _request_turn_off_message_logging(request_data)
|
||||
litellm_params: Final = MappingProxyType(
|
||||
{
|
||||
key: request_data.get(key)
|
||||
for key in ("metadata", "litellm_metadata")
|
||||
if key == "metadata" or key in request_data
|
||||
}
|
||||
)
|
||||
return should_redact_failure_logging(
|
||||
{
|
||||
"litellm_params": litellm_params,
|
||||
"standard_callback_dynamic_params": {"turn_off_message_logging": dynamic_param},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def maybe_redact_error_information(
|
||||
error_information: StandardLoggingPayloadErrorInformation,
|
||||
request_data: Mapping[str, object],
|
||||
) -> StandardLoggingPayloadErrorInformation:
|
||||
if should_redact_failed_request(request_data):
|
||||
return redact_error_information(error_information)
|
||||
return error_information
|
||||
|
||||
|
||||
def _redact_standard_logging_object(
|
||||
payload: Mapping[str, object], *, redact_instructions: bool = True
|
||||
) -> dict[str, object]:
|
||||
|
|
@ -401,6 +521,8 @@ def _redact_standard_logging_object(
|
|||
else:
|
||||
# For other formats (empty dict, None, etc.), use simple text format
|
||||
standard_logging_object["response"] = {"text": redacted_str}
|
||||
|
||||
standard_logging_object.update(redacted_error_fields(standard_logging_object))
|
||||
return standard_logging_object
|
||||
|
||||
|
||||
|
|
@ -535,6 +657,12 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
|
|||
model_call_details["standard_logging_object"] = _redact_standard_logging_object(
|
||||
standard_logging_object, redact_instructions=redact_streaming_responses
|
||||
)
|
||||
if (
|
||||
"traceback_exception" in model_call_details
|
||||
and isinstance(model_call_details["traceback_exception"], str)
|
||||
and model_call_details["traceback_exception"]
|
||||
):
|
||||
model_call_details["traceback_exception"] = REDACTED_BY_LITELLM # rebind-ok: redacts in place
|
||||
redact_vertex_ai_metadata_from_litellm_params(model_call_details)
|
||||
|
||||
# Redact streaming response
|
||||
|
|
@ -596,7 +724,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
|
|||
return _result
|
||||
|
||||
|
||||
def should_redact_message_logging(model_call_details: dict) -> bool:
|
||||
def should_redact_message_logging(model_call_details: Mapping) -> bool:
|
||||
"""
|
||||
Determine if message logging should be redacted.
|
||||
|
||||
|
|
@ -605,18 +733,23 @@ def should_redact_message_logging(model_call_details: dict) -> bool:
|
|||
2. Headers (litellm-disable-message-redaction / litellm-enable-message-redaction)
|
||||
3. Global setting (litellm.turn_off_message_logging)
|
||||
"""
|
||||
litellm_params: Final = model_call_details.get("litellm_params", {})
|
||||
raw_litellm_params: Final[object] = model_call_details.get("litellm_params")
|
||||
litellm_params: Final = _str_keyed_mapping(raw_litellm_params)
|
||||
|
||||
metadata_field: Final = get_metadata_variable_name_from_kwargs(litellm_params)
|
||||
metadata = litellm_params.get(metadata_field, {})
|
||||
if not isinstance(metadata, dict):
|
||||
# Fall back: litellm_metadata was None, try metadata
|
||||
metadata = litellm_params.get("metadata", {})
|
||||
if not isinstance(metadata, dict):
|
||||
metadata = {}
|
||||
# Fall back to metadata when the resolved field (litellm_metadata) is not a mapping
|
||||
metadata: Final = _str_keyed_mapping(
|
||||
litellm_params[metadata_field]
|
||||
if isinstance(litellm_params.get(metadata_field), Mapping)
|
||||
else litellm_params.get("metadata")
|
||||
)
|
||||
|
||||
# Get headers from the metadata
|
||||
request_headers: Final = metadata.get("headers", {})
|
||||
raw_headers: Final = metadata.get("headers", {})
|
||||
if not isinstance(raw_headers, Mapping):
|
||||
# Callers such as the complexity router read AttributeError as an undecidable request and redact
|
||||
raise AttributeError(f"request headers are {type(raw_headers).__name__}, not a mapping") # noqa: TRY004 # callers catch AttributeError
|
||||
request_headers: Final = _str_keyed_mapping(metadata.get("headers"))
|
||||
|
||||
# Check for headers that explicitly control redaction
|
||||
if request_headers and bool(request_headers.get("litellm-disable-message-redaction", False)):
|
||||
|
|
@ -629,14 +762,33 @@ def should_redact_message_logging(model_call_details: dict) -> bool:
|
|||
is_redaction_enabled_via_header = True
|
||||
break
|
||||
|
||||
return _redaction_decision(
|
||||
_get_turn_off_message_logging_from_dynamic_params(model_call_details),
|
||||
enabled_via_header=is_redaction_enabled_via_header,
|
||||
)
|
||||
|
||||
|
||||
def should_redact_failure_logging(model_call_details: Mapping) -> bool:
|
||||
"""
|
||||
`should_redact_message_logging` for failure logging, where request headers that are not
|
||||
a mapping carry no redaction header instead of raising.
|
||||
"""
|
||||
try:
|
||||
return should_redact_message_logging(model_call_details)
|
||||
except AttributeError:
|
||||
return _redaction_decision(
|
||||
_get_turn_off_message_logging_from_dynamic_params(model_call_details), enabled_via_header=False
|
||||
)
|
||||
|
||||
|
||||
def _redaction_decision(dynamic_turn_off: bool | None, *, enabled_via_header: bool) -> bool:
|
||||
# Priority 1: Check dynamic parameter first (if explicitly set)
|
||||
dynamic_turn_off: Final = _get_turn_off_message_logging_from_dynamic_params(model_call_details)
|
||||
if dynamic_turn_off is not None:
|
||||
# Dynamic parameter is explicitly set, use it
|
||||
return dynamic_turn_off
|
||||
|
||||
# Priority 2: Check if header explicitly enables redaction
|
||||
if is_redaction_enabled_via_header:
|
||||
if enabled_via_header:
|
||||
return True
|
||||
|
||||
# Priority 3: Fall back to global setting
|
||||
|
|
@ -653,8 +805,14 @@ def redact_message_input_output_from_logging(model_call_details: dict, result, i
|
|||
return result
|
||||
|
||||
|
||||
def redact_failure_from_logging(model_call_details: dict[str, object]) -> None:
|
||||
"""Redacts the failure logging object in place when `should_redact_failure_logging` says so"""
|
||||
if should_redact_failure_logging(model_call_details):
|
||||
perform_redaction(model_call_details, None)
|
||||
|
||||
|
||||
def _get_turn_off_message_logging_from_dynamic_params(
|
||||
model_call_details: dict,
|
||||
model_call_details: Mapping,
|
||||
) -> bool | None:
|
||||
"""
|
||||
gets the value of `turn_off_message_logging` from the dynamic params, if it exists.
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ from litellm.integrations.otel.model.config import is_otel_v2_enabled
|
|||
from litellm.integrations.otel.runtime import phase_event, phase_span, seed_request_identity
|
||||
from litellm.litellm_core_utils.dd_tracing import tracer
|
||||
from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value
|
||||
from litellm.litellm_core_utils.redact_messages import request_opts_into_message_redaction
|
||||
from litellm.proxy._types import *
|
||||
from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_from_headers
|
||||
from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module exports
|
||||
|
|
@ -4284,7 +4285,10 @@ async def user_api_key_auth(
|
|||
# onto the server span, and name every cache read in it an auth-object read.
|
||||
# No-op when OTel V2 isn't active.
|
||||
with (
|
||||
phase_span(f"auth {route}"),
|
||||
phase_span(
|
||||
f"auth {route}",
|
||||
redact_content=request_opts_into_message_redaction(safe_get_request_headers(request), request_data),
|
||||
),
|
||||
service_target(AUTH_OBJECTS_TARGET),
|
||||
spend_counter_batch_scope(_spend_counter_redis_cache()),
|
||||
):
|
||||
|
|
|
|||
|
|
@ -22,7 +22,10 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost
|
||||
from litellm.litellm_core_utils.redact_messages import MESSAGE_REDACTION_ENABLE_HEADERS
|
||||
from litellm.litellm_core_utils.redact_messages import (
|
||||
MESSAGE_REDACTION_ENABLE_HEADERS,
|
||||
maybe_redact_error_information,
|
||||
)
|
||||
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
|
|
@ -418,9 +421,12 @@ class _ProxyDBLogger(CustomLogger):
|
|||
)
|
||||
_metadata["user_api_key"] = LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict)
|
||||
_metadata["status"] = "failure"
|
||||
_error_information = StandardLoggingPayloadSetup.get_error_information(
|
||||
original_exception=original_exception,
|
||||
traceback_str=traceback_str,
|
||||
_error_information = maybe_redact_error_information(
|
||||
error_information=StandardLoggingPayloadSetup.get_error_information(
|
||||
original_exception=original_exception,
|
||||
traceback_str=traceback_str,
|
||||
),
|
||||
request_data=request_data,
|
||||
)
|
||||
if should_suppress_spend_log_tracebacks():
|
||||
# Drop the traceback key entirely so the per-row Metadata pane in
|
||||
|
|
|
|||
1122
tests/integration/observability/test_failure_redaction.py
Normal file
1122
tests/integration/observability/test_failure_redaction.py
Normal file
File diff suppressed because it is too large
Load diff
328
tests/integration/observability/test_failure_redaction_chaos.py
Normal file
328
tests/integration/observability/test_failure_redaction_chaos.py
Normal file
|
|
@ -0,0 +1,328 @@
|
|||
import json
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import Gateway, JsonValue, eventually, gateway_from_environment, object_value
|
||||
from integration._support.process import OwnedProxy, OwnedState, owned_proxy_process
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
|
||||
BURST: Final = 30
|
||||
|
||||
|
||||
def _provider(request: Request) -> Reply:
|
||||
try:
|
||||
body: Final = json.loads(request.body)
|
||||
except json.JSONDecodeError:
|
||||
return Reply(status=404, body=b"{}")
|
||||
text: Final = (
|
||||
body["messages"][-1]["content"] if "messages" in body else str(body.get("input", json.dumps(body)[:200]))
|
||||
)
|
||||
return Reply(
|
||||
status=400,
|
||||
body=json.dumps(
|
||||
{"error": {"type": "invalid_request_error", "message": f"Unsupported content: {text}"}}
|
||||
).encode(),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Rig:
|
||||
proxy: Gateway
|
||||
process: OwnedProxy
|
||||
provider: Wire
|
||||
sink: Wire
|
||||
model: str
|
||||
outage: threading.Event
|
||||
slow: threading.Event
|
||||
batches: list[Request]
|
||||
|
||||
def failure_events(self) -> tuple[dict[str, JsonValue], ...]:
|
||||
self.batches.extend(self.sink.drain()) # mutable-ok: drain consumes, polls keep earlier batches
|
||||
return tuple(
|
||||
object_value(event)
|
||||
for batch in self.batches
|
||||
for event in json.loads(batch.body)
|
||||
if self.model in json.dumps(event)
|
||||
)
|
||||
|
||||
|
||||
def _bodies(model: str, marker: str) -> tuple[tuple[str, dict[str, JsonValue]], ...]:
|
||||
chat: Final = tuple(
|
||||
(
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": f"burst {marker} {index}"}],
|
||||
**({"stream": True} if index % 2 else {}),
|
||||
},
|
||||
)
|
||||
for index in range(BURST // 3 * 2)
|
||||
)
|
||||
messages: Final = tuple(
|
||||
(
|
||||
"/v1/messages",
|
||||
{
|
||||
"model": model,
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": f"burst {marker} m{index}"}],
|
||||
},
|
||||
)
|
||||
for index in range(BURST // 6)
|
||||
)
|
||||
responses: Final = tuple(
|
||||
("/v1/responses", {"model": model, "input": f"burst {marker} r{index}"})
|
||||
for index in range(BURST - len(chat) - len(messages))
|
||||
)
|
||||
return chat + messages + responses
|
||||
|
||||
|
||||
def _json_body_ok(body: bytes) -> bool:
|
||||
try:
|
||||
json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _fire(rig: Rig, bodies: tuple[tuple[str, dict[str, JsonValue]], ...]) -> tuple[tuple[int, str | None], ...]:
|
||||
def call(item: tuple[str, dict[str, JsonValue]]) -> tuple[int, str | None]:
|
||||
try:
|
||||
response: Final = rig.proxy.request("POST", item[0], item[1])
|
||||
response.read()
|
||||
return response.status_code, response.headers.get("x-litellm-call-id")
|
||||
except httpx.HTTPError:
|
||||
return -1, None
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
return tuple(pool.map(call, bodies))
|
||||
|
||||
|
||||
@pytest.mark.timeout(280)
|
||||
def test_g1_sink_outage_mid_burst_lands_each_call_id_once_redacted(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
outage: Final = threading.Event()
|
||||
|
||||
def sink(request: Request) -> Reply:
|
||||
if outage.is_set():
|
||||
return Reply(status=503, body=b'{"error":"sink down"}')
|
||||
return Reply()
|
||||
|
||||
with wire_server(_provider) as provider, wire_server(sink) as endpoint:
|
||||
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
config["litellm_settings"].update(
|
||||
{"callbacks": ["generic_api"], "DEFAULT_FLUSH_INTERVAL_SECONDS": 1, "turn_off_message_logging": True}
|
||||
)
|
||||
path: Final = tmp_path / "chaos.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
with (
|
||||
gateway_from_environment() as gateway,
|
||||
owned_proxy_process(
|
||||
gateway, tmp_path, {"GENERIC_LOGGER_ENDPOINT": endpoint.url}, config=path, workers=2
|
||||
) as owned,
|
||||
owned.gateway.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(api_base=provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
rig: Final = Rig(owned.gateway, owned, provider, endpoint, model, outage, threading.Event(), [])
|
||||
bodies: Final = _bodies(model, marker)
|
||||
outage.set()
|
||||
first_half: Final = _fire(rig, bodies[: BURST // 2])
|
||||
outage.clear()
|
||||
second_half: Final = _fire(rig, bodies[BURST // 2 :])
|
||||
outcomes: Final = first_half + second_half
|
||||
answered: Final = tuple(outcome for outcome in outcomes if outcome[0] >= 0)
|
||||
assert all(status == 400 for status, _ in answered), outcomes
|
||||
events: Final = eventually(rig.failure_events, lambda values: len(values) >= len(second_half), seconds=70)
|
||||
seen: Final = tuple(str(event.get("litellm_call_id")) for event in events)
|
||||
assert len(seen) == len(set(seen)), ("duplicate failure events", seen)
|
||||
for event in events:
|
||||
assert marker not in json.dumps(event), json.dumps(event)[:400]
|
||||
|
||||
|
||||
@pytest.mark.timeout(280)
|
||||
def test_g2_slow_sink_no_deadlock_no_duplicates(tmp_path: Path) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
slow: Final = threading.Event()
|
||||
|
||||
def sink(request: Request) -> Reply:
|
||||
if slow.is_set():
|
||||
time.sleep(1)
|
||||
return Reply()
|
||||
|
||||
with wire_server(_provider) as provider, wire_server(sink) as endpoint:
|
||||
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
config["litellm_settings"].update(
|
||||
{"callbacks": ["generic_api"], "DEFAULT_FLUSH_INTERVAL_SECONDS": 1, "turn_off_message_logging": True}
|
||||
)
|
||||
path: Final = tmp_path / "chaos_slow.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
with (
|
||||
gateway_from_environment() as gateway,
|
||||
owned_proxy_process(
|
||||
gateway, tmp_path, {"GENERIC_LOGGER_ENDPOINT": endpoint.url}, config=path, workers=2
|
||||
) as owned,
|
||||
owned.gateway.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(api_base=provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
rig: Final = Rig(owned.gateway, owned, provider, endpoint, model, threading.Event(), slow, [])
|
||||
slow.set()
|
||||
bodies: Final = _bodies(model, marker)[:6]
|
||||
outcomes: Final = _fire(rig, bodies)
|
||||
assert all(status == 400 for status, _ in outcomes), outcomes
|
||||
call_ids: Final = tuple(cid for _, cid in outcomes if cid)
|
||||
events: Final = eventually(rig.failure_events, lambda values: len(values) >= len(bodies), seconds=120)
|
||||
landed: Final = tuple(str(event.get("litellm_call_id")) for event in events)
|
||||
assert len(landed) == len(set(landed)), ("duplicate failure events", landed)
|
||||
for event in events:
|
||||
assert marker not in json.dumps(event), json.dumps(event)[:400]
|
||||
assert set(call_ids) <= set(landed), (call_ids, landed)
|
||||
|
||||
|
||||
@pytest.mark.timeout(280)
|
||||
def test_g3_proxy_restart_mid_burst(tmp_path: Path) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
model: Final = f"restart-model-{uuid.uuid4().hex}"
|
||||
with wire_server(_provider) as provider, wire_server(lambda _: Reply()) as endpoint:
|
||||
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
config["litellm_settings"].update(
|
||||
{"callbacks": ["generic_api"], "DEFAULT_FLUSH_INTERVAL_SECONDS": 1, "turn_off_message_logging": True}
|
||||
)
|
||||
path: Final = tmp_path / "chaos_restart.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
overrides: Final = {"GENERIC_LOGGER_ENDPOINT": endpoint.url}
|
||||
with gateway_from_environment() as gateway:
|
||||
bodies: Final = _bodies(model, marker)
|
||||
with owned_proxy_process(
|
||||
gateway, tmp_path, overrides, config=path, workers=2, state=OwnedState.SHARED_WITH_WORKER
|
||||
) as owned_one:
|
||||
owned_one.gateway.post(
|
||||
"/model/new",
|
||||
{
|
||||
"model_name": model,
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_base": provider.url + "/v1",
|
||||
"api_key": "synthetic-provider-key",
|
||||
},
|
||||
},
|
||||
)
|
||||
first: Final = _fire(
|
||||
Rig(
|
||||
owned_one.gateway,
|
||||
owned_one,
|
||||
provider,
|
||||
endpoint,
|
||||
model,
|
||||
threading.Event(),
|
||||
threading.Event(),
|
||||
[],
|
||||
),
|
||||
bodies[: BURST // 2],
|
||||
)
|
||||
with owned_proxy_process(
|
||||
gateway, tmp_path, overrides, config=path, workers=2, state=OwnedState.SHARED_WITH_WORKER
|
||||
) as owned_two:
|
||||
rig_two: Final = Rig(
|
||||
owned_two.gateway,
|
||||
owned_two,
|
||||
provider,
|
||||
endpoint,
|
||||
model,
|
||||
threading.Event(),
|
||||
threading.Event(),
|
||||
[],
|
||||
)
|
||||
|
||||
probes: list[tuple[int, str]] = [] # mutable-ok: readiness polls are real served requests
|
||||
|
||||
def served() -> tuple[int, ...]:
|
||||
try:
|
||||
probe: Final = rig_two.proxy.request("POST", *bodies[BURST // 2])
|
||||
except httpx.HTTPError:
|
||||
return (-1,)
|
||||
probes.append((probe.status_code, probe.headers.get("x-litellm-call-id") or ""))
|
||||
return (probe.status_code,)
|
||||
|
||||
eventually(served, lambda statuses: statuses[0] == 400, seconds=60)
|
||||
provider.drain()
|
||||
second: Final = _fire(rig_two, bodies[BURST // 2 :])
|
||||
answered: Final = first + second + tuple(probes)
|
||||
assert all(status in (400, 500) for status, _ in answered), answered
|
||||
post_restart_ids: Final = frozenset(
|
||||
cid for status, cid in second + tuple(probes) if status == 400 and cid
|
||||
)
|
||||
assert post_restart_ids, answered
|
||||
assert any(marker.encode() in request.body for request in provider.drain())
|
||||
collected: list[Request] = [] # mutable-ok: drain consumes batches, later polls keep earlier ones
|
||||
|
||||
def landed() -> tuple[str, ...]:
|
||||
collected.extend(endpoint.drain())
|
||||
return tuple(
|
||||
str(event.get("litellm_call_id"))
|
||||
for batch in collected
|
||||
if _json_body_ok(batch.body)
|
||||
for event in json.loads(batch.body)
|
||||
)
|
||||
|
||||
landed_ids: Final = eventually(landed, lambda ids: post_restart_ids <= set(ids), seconds=60)
|
||||
assert all(not batch.body.strip() for batch in collected if not _json_body_ok(batch.body)), collected
|
||||
assert len(landed_ids) == len(set(landed_ids)), ("duplicate events after restart", landed_ids)
|
||||
events: Final = tuple(
|
||||
object_value(event)
|
||||
for batch in collected
|
||||
if _json_body_ok(batch.body)
|
||||
for event in json.loads(batch.body)
|
||||
)
|
||||
for event in events:
|
||||
assert marker not in json.dumps(event), json.dumps(event)[:400]
|
||||
|
||||
|
||||
@pytest.mark.timeout(280)
|
||||
def test_g4_worker_kill_keeps_serving_redacted(tmp_path: Path) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_provider) as provider, wire_server(lambda _: Reply()) as endpoint:
|
||||
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
config["litellm_settings"].update(
|
||||
{"callbacks": ["generic_api"], "DEFAULT_FLUSH_INTERVAL_SECONDS": 1, "turn_off_message_logging": True}
|
||||
)
|
||||
path: Final = tmp_path / "chaos_worker.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
with (
|
||||
gateway_from_environment() as gateway,
|
||||
owned_proxy_process(
|
||||
gateway, tmp_path, {"GENERIC_LOGGER_ENDPOINT": endpoint.url}, config=path, workers=2
|
||||
) as owned,
|
||||
owned.gateway.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(api_base=provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
rig: Final = Rig(owned.gateway, owned, provider, endpoint, model, threading.Event(), threading.Event(), [])
|
||||
bodies: Final = _bodies(model, marker)[:12]
|
||||
workers: Final = sorted(owned.worker_pids())
|
||||
assert workers, "no uvicorn worker children found"
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
futures: Final = tuple(
|
||||
pool.submit(lambda b: rig.proxy.request("POST", b[0], b[1]), body) for body in bodies
|
||||
)
|
||||
eventually(lambda: provider.received.qsize() >= 3, bool, seconds=30)
|
||||
owned.kill_worker(workers[0])
|
||||
statuses: list[int] = [] # mutable-ok: collect per-request outcomes from concurrent futures
|
||||
for future in futures:
|
||||
try:
|
||||
statuses.append(future.result().status_code)
|
||||
except httpx.HTTPError:
|
||||
statuses.append(-1)
|
||||
assert all(status == 400 for status in statuses if status >= 0), statuses
|
||||
events: Final = eventually(rig.failure_events, lambda values: len(values) >= 1, seconds=70)
|
||||
landed: Final = tuple(str(event.get("litellm_call_id")) for event in events)
|
||||
assert len(landed) == len(set(landed)), ("duplicate failure events", landed)
|
||||
for event in events:
|
||||
assert marker not in json.dumps(event), json.dumps(event)[:400]
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
import gzip
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import Gateway, JsonValue, eventually, gateway_from_environment, object_value
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
|
||||
|
||||
def _body(batch: Request) -> bytes:
|
||||
return gzip.decompress(batch.body) if batch.body[:2] == b"\x1f\x8b" else batch.body
|
||||
|
||||
|
||||
def _provider(request: Request) -> Reply:
|
||||
try:
|
||||
body: Final = json.loads(request.body)
|
||||
except json.JSONDecodeError:
|
||||
return Reply(status=404, body=b"{}")
|
||||
text: Final = body["messages"][-1]["content"]
|
||||
return Reply(
|
||||
status=400,
|
||||
body=json.dumps(
|
||||
{"error": {"type": "invalid_request_error", "message": f"Unsupported content: {text}"}}
|
||||
).encode(),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Rig:
|
||||
proxy: Gateway
|
||||
provider: Wire
|
||||
sink: Wire
|
||||
|
||||
def log_entries(self, model: str) -> tuple[dict[str, JsonValue], ...]:
|
||||
return tuple(
|
||||
object_value(entry)
|
||||
for batch in self.sink.drain()
|
||||
for entry in json.loads(_body(batch))
|
||||
if model in json.dumps(entry)
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
|
||||
root: Final = tmp_path_factory.mktemp("failure_redaction_datadog")
|
||||
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
config["litellm_settings"].update(
|
||||
{"callbacks": ["datadog"], "DEFAULT_FLUSH_INTERVAL_SECONDS": 1, "turn_off_message_logging": True}
|
||||
)
|
||||
path: Final = root / "datadog_failure.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
with (
|
||||
wire_server(_provider) as provider,
|
||||
wire_server(lambda _: Reply()) as sink,
|
||||
gateway_from_environment() as gateway,
|
||||
owned_proxy(
|
||||
gateway,
|
||||
root,
|
||||
{"DD_API_KEY": "synthetic-dd-key", "DD_BASE_URL": sink.url, "DD_SITE": "localhost"},
|
||||
config=path,
|
||||
workers=2,
|
||||
) as proxy,
|
||||
):
|
||||
yield Rig(proxy, provider, sink)
|
||||
|
||||
|
||||
def test_c5_datadog_failure_log_redacted_keeps_status(rig: Rig) -> None:
|
||||
secret: Final = "dd-secret-" + uuid.uuid4().hex
|
||||
with rig.proxy.scenario() as scenario:
|
||||
model: Final = scenario.model(api_base=rig.provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
response: Final = rig.proxy.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": secret}]},
|
||||
)
|
||||
assert response.status_code == 400, response.text
|
||||
assert any(secret.encode() in request.body for request in rig.provider.drain())
|
||||
entries: Final = eventually(lambda: rig.log_entries(model), lambda values: len(values) >= 1, seconds=30)
|
||||
entry: Final = entries[0]
|
||||
assert secret not in json.dumps(entry), json.dumps(entry)[:2000]
|
||||
message: Final = object_value(json.loads(str(entry.get("message", "{}"))))
|
||||
assert secret not in json.dumps(message), json.dumps(message)[:2000]
|
||||
error_information: Final = object_value(message.get("error_information"))
|
||||
assert error_information.get("error_class"), error_information
|
||||
330
tests/integration/observability/test_failure_redaction_otel.py
Normal file
330
tests/integration/observability/test_failure_redaction_otel.py
Normal file
|
|
@ -0,0 +1,330 @@
|
|||
import json
|
||||
import uuid
|
||||
from collections.abc import Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
from integration._support.client import Gateway, JsonValue, eventually, gateway_from_environment, object_value
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
|
||||
|
||||
|
||||
def _provider(request: Request) -> Reply:
|
||||
try:
|
||||
body: Final = json.loads(request.body)
|
||||
except json.JSONDecodeError:
|
||||
return Reply(status=404, body=b"{}")
|
||||
text: Final = body["messages"][-1]["content"]
|
||||
return Reply(
|
||||
status=400,
|
||||
body=json.dumps(
|
||||
{"error": {"type": "invalid_request_error", "message": f"Unsupported content: {text}"}}
|
||||
).encode(),
|
||||
)
|
||||
|
||||
|
||||
def _decode(body: bytes) -> dict[str, JsonValue]:
|
||||
if body[:1] == b"{":
|
||||
return json.loads(body)
|
||||
request: Final = ExportTraceServiceRequest()
|
||||
request.ParseFromString(body)
|
||||
return object_value(MessageToDict(request))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Spans:
|
||||
wire: Wire
|
||||
batches: list[Request]
|
||||
|
||||
def all(self) -> tuple[dict[str, JsonValue], ...]:
|
||||
self.batches.extend(self.wire.drain())
|
||||
return tuple(
|
||||
span
|
||||
for batch in self.batches
|
||||
for resource in _decode(batch.body).get("resourceSpans", ())
|
||||
for scope in resource.get("scopeSpans", ())
|
||||
for span in scope.get("spans", ())
|
||||
)
|
||||
|
||||
def named(self, name: str, model: str) -> tuple[dict[str, JsonValue], ...]:
|
||||
return tuple(span for span in self.all() if span.get("name") == name and model in json.dumps(span))
|
||||
|
||||
def in_trace(self, name: str, trace_id: str) -> tuple[dict[str, JsonValue], ...]:
|
||||
return tuple(span for span in self.all() if span.get("name") == name and span.get("traceId") == trace_id)
|
||||
|
||||
|
||||
def _span_attributes(span: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
|
||||
return {
|
||||
str(attribute["key"]): object_value(attribute["value"]).get("stringValue")
|
||||
or object_value(attribute["value"]).get("intValue")
|
||||
for attribute in span.get("attributes", ())
|
||||
if isinstance(attribute, dict)
|
||||
}
|
||||
|
||||
|
||||
def _exception_events(span: Mapping[str, JsonValue]) -> tuple[dict[str, JsonValue], ...]:
|
||||
return tuple(
|
||||
object_value(event)
|
||||
for event in span.get("events", ())
|
||||
if isinstance(event, dict) and event.get("name") == "exception"
|
||||
)
|
||||
|
||||
|
||||
def _exception_texts(span: Mapping[str, JsonValue]) -> str:
|
||||
return json.dumps(_exception_events(span))
|
||||
|
||||
|
||||
def _error_attribute(span: Mapping[str, JsonValue]) -> str:
|
||||
attributes: Final = _span_attributes(span)
|
||||
return str(attributes.get("error.message", ""))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Rig:
|
||||
proxy: Gateway
|
||||
provider: Wire
|
||||
sink: Spans
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _otel_rig(root: Path, provider: Wire, sink: Wire, v2: bool, global_on: bool) -> Iterator[Rig]:
|
||||
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
settings: Final[dict[str, JsonValue]] = {"callbacks": ["otel"]}
|
||||
if global_on:
|
||||
settings["turn_off_message_logging"] = True
|
||||
config["litellm_settings"].update(settings)
|
||||
if v2:
|
||||
config["callback_settings"] = {
|
||||
"otel": {"exporter": "http/json", "endpoint": sink.url, "mapper_names": ["genai"]}
|
||||
}
|
||||
path: Final = root / "otel_failure.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
env: Final = (
|
||||
{"LITELLM_OTEL_V2": "1", "OTEL_BSP_SCHEDULE_DELAY": "300"}
|
||||
if v2
|
||||
else {
|
||||
"OTEL_EXPORTER": "http/json",
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT": sink.url,
|
||||
"OTEL_BSP_SCHEDULE_DELAY": "300",
|
||||
}
|
||||
)
|
||||
with (
|
||||
gateway_from_environment() as gateway,
|
||||
owned_proxy(gateway, root, env, config=path, workers=2) as proxy,
|
||||
):
|
||||
yield Rig(proxy, provider, Spans(sink, [])) # mutable-ok: drain consumes, polls keep earlier batches
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def provider() -> Iterator[Wire]:
|
||||
with wire_server(_provider) as wire:
|
||||
yield wire
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def sink() -> Iterator[Wire]:
|
||||
with wire_server(lambda _: Reply()) as wire:
|
||||
yield wire
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rig_v1_on(tmp_path_factory: pytest.TempPathFactory, provider: Wire, sink: Wire) -> Iterator[Rig]:
|
||||
with _otel_rig(tmp_path_factory.mktemp("otel_v1_on"), provider, sink, v2=False, global_on=True) as booted:
|
||||
yield booted
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rig_v2_on(tmp_path_factory: pytest.TempPathFactory, provider: Wire, sink: Wire) -> Iterator[Rig]:
|
||||
with _otel_rig(tmp_path_factory.mktemp("otel_v2_on"), provider, sink, v2=True, global_on=True) as booted:
|
||||
yield booted
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rig_v2_off(tmp_path_factory: pytest.TempPathFactory, provider: Wire, sink: Wire) -> Iterator[Rig]:
|
||||
with _otel_rig(tmp_path_factory.mktemp("otel_v2_off"), provider, sink, v2=True, global_on=False) as booted:
|
||||
yield booted
|
||||
|
||||
|
||||
def _secret() -> str:
|
||||
return "otel-secret-" + uuid.uuid4().hex
|
||||
|
||||
|
||||
def _fail(rig: Rig, model: str, secret: str, **kwargs: JsonValue) -> httpx.Response:
|
||||
headers: Final = kwargs.pop("headers", None)
|
||||
key: Final = kwargs.pop("key", None)
|
||||
return rig.proxy.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": secret}], **kwargs},
|
||||
headers=headers if isinstance(headers, dict) else None,
|
||||
key=key if isinstance(key, str) else None,
|
||||
)
|
||||
|
||||
|
||||
def _llm_span(rig: Rig, call_id: str) -> dict[str, JsonValue]:
|
||||
def found() -> tuple[dict[str, JsonValue], ...]:
|
||||
return tuple(
|
||||
span
|
||||
for span in rig.sink.all()
|
||||
if call_id in json.dumps(span) and str(span.get("name", "")).startswith(("chat ", "litellm_request"))
|
||||
)
|
||||
|
||||
return eventually(found, lambda values: len(values) >= 1, seconds=260)[0]
|
||||
|
||||
|
||||
_SERVER_SPAN_NAMES: Final = ("Received Proxy Server Request", "POST /v1/chat/completions", "POST /v1/messages")
|
||||
|
||||
|
||||
def _server_span(rig: Rig, call_id: str) -> dict[str, JsonValue]:
|
||||
trace_id: Final = str(_llm_span(rig, call_id)["traceId"])
|
||||
|
||||
def found() -> tuple[dict[str, JsonValue], ...]:
|
||||
return tuple(
|
||||
span
|
||||
for span in rig.sink.all()
|
||||
if span.get("traceId") == trace_id and str(span.get("name", "")) in _SERVER_SPAN_NAMES
|
||||
)
|
||||
|
||||
spans: Final = eventually(found, lambda values: len(values) >= 1, seconds=60, return_last_on_timeout=True)
|
||||
assert spans, [(span.get("name"), span.get("traceId")) for span in rig.sink.all()]
|
||||
return spans[0]
|
||||
|
||||
|
||||
def _span_for(rig: Rig, name: str, model: str) -> dict[str, JsonValue]:
|
||||
spans: Final = eventually(lambda: rig.sink.named(name, model), lambda values: len(values) >= 1, seconds=260)
|
||||
return spans[0]
|
||||
|
||||
|
||||
def _auth_exception_span(rig: Rig, trace_id: str) -> dict[str, JsonValue]:
|
||||
def found() -> tuple[dict[str, JsonValue], ...]:
|
||||
return tuple(
|
||||
span
|
||||
for span in rig.sink.all()
|
||||
if span.get("traceId") == trace_id
|
||||
and str(span.get("name", "")).startswith("auth")
|
||||
and _exception_events(span)
|
||||
)
|
||||
|
||||
return eventually(found, lambda values: len(values) >= 1, seconds=260)[0]
|
||||
|
||||
|
||||
# C6: OTEL v1 failure span redaction under global on
|
||||
@pytest.mark.timeout(320)
|
||||
def test_c6_v1_provider_error_spans_redacted(rig_v1_on: Rig) -> None:
|
||||
secret: Final = _secret()
|
||||
with rig_v1_on.proxy.scenario() as scenario:
|
||||
model: Final = scenario.model(api_base=rig_v1_on.provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
response: Final = _fail(rig_v1_on, model, secret)
|
||||
assert response.status_code == 400, response.text
|
||||
call_id: Final = response.headers["x-litellm-call-id"]
|
||||
assert any(secret.encode() in request.body for request in rig_v1_on.provider.drain())
|
||||
server: Final = _server_span(rig_v1_on, call_id)
|
||||
assert _error_attribute(server) == "redacted-by-litellm", _span_attributes(server)
|
||||
assert secret not in _exception_texts(server), _exception_texts(server)[:600]
|
||||
assert secret not in json.dumps(server), json.dumps(server)[:600]
|
||||
request_span: Final = _llm_span(rig_v1_on, call_id)
|
||||
assert _error_attribute(request_span) == "redacted-by-litellm", _span_attributes(request_span)
|
||||
assert secret not in _exception_texts(request_span), _exception_texts(request_span)[:600]
|
||||
assert secret not in json.dumps(request_span), json.dumps(request_span)[:600]
|
||||
|
||||
|
||||
# C7: OTEL v2 request + server spans redacted under global on
|
||||
@pytest.mark.timeout(320)
|
||||
def test_c7_v2_provider_error_spans_redacted(rig_v2_on: Rig) -> None:
|
||||
secret: Final = _secret()
|
||||
with rig_v2_on.proxy.scenario() as scenario:
|
||||
model: Final = scenario.model(api_base=rig_v2_on.provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
response: Final = _fail(rig_v2_on, model, secret)
|
||||
assert response.status_code == 400, response.text
|
||||
call_id: Final = response.headers["x-litellm-call-id"]
|
||||
server: Final = _server_span(rig_v2_on, call_id)
|
||||
assert _error_attribute(server) == "redacted-by-litellm", _span_attributes(server)
|
||||
assert secret not in _exception_texts(server), _exception_texts(server)[:600]
|
||||
assert secret not in json.dumps(server), json.dumps(server)[:600]
|
||||
|
||||
|
||||
# C8: v2 server span restamp keeps request opt-in under global off
|
||||
@pytest.mark.timeout(320)
|
||||
def test_c8_v2_header_opt_in_restamped_server_span_redacted(rig_v2_off: Rig) -> None:
|
||||
secret: Final = _secret()
|
||||
with rig_v2_off.proxy.scenario() as scenario:
|
||||
model: Final = scenario.model(api_base=rig_v2_off.provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
response: Final = _fail(rig_v2_off, model, secret, headers={"x-litellm-enable-message-redaction": "true"})
|
||||
assert response.status_code == 400, response.text
|
||||
call_id: Final = response.headers["x-litellm-call-id"]
|
||||
server: Final = _server_span(rig_v2_off, call_id)
|
||||
assert _error_attribute(server) == "redacted-by-litellm", _span_attributes(server)
|
||||
assert secret not in _exception_texts(server), _exception_texts(server)[:600]
|
||||
|
||||
|
||||
# C9: v2 server span keeps permitted opt-out raw under global on
|
||||
@pytest.mark.timeout(320)
|
||||
def test_c9_v2_permitted_opt_out_keeps_server_span_raw(rig_v2_on: Rig) -> None:
|
||||
secret: Final = _secret()
|
||||
with rig_v2_on.proxy.scenario() as scenario:
|
||||
model: Final = scenario.model(api_base=rig_v2_on.provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
key: Final = scenario.key(models=[model], metadata={"allow_client_message_redaction_opt_out": True})
|
||||
response: Final = _fail(
|
||||
rig_v2_on, model, secret, key=key, headers={"litellm-disable-message-redaction": "true"}
|
||||
)
|
||||
assert response.status_code == 400, response.text
|
||||
call_id: Final = response.headers["x-litellm-call-id"]
|
||||
server: Final = _server_span(rig_v2_on, call_id)
|
||||
assert secret in _error_attribute(server), _span_attributes(server)
|
||||
|
||||
|
||||
# C10: v2 auth phase span honors request opt-in under global off
|
||||
@pytest.mark.timeout(320)
|
||||
def test_c10_v2_auth_span_honors_header_opt_in(rig_v2_off: Rig) -> None:
|
||||
with rig_v2_off.proxy.scenario() as scenario:
|
||||
allowed: Final = scenario.model(api_base=rig_v2_off.provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
denied: Final = scenario.model(api_base=rig_v2_off.provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
key: Final = scenario.key(models=[allowed])
|
||||
trace_id: Final = uuid.uuid4().hex
|
||||
response: Final = rig_v2_off.proxy.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": denied, "messages": [{"role": "user", "content": "hi"}]},
|
||||
key=key,
|
||||
headers={
|
||||
"x-litellm-enable-message-redaction": "true",
|
||||
"traceparent": f"00-{trace_id}-{uuid.uuid4().hex[:16]}-01",
|
||||
},
|
||||
)
|
||||
assert response.status_code in (400, 401, 403, 404), response.text
|
||||
auth: Final = _auth_exception_span(rig_v2_off, trace_id)
|
||||
assert allowed not in _exception_texts(auth), _exception_texts(auth)[:600]
|
||||
assert denied not in _exception_texts(auth), _exception_texts(auth)[:600]
|
||||
assert "redacted-by-litellm" in _exception_texts(auth), _exception_texts(auth)[:600]
|
||||
|
||||
|
||||
# C11: v2 auth phase span redacts under global on even with unpermitted disable header
|
||||
@pytest.mark.timeout(320)
|
||||
def test_c11_v2_auth_span_redacts_under_global_on(rig_v2_on: Rig) -> None:
|
||||
with rig_v2_on.proxy.scenario() as scenario:
|
||||
allowed: Final = scenario.model(api_base=rig_v2_on.provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
denied: Final = scenario.model(api_base=rig_v2_on.provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
key: Final = scenario.key(models=[allowed])
|
||||
trace_id: Final = uuid.uuid4().hex
|
||||
response: Final = rig_v2_on.proxy.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": denied, "messages": [{"role": "user", "content": "hi"}]},
|
||||
key=key,
|
||||
headers={
|
||||
"litellm-disable-message-redaction": "true",
|
||||
"traceparent": f"00-{trace_id}-{uuid.uuid4().hex[:16]}-01",
|
||||
},
|
||||
)
|
||||
assert response.status_code in (400, 401, 403, 404), response.text
|
||||
auth: Final = _auth_exception_span(rig_v2_on, trace_id)
|
||||
assert allowed not in _exception_texts(auth), _exception_texts(auth)[:600]
|
||||
assert denied not in _exception_texts(auth), _exception_texts(auth)[:600]
|
||||
assert "redacted-by-litellm" in _exception_texts(auth), _exception_texts(auth)[:600]
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import json
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import httpx
|
||||
|
|
@ -6,6 +7,7 @@ import pytest
|
|||
from httpx import Request, Response
|
||||
from pydantic import BaseModel, computed_field
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.datadog.datadog import DataDogLogger
|
||||
from litellm.llms.custom_httpx.http_handler import MaskedHTTPStatusError
|
||||
from litellm.types.integrations.datadog import (
|
||||
|
|
@ -558,3 +560,44 @@ async def test_raised_intake_error_preserves_datadog_requeue_behavior(datadog_en
|
|||
await logger.async_send_batch()
|
||||
|
||||
assert [event["message"] for event in logger.log_queue] == ['{"event": 0}', '{"event": 1}']
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_hook_redacts_exception_payload_when_redaction_on(datadog_env, monkeypatch):
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", True)
|
||||
with patch("asyncio.create_task"):
|
||||
logger = DataDogLogger()
|
||||
|
||||
secret = "secret-prompt-marker"
|
||||
await logger.async_post_call_failure_hook(
|
||||
request_data={"metadata": {}},
|
||||
original_exception=litellm.BadRequestError(
|
||||
message=f"Unsupported content: {secret}", model="gpt-4o", llm_provider="openai"
|
||||
),
|
||||
user_api_key_dict=type("UserKey", (), {})(),
|
||||
traceback_str=f"Traceback ... {secret} ...",
|
||||
)
|
||||
|
||||
message = json.loads(logger.log_queue[0]["message"])
|
||||
assert secret not in json.dumps(message)
|
||||
assert message["exception"] == "redacted-by-litellm"
|
||||
assert message["traceback"] == "redacted-by-litellm"
|
||||
assert message["error_class"] == "BadRequestError"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_hook_keeps_exception_payload_when_redaction_off(datadog_env, monkeypatch):
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", False)
|
||||
with patch("asyncio.create_task"):
|
||||
logger = DataDogLogger()
|
||||
|
||||
secret = "secret-prompt-marker"
|
||||
await logger.async_post_call_failure_hook(
|
||||
request_data={"metadata": {}},
|
||||
original_exception=Exception(f"boom {secret}"),
|
||||
user_api_key_dict=type("UserKey", (), {})(),
|
||||
traceback_str="trace",
|
||||
)
|
||||
|
||||
message = json.loads(logger.log_queue[0]["message"])
|
||||
assert message["exception"] == f"boom {secret}"
|
||||
|
|
|
|||
|
|
@ -3453,6 +3453,175 @@ def test_provisional_close_then_payload_close_does_not_duplicate():
|
|||
assert len(llm_spans) == 1
|
||||
|
||||
|
||||
def test_async_post_call_failure_hook_redacts_error_text_when_gated(monkeypatch):
|
||||
"""With message redaction on, the proxy-level failure span must not carry the
|
||||
prompt through error.message / the exception event, while error.type and the
|
||||
provider error code stay intact."""
|
||||
import litellm
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", True)
|
||||
logger, exporter = _logger()
|
||||
server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
|
||||
set_request_root_span(server)
|
||||
secret = "secret-prompt-marker"
|
||||
result = asyncio.run(
|
||||
logger.async_post_call_failure_hook(
|
||||
request_data={"metadata": {}},
|
||||
original_exception=_proxy_exc(f"Unsupported content: {secret}", 400),
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
traceback_str=f"Traceback ... {secret} ...",
|
||||
)
|
||||
)
|
||||
server.end()
|
||||
assert result is None
|
||||
(span,) = exporter.get_finished_spans()
|
||||
assert secret not in str(dict(span.attributes or {}))
|
||||
assert secret not in str([dict(e.attributes or {}) for e in span.events])
|
||||
assert span.attributes["error.message"] == "redacted-by-litellm"
|
||||
assert span.attributes["error.type"] == "ProxyException"
|
||||
assert span.attributes["litellm.provider.error.code"] == "400"
|
||||
assert span.attributes["litellm.provider.error.stack_trace"] == "redacted-by-litellm"
|
||||
|
||||
|
||||
def test_async_post_call_failure_hook_keeps_error_text_when_not_gated():
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
logger, exporter = _logger()
|
||||
server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
|
||||
set_request_root_span(server)
|
||||
secret = "secret-prompt-marker"
|
||||
asyncio.run(
|
||||
logger.async_post_call_failure_hook(
|
||||
request_data={"metadata": {}},
|
||||
original_exception=_proxy_exc(f"Unsupported content: {secret}", 400),
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
traceback_str="trace",
|
||||
)
|
||||
)
|
||||
server.end()
|
||||
(span,) = exporter.get_finished_spans()
|
||||
assert secret in span.attributes["error.message"]
|
||||
|
||||
|
||||
def test_record_error_attributes_on_span_preserves_request_opt_in_redaction(monkeypatch):
|
||||
"""Global flag off but the request opts in via header: the failure hook
|
||||
stamps redacted values on the SERVER span; the exception-handler restamp
|
||||
must keep them instead of re-writing raw error text from the global probe."""
|
||||
import litellm
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", False)
|
||||
logger, exporter = _logger()
|
||||
server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
|
||||
set_request_root_span(server)
|
||||
secret = "secret-prompt-marker"
|
||||
exc = _proxy_exc(f"Unsupported content: {secret}", 400)
|
||||
asyncio.run(
|
||||
logger.async_post_call_failure_hook(
|
||||
request_data={"metadata": {"headers": {"x-litellm-enable-message-redaction": "true"}}},
|
||||
original_exception=exc,
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
traceback_str=f"Traceback ... {secret} ...",
|
||||
)
|
||||
)
|
||||
logger.record_error_attributes_on_span(server, exc, 400)
|
||||
server.end()
|
||||
(span,) = exporter.get_finished_spans()
|
||||
assert secret not in str(dict(span.attributes or {}))
|
||||
assert secret not in str([dict(e.attributes or {}) for e in span.events])
|
||||
assert secret not in str(span.status.description or "")
|
||||
assert span.attributes["error.message"] == "redacted-by-litellm"
|
||||
assert span.attributes["litellm.provider.error.code"] == "400"
|
||||
|
||||
|
||||
def test_record_error_attributes_on_span_preserves_opt_out_raw_message(monkeypatch):
|
||||
"""Global flag on but the request opts out via header: the failure hook
|
||||
stamps the raw error text; the restamp must not overwrite it with the
|
||||
redaction marker."""
|
||||
import litellm
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", True)
|
||||
logger, exporter = _logger()
|
||||
server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
|
||||
set_request_root_span(server)
|
||||
secret = "secret-prompt-marker"
|
||||
exc = _proxy_exc(f"Unsupported content: {secret}", 400)
|
||||
asyncio.run(
|
||||
logger.async_post_call_failure_hook(
|
||||
request_data={"metadata": {"headers": {"litellm-disable-message-redaction": "true"}}},
|
||||
original_exception=exc,
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
traceback_str=f"Traceback ... {secret} ...",
|
||||
)
|
||||
)
|
||||
logger.record_error_attributes_on_span(server, exc, 400)
|
||||
server.end()
|
||||
(span,) = exporter.get_finished_spans()
|
||||
assert secret in span.attributes["error.message"]
|
||||
assert span.attributes["litellm.provider.error.code"] == "400"
|
||||
|
||||
|
||||
def test_start_phase_span_does_not_record_raw_exception_when_gated(monkeypatch):
|
||||
"""With message redaction on, an exception raised inside a phase span must
|
||||
not leak through use_span's automatic exception recording: exactly one
|
||||
exception event lands, carrying only the redaction marker."""
|
||||
import litellm
|
||||
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", True)
|
||||
logger, exporter = _logger()
|
||||
secret = "secret-prompt-marker"
|
||||
with pytest.raises(RuntimeError):
|
||||
with logger.start_phase_span("auth"):
|
||||
raise RuntimeError(f"auth exploded: {secret}")
|
||||
(span,) = exporter.get_finished_spans()
|
||||
assert secret not in str(dict(span.attributes or {}))
|
||||
assert secret not in str([dict(e.attributes or {}) for e in span.events])
|
||||
assert secret not in str(span.status.description or "")
|
||||
exception_events = [e for e in span.events if e.name == "exception"]
|
||||
assert len(exception_events) == 1
|
||||
assert (exception_events[0].attributes or {}).get("exception.message") == "redacted-by-litellm"
|
||||
|
||||
|
||||
def test_start_phase_span_redact_content_kwarg_redacts_exception(monkeypatch):
|
||||
"""A caller-provided opt-in (e.g. an enable header seen at auth time, where
|
||||
the key's opt-out permission is not known yet) must redact the phase-span
|
||||
exception even when the global flag is off."""
|
||||
import litellm
|
||||
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", False)
|
||||
logger, exporter = _logger()
|
||||
secret = "secret-prompt-marker"
|
||||
with pytest.raises(RuntimeError):
|
||||
with logger.start_phase_span("auth", redact_content=True):
|
||||
raise RuntimeError(f"auth exploded: {secret}")
|
||||
(span,) = exporter.get_finished_spans()
|
||||
assert secret not in str(dict(span.attributes or {}))
|
||||
assert secret not in str([dict(e.attributes or {}) for e in span.events])
|
||||
assert secret not in str(span.status.description or "")
|
||||
exception_events = [e for e in span.events if e.name == "exception"]
|
||||
assert len(exception_events) == 1
|
||||
assert (exception_events[0].attributes or {}).get("exception.message") == "redacted-by-litellm"
|
||||
|
||||
|
||||
def test_start_phase_span_without_opt_in_keeps_raw_exception(monkeypatch):
|
||||
"""The default ``redact_content=False`` must leave use_span's own raw
|
||||
exception recording untouched when the global flag is off."""
|
||||
import litellm
|
||||
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", False)
|
||||
logger, exporter = _logger()
|
||||
secret = "secret-prompt-marker"
|
||||
with pytest.raises(RuntimeError):
|
||||
with logger.start_phase_span("auth"):
|
||||
raise RuntimeError(f"auth exploded: {secret}")
|
||||
(span,) = exporter.get_finished_spans()
|
||||
exception_events = [e for e in span.events if e.name == "exception"]
|
||||
assert len(exception_events) == 1
|
||||
assert secret in str(dict(exception_events[0].attributes or {}))
|
||||
|
||||
|
||||
def test_pipeline_op_count_lands_as_an_int_on_both_metadata_keys():
|
||||
"""A ``RedisBatch`` flush reports ``call_type=request_redis_batch`` with
|
||||
``event_metadata={"op_count": N}``; the span is ``redis.pipeline`` (no ``[N]`` in the
|
||||
|
|
|
|||
|
|
@ -71,6 +71,36 @@ def test_wrappers_no_op_when_runtime_absent(monkeypatch):
|
|||
assert runtime.seed_request_identity({"token": "sk-x"}, model="gpt-4o") is None
|
||||
|
||||
|
||||
def test_phase_span_forwards_redact_content_to_registered_logger(monkeypatch):
|
||||
"""``redact_content`` must reach the v2 logger through the runtime shim: an
|
||||
exception raised inside the forwarded span carries only the redaction marker,
|
||||
even with the global flag off."""
|
||||
pytest.importorskip("opentelemetry")
|
||||
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
|
||||
import litellm
|
||||
import litellm.integrations.otel.logger as otel_logger
|
||||
from litellm.integrations.otel import OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2
|
||||
from litellm.integrations.otel.plumbing import providers
|
||||
|
||||
cfg = OpenTelemetryV2Config(exporter="in_memory", legacy_compat=True, baggage_team_metadata_keys=[])
|
||||
exporter = InMemorySpanExporter()
|
||||
logger = OpenTelemetryV2(config=cfg, tracer_provider=providers.build_tracer_provider(cfg, exporter=exporter))
|
||||
monkeypatch.setattr(otel_logger, "_registered_v2_logger", lambda: logger)
|
||||
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", False)
|
||||
secret = "secret-prompt-marker"
|
||||
with pytest.raises(RuntimeError):
|
||||
with runtime.phase_span("auth", redact_content=True):
|
||||
raise RuntimeError(f"auth exploded: {secret}")
|
||||
(span,) = exporter.get_finished_spans()
|
||||
assert secret not in str(dict(span.attributes or {}))
|
||||
assert secret not in str([dict(e.attributes or {}) for e in span.events])
|
||||
assert secret not in str(span.status.description or "")
|
||||
|
||||
|
||||
def test_phase_event_no_ops_when_runtime_absent(monkeypatch):
|
||||
monkeypatch.setattr(runtime, "_otel_runtime", lambda: None)
|
||||
|
||||
|
|
|
|||
|
|
@ -268,3 +268,79 @@ def test_mlflow_end_span_or_trace_works_with_mlflow_2x_client():
|
|||
span=child_span, outputs="out", end_time_ns=1, status="OK"
|
||||
)
|
||||
assert client.ended_spans == [("req-2", "span-2")]
|
||||
|
||||
def _failure_modules():
|
||||
modules = _mock_mlflow_modules()
|
||||
|
||||
class RecordingSpanEvent:
|
||||
calls = []
|
||||
|
||||
def __init__(self, name, attributes):
|
||||
self.name = name
|
||||
self.attributes = attributes
|
||||
|
||||
@classmethod
|
||||
def from_exception(cls, exception):
|
||||
cls.calls.append(exception)
|
||||
return cls("exception-from-exception", {"exception.message": str(exception)})
|
||||
|
||||
RecordingSpanEvent.calls = []
|
||||
modules["mlflow.entities"].SpanEvent = RecordingSpanEvent
|
||||
modules["_span_event_cls"] = RecordingSpanEvent
|
||||
return modules
|
||||
|
||||
|
||||
def test_mlflow_failure_event_redacts_exception_when_gated(monkeypatch):
|
||||
import litellm
|
||||
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", True)
|
||||
modules = _failure_modules()
|
||||
with patch.dict("sys.modules", modules):
|
||||
from litellm.integrations.mlflow import MlflowLogger
|
||||
|
||||
mlflow_logger = MlflowLogger()
|
||||
span = MagicMock()
|
||||
mlflow_logger._start_span_or_trace = MagicMock(return_value=span)
|
||||
mlflow_logger._end_span_or_trace = MagicMock()
|
||||
mlflow_logger._extract_and_set_chat_attributes = MagicMock()
|
||||
|
||||
secret = "secret-prompt-marker"
|
||||
mlflow_logger._handle_failure(
|
||||
kwargs={"litellm_call_id": "x", "exception": Exception(f"boom {secret}")},
|
||||
response_obj=None,
|
||||
start_time=datetime.utcnow(),
|
||||
end_time=datetime.utcnow(),
|
||||
)
|
||||
|
||||
event = span.add_event.call_args.args[0]
|
||||
assert event.attributes == {
|
||||
"exception.type": "Exception",
|
||||
"exception.message": "redacted-by-litellm",
|
||||
"exception.stacktrace": "redacted-by-litellm",
|
||||
}
|
||||
assert modules["_span_event_cls"].calls == []
|
||||
|
||||
|
||||
def test_mlflow_failure_event_uses_from_exception_when_not_gated(monkeypatch):
|
||||
import litellm
|
||||
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", False)
|
||||
modules = _failure_modules()
|
||||
with patch.dict("sys.modules", modules):
|
||||
from litellm.integrations.mlflow import MlflowLogger
|
||||
|
||||
mlflow_logger = MlflowLogger()
|
||||
span = MagicMock()
|
||||
mlflow_logger._start_span_or_trace = MagicMock(return_value=span)
|
||||
mlflow_logger._end_span_or_trace = MagicMock()
|
||||
mlflow_logger._extract_and_set_chat_attributes = MagicMock()
|
||||
|
||||
exc = Exception("boom secret-prompt-marker")
|
||||
mlflow_logger._handle_failure(
|
||||
kwargs={"litellm_call_id": "x", "exception": exc},
|
||||
response_obj=None,
|
||||
start_time=datetime.utcnow(),
|
||||
end_time=datetime.utcnow(),
|
||||
)
|
||||
|
||||
assert modules["_span_event_cls"].calls == [exc]
|
||||
|
|
|
|||
|
|
@ -7134,6 +7134,121 @@ class TestOpenTelemetryNonInferenceUsage(unittest.TestCase):
|
|||
)
|
||||
|
||||
|
||||
SECRET_PROMPT = "secret-prompt-marker"
|
||||
|
||||
|
||||
class TestOpenTelemetryFailureHookRedaction(unittest.TestCase):
|
||||
def _run_hook(self, request_data, exception, redact):
|
||||
with patch.object(litellm, "turn_off_message_logging", redact):
|
||||
exporter = InMemorySpanExporter()
|
||||
provider = TracerProvider()
|
||||
provider.add_span_processor(SimpleSpanProcessor(exporter))
|
||||
tracer = provider.get_tracer(__name__)
|
||||
|
||||
otel = OpenTelemetry()
|
||||
otel.tracer = tracer
|
||||
server_span = tracer.start_span("Received Proxy Server Request")
|
||||
|
||||
user_api_key_dict = MagicMock()
|
||||
user_api_key_dict.parent_otel_span = server_span
|
||||
|
||||
asyncio.run(
|
||||
otel.async_post_call_failure_hook(
|
||||
request_data=request_data,
|
||||
original_exception=exception,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
traceback_str=f"Traceback ... {SECRET_PROMPT} ...",
|
||||
)
|
||||
)
|
||||
|
||||
finished = {s.name: s for s in exporter.get_finished_spans()}
|
||||
return finished
|
||||
|
||||
def test_failure_hook_redacts_error_text_when_gated(self):
|
||||
secret = SECRET_PROMPT
|
||||
finished = self._run_hook(
|
||||
{"metadata": {}},
|
||||
litellm.BadRequestError(message=f"Unsupported content: {secret}", model="gpt-4o", llm_provider="openai"),
|
||||
redact=True,
|
||||
)
|
||||
server = finished["Received Proxy Server Request"]
|
||||
child = finished["Failed Proxy Server Request"]
|
||||
for span in (server, child):
|
||||
assert secret not in str(dict(span.attributes or {}))
|
||||
for event in span.events:
|
||||
assert secret not in str(dict(event.attributes or {}))
|
||||
assert server.attributes["error.message"] == "redacted-by-litellm"
|
||||
assert server.attributes["error.type"] == "BadRequestError"
|
||||
assert child.attributes["exception"] == "redacted-by-litellm"
|
||||
|
||||
def test_failure_hook_keeps_error_text_when_not_gated(self):
|
||||
secret = SECRET_PROMPT
|
||||
finished = self._run_hook(
|
||||
{"metadata": {}},
|
||||
litellm.BadRequestError(message=f"Unsupported content: {secret}", model="gpt-4o", llm_provider="openai"),
|
||||
redact=False,
|
||||
)
|
||||
child = finished["Failed Proxy Server Request"]
|
||||
assert secret in child.attributes["exception"]
|
||||
|
||||
def test_record_error_attributes_on_span_redacts_via_global_gate(self):
|
||||
with patch.object(litellm, "turn_off_message_logging", True):
|
||||
otel = OpenTelemetry()
|
||||
span = MagicMock()
|
||||
otel.record_error_attributes_on_span(
|
||||
span=span,
|
||||
exception=litellm.BadRequestError(
|
||||
message=f"Unsupported content: {SECRET_PROMPT}", model="gpt-4o", llm_provider="openai"
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
stamped = {call.args[0]: call.args[1] for call in span.set_attribute.call_args_list}
|
||||
assert stamped.get("error.message") == "redacted-by-litellm"
|
||||
assert stamped.get("error.type") == "BadRequestError"
|
||||
|
||||
def _restamped_span(self, stamped_message, redact, exception):
|
||||
with patch.object(litellm, "turn_off_message_logging", redact):
|
||||
exporter = InMemorySpanExporter()
|
||||
provider = TracerProvider()
|
||||
provider.add_span_processor(SimpleSpanProcessor(exporter))
|
||||
otel = OpenTelemetry()
|
||||
otel.tracer = provider.get_tracer(__name__)
|
||||
span = otel.tracer.start_span("Received Proxy Server Request")
|
||||
otel.safe_set_attribute(span=span, key="error.message", value=stamped_message)
|
||||
otel.record_error_attributes_on_span(span=span, exception=exception, status_code=400)
|
||||
span.end()
|
||||
return exporter.get_finished_spans()[0]
|
||||
|
||||
def test_record_error_attributes_on_span_preserves_request_opt_in_redaction(self):
|
||||
"""Global flag off but the request opted in: the failure hook already
|
||||
stamped the redaction marker on the SERVER span; the exception-handler
|
||||
restamp must keep it instead of writing raw error text."""
|
||||
span = self._restamped_span(
|
||||
"redacted-by-litellm",
|
||||
redact=False,
|
||||
exception=litellm.BadRequestError(
|
||||
message=f"Unsupported content: {SECRET_PROMPT}", model="gpt-4o", llm_provider="openai"
|
||||
),
|
||||
)
|
||||
assert span.attributes["error.message"] == "redacted-by-litellm"
|
||||
assert span.attributes["error.code"] == "400"
|
||||
assert SECRET_PROMPT not in str(dict(span.attributes or {}))
|
||||
|
||||
def test_record_error_attributes_on_span_preserves_opt_out_raw_message(self):
|
||||
"""Global flag on but the request opted out: the failure hook stamped
|
||||
the raw error text; the restamp must not overwrite it with the
|
||||
redaction marker."""
|
||||
span = self._restamped_span(
|
||||
f"Unsupported content: {SECRET_PROMPT}",
|
||||
redact=True,
|
||||
exception=litellm.BadRequestError(
|
||||
message=f"Unsupported content: {SECRET_PROMPT}", model="gpt-4o", llm_provider="openai"
|
||||
),
|
||||
)
|
||||
assert SECRET_PROMPT in span.attributes["error.message"]
|
||||
assert span.attributes["error.code"] == "400"
|
||||
|
||||
|
||||
def _raw_response_span_attributes(original_response: str) -> dict[str, object]:
|
||||
span_exporter = InMemorySpanExporter()
|
||||
tracer_provider = TracerProvider()
|
||||
|
|
|
|||
|
|
@ -11538,6 +11538,166 @@ def test_uncached_price_keeps_an_earlier_price_failure(pr44187_priced_model: str
|
|||
assert logging_obj.model_call_details["response_cost_failure_debug_information"] == earlier_failure
|
||||
|
||||
|
||||
class _CapturingFailureLogger(CustomLogger):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.sync_kwargs = None
|
||||
self.async_kwargs = None
|
||||
|
||||
def log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
self.sync_kwargs = kwargs
|
||||
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
self.async_kwargs = kwargs
|
||||
|
||||
|
||||
def _failure_logging_obj(secret):
|
||||
return LitellmLogging(
|
||||
model="gpt-4o",
|
||||
messages=[{"role": "user", "content": secret}],
|
||||
stream=False,
|
||||
call_type="completion",
|
||||
start_time=time.time(),
|
||||
litellm_call_id="failure-redaction-test",
|
||||
function_id="failure-redaction-test",
|
||||
)
|
||||
|
||||
|
||||
def _assert_no_secret_leaks(kwargs, secret):
|
||||
payload = kwargs.get("standard_logging_object") or {}
|
||||
assert secret not in json.dumps(payload, default=str)
|
||||
assert secret not in str(kwargs.get("traceback_exception"))
|
||||
error_information = payload.get("error_information") or {}
|
||||
assert error_information.get("error_class") == "BadRequestError"
|
||||
assert error_information.get("error_code") == "400"
|
||||
assert error_information.get("llm_provider") == "openai"
|
||||
|
||||
|
||||
def test_sync_failure_handler_redacts_error_text_for_custom_logger(monkeypatch):
|
||||
secret = "secret-prompt-marker"
|
||||
capture = _CapturingFailureLogger()
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", True)
|
||||
monkeypatch.setattr(litellm, "failure_callback", [capture])
|
||||
monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", None)
|
||||
logging_obj = _failure_logging_obj(secret)
|
||||
|
||||
logging_obj.failure_handler(
|
||||
litellm.BadRequestError(message=f"Unsupported content: {secret}", model="gpt-4o", llm_provider="openai"),
|
||||
f"Traceback ... {secret} ...",
|
||||
)
|
||||
|
||||
assert capture.sync_kwargs is not None
|
||||
_assert_no_secret_leaks(capture.sync_kwargs, secret)
|
||||
|
||||
|
||||
def test_sync_failure_handler_honours_excluded_fields(monkeypatch):
|
||||
capture = _CapturingFailureLogger()
|
||||
monkeypatch.setattr(litellm, "failure_callback", [capture])
|
||||
monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", ["hidden_params"])
|
||||
logging_obj = _failure_logging_obj("anything")
|
||||
|
||||
logging_obj.failure_handler(litellm.BadRequestError(message="denied", model="gpt-4o", llm_provider="openai"), "tb")
|
||||
|
||||
payload = capture.sync_kwargs.get("standard_logging_object") or {}
|
||||
assert "hidden_params" not in payload
|
||||
|
||||
|
||||
def test_sync_failure_handler_leaves_error_text_alone_when_redaction_off(monkeypatch):
|
||||
secret = "secret-prompt-marker"
|
||||
capture = _CapturingFailureLogger()
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", False)
|
||||
monkeypatch.setattr(litellm, "failure_callback", [capture])
|
||||
monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", None)
|
||||
logging_obj = _failure_logging_obj(secret)
|
||||
|
||||
logging_obj.failure_handler(
|
||||
litellm.BadRequestError(message=f"Unsupported content: {secret}", model="gpt-4o", llm_provider="openai"),
|
||||
f"Traceback ... {secret} ...",
|
||||
)
|
||||
|
||||
payload = capture.sync_kwargs.get("standard_logging_object") or {}
|
||||
assert secret in payload.get("error_information", {}).get("error_message", "")
|
||||
assert secret in str(payload.get("error_str"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_failure_handler_redacts_error_text_for_custom_logger(monkeypatch):
|
||||
secret = "secret-prompt-marker"
|
||||
capture = _CapturingFailureLogger()
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", True)
|
||||
monkeypatch.setattr(litellm, "_async_failure_callback", [capture])
|
||||
monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", None)
|
||||
logging_obj = _failure_logging_obj(secret)
|
||||
|
||||
await logging_obj.async_failure_handler(
|
||||
litellm.BadRequestError(message=f"Unsupported content: {secret}", model="gpt-4o", llm_provider="openai"),
|
||||
f"Traceback ... {secret} ...",
|
||||
)
|
||||
|
||||
assert capture.async_kwargs is not None
|
||||
_assert_no_secret_leaks(capture.async_kwargs, secret)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_failure_handler_honours_excluded_fields(monkeypatch):
|
||||
capture = _CapturingFailureLogger()
|
||||
monkeypatch.setattr(litellm, "_async_failure_callback", [capture])
|
||||
monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", ["hidden_params"])
|
||||
logging_obj = _failure_logging_obj("anything")
|
||||
|
||||
await logging_obj.async_failure_handler(
|
||||
litellm.BadRequestError(message="denied", model="gpt-4o", llm_provider="openai"), "tb"
|
||||
)
|
||||
|
||||
payload = capture.async_kwargs.get("standard_logging_object") or {}
|
||||
assert "hidden_params" not in payload
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("headers", [None, "x", ["a"]], ids=["none", "str", "list"])
|
||||
@pytest.mark.parametrize("redaction_on", [True, False], ids=["redaction_on", "redaction_off"])
|
||||
async def test_async_failure_handler_logs_requests_with_non_mapping_headers(monkeypatch, headers, redaction_on):
|
||||
secret = "secret-prompt-marker"
|
||||
capture = _CapturingFailureLogger()
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", redaction_on)
|
||||
monkeypatch.setattr(litellm, "_async_failure_callback", [capture])
|
||||
monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", None)
|
||||
logging_obj = _failure_logging_obj(secret)
|
||||
logging_obj.model_call_details["litellm_params"] = {"metadata": {"headers": headers}}
|
||||
|
||||
await logging_obj.async_failure_handler(
|
||||
litellm.BadRequestError(message=f"Unsupported content: {secret}", model="gpt-4o", llm_provider="openai"),
|
||||
f"Traceback ... {secret} ...",
|
||||
)
|
||||
|
||||
assert capture.async_kwargs is not None
|
||||
logged = str(capture.async_kwargs.get("traceback_exception"))
|
||||
assert (secret in logged) is not redaction_on, logged
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_per_callback_turn_off_redacts_error_fields_even_for_self_redacting_logger(monkeypatch):
|
||||
secret = "secret-prompt-marker"
|
||||
|
||||
class SelfRedactingLogger(_CapturingFailureLogger):
|
||||
def redacts_messages_itself(self):
|
||||
return True
|
||||
|
||||
capture = SelfRedactingLogger(turn_off_message_logging=True)
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", False)
|
||||
monkeypatch.setattr(litellm, "_async_failure_callback", [capture])
|
||||
monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", None)
|
||||
logging_obj = _failure_logging_obj(secret)
|
||||
|
||||
await logging_obj.async_failure_handler(
|
||||
litellm.BadRequestError(message=f"Unsupported content: {secret}", model="gpt-4o", llm_provider="openai"),
|
||||
f"Traceback ... {secret} ...",
|
||||
)
|
||||
|
||||
assert capture.async_kwargs is not None
|
||||
_assert_no_secret_leaks(capture.async_kwargs, secret)
|
||||
|
||||
|
||||
def test_enterprise_alerting_loggers_resolves_real_classes():
|
||||
from litellm_enterprise.enterprise_callbacks.pagerduty.pagerduty import PagerDutyAlerting
|
||||
from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import ResendEmailLogger
|
||||
|
|
|
|||
|
|
@ -1269,6 +1269,218 @@ def test_perform_redaction_drops_the_served_output_texts_from_the_callback_kwarg
|
|||
assert SERVED_OUTPUT_TEXTS_KEY not in details
|
||||
|
||||
|
||||
def _error_information(**overrides):
|
||||
info = {
|
||||
"error_code": "400",
|
||||
"error_class": "BadRequestError",
|
||||
"llm_provider": "openai",
|
||||
"traceback": "Traceback ... secret-prompt-marker ...",
|
||||
"error_message": "Unsupported content: secret-prompt-marker",
|
||||
"error_rate_limit_category": None,
|
||||
"normalized_error": None,
|
||||
}
|
||||
info.update(overrides)
|
||||
return info
|
||||
|
||||
|
||||
class TestRedactErrorInformation:
|
||||
def test_replaces_message_and_traceback_keeps_everything_else(self):
|
||||
from litellm.litellm_core_utils.redact_messages import redact_error_information
|
||||
|
||||
info = _error_information()
|
||||
redacted = redact_error_information(info)
|
||||
assert redacted == {
|
||||
"error_code": "400",
|
||||
"error_class": "BadRequestError",
|
||||
"llm_provider": "openai",
|
||||
"traceback": "redacted-by-litellm",
|
||||
"error_message": "redacted-by-litellm",
|
||||
"error_rate_limit_category": None,
|
||||
"normalized_error": None,
|
||||
}
|
||||
|
||||
def test_empty_fields_stay_empty(self):
|
||||
from litellm.litellm_core_utils.redact_messages import redact_error_information
|
||||
|
||||
redacted = redact_error_information(_error_information(traceback="", error_message=None))
|
||||
assert redacted["traceback"] == ""
|
||||
assert redacted["error_message"] is None
|
||||
assert "secret-prompt-marker" not in str(redacted)
|
||||
|
||||
def test_input_not_mutated(self):
|
||||
from litellm.litellm_core_utils.redact_messages import redact_error_information
|
||||
|
||||
info = _error_information()
|
||||
redact_error_information(info)
|
||||
assert info["error_message"] == "Unsupported content: secret-prompt-marker"
|
||||
assert info["traceback"] == "Traceback ... secret-prompt-marker ..."
|
||||
|
||||
|
||||
class TestFailureRedactionOnStandardLoggingObject:
|
||||
def test_redacted_standard_logging_payload_covers_error_fields(self):
|
||||
payload = {
|
||||
"messages": [{"role": "user", "content": "secret-prompt-marker"}],
|
||||
"error_str": "Error: secret-prompt-marker",
|
||||
"error_information": _error_information(),
|
||||
"status": "failure",
|
||||
}
|
||||
redacted = redacted_standard_logging_payload(payload)
|
||||
assert "secret-prompt-marker" not in str(redacted)
|
||||
assert redacted["error_str"] == "redacted-by-litellm"
|
||||
assert redacted["error_information"] == {
|
||||
"error_code": "400",
|
||||
"error_class": "BadRequestError",
|
||||
"llm_provider": "openai",
|
||||
"traceback": "redacted-by-litellm",
|
||||
"error_message": "redacted-by-litellm",
|
||||
"error_rate_limit_category": None,
|
||||
"normalized_error": None,
|
||||
}
|
||||
|
||||
def test_perform_redaction_covers_traceback_exception(self):
|
||||
details = {
|
||||
"litellm_params": {},
|
||||
"traceback_exception": "Traceback ... secret-prompt-marker",
|
||||
"standard_logging_object": {"error_str": "secret-prompt-marker"},
|
||||
}
|
||||
perform_redaction(details, None)
|
||||
assert "secret-prompt-marker" not in str(details)
|
||||
assert details["traceback_exception"] == "redacted-by-litellm"
|
||||
assert details["standard_logging_object"]["error_str"] == "redacted-by-litellm"
|
||||
|
||||
def test_perform_redaction_leaves_empty_traceback_exception_alone(self):
|
||||
details = {"litellm_params": {}, "traceback_exception": ""}
|
||||
perform_redaction(details, None)
|
||||
assert details["traceback_exception"] == ""
|
||||
|
||||
|
||||
def _request_data(metadata=None, litellm_metadata=None, turn_off_message_logging=None):
|
||||
data = {"metadata": metadata if metadata is not None else {}}
|
||||
if litellm_metadata is not None:
|
||||
data["litellm_metadata"] = litellm_metadata
|
||||
if turn_off_message_logging is not None:
|
||||
data["turn_off_message_logging"] = turn_off_message_logging
|
||||
return data
|
||||
|
||||
|
||||
class TestShouldRedactFailedRequest:
|
||||
def test_global_on(self, monkeypatch):
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_failed_request
|
||||
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", True)
|
||||
assert should_redact_failed_request(_request_data()) is True
|
||||
|
||||
def test_global_off(self):
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_failed_request
|
||||
|
||||
assert should_redact_failed_request(_request_data()) is False
|
||||
|
||||
def test_enable_header_in_metadata(self):
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_failed_request
|
||||
|
||||
request_data = _request_data(metadata={"headers": {"x-litellm-enable-message-redaction": "true"}})
|
||||
assert should_redact_failed_request(request_data) is True
|
||||
|
||||
def test_disable_header_overrides_global_on(self, monkeypatch):
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_failed_request
|
||||
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", True)
|
||||
request_data = _request_data(metadata={"headers": {"litellm-disable-message-redaction": "true"}})
|
||||
assert should_redact_failed_request(request_data) is False
|
||||
|
||||
def test_enable_header_in_litellm_metadata(self):
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_failed_request
|
||||
|
||||
request_data = _request_data(litellm_metadata={"headers": {"x-litellm-enable-message-redaction": "true"}})
|
||||
assert should_redact_failed_request(request_data) is True
|
||||
|
||||
def test_dynamic_param_true(self):
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_failed_request
|
||||
|
||||
assert should_redact_failed_request(_request_data(turn_off_message_logging=True)) is True
|
||||
|
||||
def test_dynamic_param_false_overrides_global_on(self, monkeypatch):
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_failed_request
|
||||
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", True)
|
||||
assert should_redact_failed_request(_request_data(turn_off_message_logging=False)) is False
|
||||
|
||||
@pytest.mark.parametrize("headers", [None, "x", ["a"]], ids=["none", "str", "list"])
|
||||
@pytest.mark.parametrize("global_on", [True, False], ids=["global_on", "global_off"])
|
||||
def test_non_mapping_headers_follow_the_global_setting(self, monkeypatch, headers, global_on):
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_failed_request
|
||||
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", global_on)
|
||||
assert should_redact_failed_request(_request_data(metadata={"headers": headers})) is global_on
|
||||
|
||||
def test_non_mapping_headers_keep_the_dynamic_param(self, monkeypatch):
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_failed_request
|
||||
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", True)
|
||||
request_data = _request_data(metadata={"headers": "x"}, turn_off_message_logging=False)
|
||||
assert should_redact_failed_request(request_data) is False
|
||||
|
||||
def test_dynamic_param_in_metadata_slot(self):
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_failed_request
|
||||
|
||||
assert should_redact_failed_request(_request_data(metadata={"turn_off_message_logging": True})) is True
|
||||
|
||||
def test_dynamic_param_in_litellm_metadata_slot(self):
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_failed_request
|
||||
|
||||
assert should_redact_failed_request(_request_data(litellm_metadata={"turn_off_message_logging": True})) is True
|
||||
|
||||
def test_top_level_dynamic_param_beats_metadata_slot(self, monkeypatch):
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_failed_request
|
||||
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", True)
|
||||
request_data = _request_data(
|
||||
metadata={"turn_off_message_logging": True},
|
||||
turn_off_message_logging=False,
|
||||
)
|
||||
assert should_redact_failed_request(request_data) is False
|
||||
|
||||
|
||||
class TestRequestOptsIntoMessageRedaction:
|
||||
@pytest.mark.parametrize("header", ["litellm-enable-message-redaction", "x-litellm-enable-message-redaction"])
|
||||
def test_enable_header(self, header: str) -> None:
|
||||
from litellm.litellm_core_utils.redact_messages import request_opts_into_message_redaction
|
||||
|
||||
assert request_opts_into_message_redaction({header: "true"}, {}) is True
|
||||
|
||||
def test_top_level_dynamic_param(self) -> None:
|
||||
from litellm.litellm_core_utils.redact_messages import request_opts_into_message_redaction
|
||||
|
||||
assert request_opts_into_message_redaction({}, {"turn_off_message_logging": True}) is True
|
||||
|
||||
def test_metadata_slot_dynamic_param(self) -> None:
|
||||
from litellm.litellm_core_utils.redact_messages import request_opts_into_message_redaction
|
||||
|
||||
assert request_opts_into_message_redaction({}, {"metadata": {"turn_off_message_logging": True}}) is True
|
||||
|
||||
def test_empty_inputs(self) -> None:
|
||||
from litellm.litellm_core_utils.redact_messages import request_opts_into_message_redaction
|
||||
|
||||
assert request_opts_into_message_redaction({}, {}) is False
|
||||
|
||||
def test_disable_header_alone_is_not_an_opt_in(self) -> None:
|
||||
from litellm.litellm_core_utils.redact_messages import request_opts_into_message_redaction
|
||||
|
||||
assert request_opts_into_message_redaction({"litellm-disable-message-redaction": "true"}, {}) is False
|
||||
|
||||
def test_dynamic_param_false(self) -> None:
|
||||
from litellm.litellm_core_utils.redact_messages import request_opts_into_message_redaction
|
||||
|
||||
assert request_opts_into_message_redaction({}, {"turn_off_message_logging": False}) is False
|
||||
|
||||
@pytest.mark.parametrize("body", ([1], "x", None, 123, True))
|
||||
def test_non_object_body_is_not_an_opt_in(self, body: object) -> None:
|
||||
from litellm.litellm_core_utils.redact_messages import request_opts_into_message_redaction
|
||||
|
||||
assert request_opts_into_message_redaction({}, body) is False
|
||||
assert request_opts_into_message_redaction({"x-litellm-enable-message-redaction": "true"}, body) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("callback_only", (False, True))
|
||||
@pytest.mark.parametrize("with_standard_payload", (False, True))
|
||||
def test_baseline_snapshots_are_redacted_without_mutating_request_state(
|
||||
|
|
|
|||
|
|
@ -3666,3 +3666,57 @@ async def test_async_post_call_failure_hook_disable_pre_routing_error_logs(
|
|||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
assert len(writer.rows) == rows_written
|
||||
|
||||
|
||||
SECRET = "secret-prompt-marker"
|
||||
|
||||
|
||||
def _logger_and_writer():
|
||||
writer = MagicMock()
|
||||
writer.update_database = AsyncMock()
|
||||
logger = ProxyDBLogger(spend_writer=lambda: writer)
|
||||
return logger, writer
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_hook_redacts_persisted_error_information(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", True)
|
||||
logger, writer = _logger_and_writer()
|
||||
|
||||
request_data = {"metadata": {}}
|
||||
await logger.async_post_call_failure_hook(
|
||||
request_data=request_data,
|
||||
original_exception=litellm.BadRequestError(
|
||||
message=f"Unsupported content: {SECRET}", model="gpt-4o", llm_provider="openai"
|
||||
),
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
traceback_str=f"Traceback ... {SECRET} ...",
|
||||
)
|
||||
|
||||
persisted = request_data["litellm_params"]["metadata"]["error_information"]
|
||||
assert SECRET not in json.dumps(persisted)
|
||||
assert persisted["error_message"] == "redacted-by-litellm"
|
||||
assert persisted["traceback"] == "redacted-by-litellm"
|
||||
assert persisted["error_class"] == "BadRequestError"
|
||||
assert persisted["error_code"] == "400"
|
||||
writer.update_database.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_hook_leaves_error_information_alone_when_redaction_off(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", False)
|
||||
logger, writer = _logger_and_writer()
|
||||
|
||||
request_data = {"metadata": {}}
|
||||
await logger.async_post_call_failure_hook(
|
||||
request_data=request_data,
|
||||
original_exception=litellm.BadRequestError(
|
||||
message=f"Unsupported content: {SECRET}", model="gpt-4o", llm_provider="openai"
|
||||
),
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
traceback_str="trace",
|
||||
)
|
||||
|
||||
persisted = request_data["litellm_params"]["metadata"]["error_information"]
|
||||
assert SECRET in persisted["error_message"]
|
||||
writer.update_database.assert_awaited_once()
|
||||
|
|
|
|||
|
|
@ -2878,6 +2878,57 @@ async def test_handle_logging_proxy_only_error_syncs_normalized_call_type(
|
|||
assert logging_obj.model_call_details["call_type"] == expected_call_type
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_logging_proxy_only_error_strips_excluded_fields_for_custom_logger(
|
||||
monkeypatch,
|
||||
):
|
||||
"""A guardrail block logged through the proxy-only failure path hands a
|
||||
CustomLogger a standard_logging_object without the excluded fields."""
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
class _CapturingLogger(CustomLogger):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.failure_kwargs = None
|
||||
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
self.failure_kwargs = kwargs
|
||||
|
||||
capture = _CapturingLogger()
|
||||
monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", ["metadata", "messages"])
|
||||
monkeypatch.setattr(litellm, "callbacks", [capture])
|
||||
monkeypatch.setattr(litellm, "_async_failure_callback", [capture])
|
||||
monkeypatch.setattr(litellm, "failure_callback", [])
|
||||
|
||||
with patch("litellm.proxy.utils.threading.Thread") as mock_thread:
|
||||
mock_thread.return_value.start = Mock()
|
||||
await ProxyLogging(user_api_key_cache=DualCache())._handle_logging_proxy_only_error(
|
||||
request_data={
|
||||
"model": "gpt-4o",
|
||||
"messages": [{"role": "user", "content": "please zebrablock this"}],
|
||||
"metadata": {"tags": ["sentinel"]},
|
||||
},
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
api_key="test_key",
|
||||
token="test_token",
|
||||
request_route="/v1/chat/completions",
|
||||
),
|
||||
route="/v1/chat/completions",
|
||||
original_exception=HTTPException(
|
||||
status_code=400,
|
||||
detail={"error": "Content blocked: keyword 'zebrablock' detected"},
|
||||
),
|
||||
)
|
||||
|
||||
assert capture.failure_kwargs is not None
|
||||
payload = capture.failure_kwargs["standard_logging_object"]
|
||||
assert payload["status"] == "failure"
|
||||
assert "metadata" not in payload
|
||||
assert "messages" not in payload
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_during_call_hook_parallel_execution():
|
||||
"""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue